Merge pull request #889 from amark/dev

woops THIS is CLOSER to the BIG 2020 RELEASE, don't use old one :P

lib/evict.js

@@ -16,7 +16,8 @@
 		function check(){
 			var used = util().rss / 1024 / 1024;
 			var hused = heap().used_heap_size / 1024 / 1024;
-			if(hused < ev.max && used < ev.max){ return }
+			//if(hused < ev.max && used < ev.max){ return }
+			if(used < ev.max){ return }
 			console.LOG && Gun.log('evict memory:', hused.toFixed(), used.toFixed(), ev.max.toFixed());
 			GC();//setTimeout(GC, 1);
 		}

lib/radisk.js

@@ -108,47 +108,84 @@
 			6. Merge and write all of those to the in-memory file and back to disk.
 			7. If file too large, split. More details needed here.
 		*/
+		/* NEW APPROACH:
+			1. For each item in radix memory
+			2. Add it to a radix bucket corresponding to directory of files
+			3. Iterate over each bucket
+			4. Resume old approach.
+		*/
 		r.save = function(rad, cb){
-			var s = function Span(){};
-			s.find = function(tree, key){
-				if(key < s.start){ return }
-				s.start = key;
-				r.list(s.lex);
-				return true;
+			if(r.save.ing){
+				r.save.ing.push({rad: rad, ack: cb});
+				return;
 			}
-			s.lex = function(file){
-				file = (u === file)? u : decodeURIComponent(file);
-				if(!file || file > s.start){
-					s.mix(s.file || opt.code.from, s.start, s.end = file, s.file);
+			//console.only(99); var ID = Gun.text.random(2), S = (+new Date); console.log("[[[[[[[[", ID);
+			r.save.ing = [];
+			var ack = cb;
+			var s = function Span(err, ok){
+				var tmp = r.save.ing;
+				//console.only(99); var TMP; console.log("]]]]]]]]", ID, (TMP = +new Date) - S, 'more?', !!tmp);
+				r.save.ing = null;
+				map(tmp, function(q){ // if many, not the most efficient to requeue, but works for now.
+					if(!q || !q.rad || !q.ack){ return }
+					r.save(q.rad, q.ack);
+				})
+				ack(err, ok);
+			};
+			cb = s;
+			s.files = {};
+			s.i = 0; // TODO: revise? Using counter for critical path not my favorite.
+			s.place = function(tree, key){
+				var go = function(file, last){
+					file = decodeURIComponent(file || last || opt.code.from);
+					(s.files[file] || (s.files[file] = Radix()))(key, tree);
+					if(!(--s.i)){ s.go() } // TODO: See above, revise?
 					return true;
 				}
-				s.file = file;
+				go.reverse = 1;
+				go.end = key;
+				r.list(go);
+				++s.i; // TODO: See above, revise?
 			}
-			s.mix = function(file, start, end, f){
-				s.start = s.end = s.file = u;
+			s.go = function(){
+				if(s.gone){ return } s.gone = true;
+				s.seq = [];
+				map(s.files, function(mem, file){ s.seq.push({file: file, mem: mem}) });
+				s.files = null;
+				s.c = 0;
+				s.merge(s.c);
+			}
+			s.merge = function(i){
+				i = i || 0;
+				//var at = s.seq[i];
+				var at = s.seq.shift();
+				if(!at){
+					if(s.ok){ return cb(null, s.ok) }
+					return cb("No file to save data to.");
+				}
+				var file = at.file, mem = at.mem;
 				r.parse(file, function(err, disk){
-					if(err){ return cb(err) }	
-					if(!disk && f){ // corrupt file?
-						r.list.bad(f); // remove from dir list
+					if(err){ return cb(err) }
+					if(!disk && file !== opt.code.from){ // corrupt file?
+						r.list.bad(file); // remove from dir list
 						r.save(rad, cb); // try again
 						return;
 					}
 					disk = disk || Radix();
-					Radix.map(rad, function(val, key){
-						if(key < start){ return }
-						if(end && end < key){ return s.start = key }
+					Radix.map(mem, function(val, key){
 						// PLUGIN: consider adding HAM as an extra layer of protection
 						disk(key, val); // merge batch[key] -> disk[key]
 					});
-					r.write(file, disk, s.next);
-				});
+					r.write(file, disk, s.pop);
+				})
 			}
-			s.next = function(err, ok){
-				if(s.err = err){ return cb(err) }
-				if(s.start){ return Radix.map(rad, s.find) }
-				cb(err, ok);
+			s.pop = function(err, ok){
+				if(s.err = err || s.err){ return cb(err) }
+				s.ok = ok || s.ok || 1;
+				s.merge(++s.c);
 			}
-			Radix.map(rad, s.find);
+			Radix.map(rad, s.place);
+			if(!s.i){ s.go() }; // TODO: See above, revise?
 		}
 
 		/*
@@ -253,7 +290,6 @@
 					file = (u === file)? u : decodeURIComponent(file);
 					tmp = o.next || key || (o.reverse? o.end || '\uffff' : o.start || '');
 					if(!file || (o.reverse? file < tmp : file > tmp)){
-						console.only(2, 'RAD LEX:', JSON.stringify(tmp), 'in', JSON.stringify(g.file), !file, JSON.stringify(file) > JSON.stringify(tmp), file > tmp, JSON.stringify(file), o);
 						LOG && opt.log(S, +new Date - S, 'rad read lex'); S = +new Date;
 						if(o.next || o.reverse){ g.file = file }
 						if(tmp = Q[g.file]){
@@ -265,7 +301,7 @@
 							g.it(null, u, {});
 							return true; 
 						}
-						r.parse(g.file, g.it);
+						r.parse(g.file, g.check);
 						return true;
 					}
 					g.file = file;
@@ -308,6 +344,49 @@
 					o.next = as.file;
 					r.read(key, as.ack, o);
 				}
+				g.check = function(err, disk, info){
+					g.it(err, disk, info);
+					var good = true;
+					Radix.map(disk, function(val, key){
+						// assume in memory for now, since both write/read already call r.list which will init it.
+						var go = function(file){
+							if(info.file !== file){
+								good = false
+							}
+							return true;
+						}
+						go.reverse = 1;
+						go.end = key;
+						r.list(go);
+					});
+					if(good){ return }
+					var id = Gun.text.random(3); console.log("MISLOCATED DATA", id);
+					r.save(disk, function ack(err, ok){
+						if(err){ return r.save(disk, ack) } // ad infinitum???
+						console.log("MISLOCATED CORRECTED", id);
+					});
+				}
+				/*g.check2 = function(err, disk, info){
+					if(err || !disk){ return g.it(err, disk, info) }
+					var good = true;
+					Radix.map(disk, function(val, key){
+						// assume in memory for now, since both write/read already call r.list which will init it.
+						var go = function(file){
+							if(info.file !== file){ good = false }
+							return true;
+						}
+						go.reverse = 1;
+						go.end = key;
+						r.list(go);
+					});
+					if(good){ return g.it(err, disk, info) }
+					var id = Gun.text.random(3); console.log("MISLOCATED DATA", id);
+					r.save(disk, function ack(err, ok){
+						if(err){ return r.save(disk, ack) } // ad infinitum???
+						console.log("MISLOCATED CORRECTED", id);
+						r.read(key, cb, o);
+					});
+				}*/
 				if(o.reverse){ g.lex.reverse = true }
 				LOG && (S = +new Date);
 				r.list(g.lex);
@@ -327,7 +406,7 @@
 			var Q = {}, s = String.fromCharCode(31);
 			r.parse = function(file, cb, raw){ var q;
 				if(q = Q[file]){ return q.push(cb) } q = Q[file] = [cb];
-				var p = function Parse(){}, info = {};
+				var p = function Parse(){}, info = {file: ename(file)};
 				p.disk = Radix();
 				p.read = function(err, data){ var tmp;
 					LOG && opt.log(S, +new Date - S, 'read disk', ename(file));
@@ -414,11 +493,11 @@
 			var dir, q, f = String.fromCharCode(28), ef = ename(f);
 			r.list = function(cb){
 				if(dir){
-					var tmp = {reverse: (cb.reverse)? 1 : 0};
+					var last, tmp = {reverse: (cb.reverse)? 1 : 0, start: cb.start, end: cb.end};
 					Radix.map(dir, function(val, key){
 						if(!val){ return }
-						return cb(key);
-					}, tmp) || cb();
+						return cb(last = key);
+					}, tmp) || cb(u, last);
 					return;
 				}
 				if(q){ return q.push(cb) } q = [cb];
@@ -450,7 +529,7 @@
 				}
 				if(disk){
 					r.list.drain(disk);
-					//return;
+					return;
 				}
 				if(!opt.store.list){
 					r.list.drain(Radix());

lib/radix.js

@@ -58,21 +58,26 @@
 		//var keys = Object.keys(t).sort();
 		opt = (true === opt)? {branch: true} : (opt || {});
 		if(rev = opt.reverse){ keys = keys.slice().reverse() }
-		var start = opt.start, end = opt.end;
+		var start = opt.start, end = opt.end, END = '\uffff';
 		var i = 0, l = keys.length;
 		for(;i < l; i++){ var key = keys[i], tree = t[key], tmp, p, pt;
 			if(!tree || '' === key || _ === key){ continue }
 			p = pre.slice(); p.push(key);
 			pt = p.join('');
 			if(u !== start && pt < (start||'').slice(0,pt.length)){ continue }
-			if(u !== end && (end || '\uffff') < pt){ continue }
+			if(u !== end && (end || END) < pt){ continue }
 			if(rev){ // children must be checked first when going in reverse.
 				tmp = map(tree, cb, opt, p);
 				if(u !== tmp){ return tmp }
 			}
 			if(u !== (tmp = tree[''])){
-				tmp = cb(tmp, pt, key, pre);
-				if(u !== tmp){ return tmp }
+				var yes = 1;
+				if(u !== start && pt < (start||'')){ yes = 0 }
+				if(u !== end && pt > (end || END)){ yes = 0 }
+				if(yes){
+					tmp = cb(tmp, pt, key, pre);
+					if(u !== tmp){ return tmp }
+				}
 			} else
 			if(opt.branch){
 				tmp = cb(u, pt, key, pre);

lib/rfs.js

@@ -56,6 +56,14 @@ function Store(opt){
 			}
 		});
 	};
+
+	store.list = function(cb, match, params, cbs){
+		var dir = fs.readdirSync(opt.file);
+		dir.forEach(function(file){
+			cb(file);
+		})
+		cb();
+	};
 	
 	return store;
 }

lib/server.js

@@ -2,7 +2,7 @@
 	var Gun = require('../gun'), u;
 	Gun.serve = require('./serve');
 	//process.env.GUN_ENV = process.env.GUN_ENV || 'debug';
-	//console.LOG = true; // only do this for dev.
+	console.LOG = true; // only do this for dev.
 	Gun.on('opt', function(root){
 		if(u === root.opt.super){ root.opt.super = true }
 		if(u === root.opt.faith){ root.opt.faith = true } // HNPERF: This should probably be off, but we're testing performance improvements, please audit.

lib/store.js

@@ -79,11 +79,16 @@ Gun.on('create', function(root){
         }
         if(has['-'] || (soul||{})['-']){ o.reverse = true }
         if((tmp = (root.next||empty)[soul]) && tmp.put){
+            var SPUT = tmp.put;
             if(o.atom){
                 tmp = (tmp.next||empty)[o.atom] ;
-                if(tmp && tmp.rad){ return }
+                if(tmp && tmp.rad){
+                    LOG && Gun.log("still cached atom", JSON.stringify(get), Object.keys(SPUT||{}).length);
+                    return;
+                }
             } else
             if(tmp && tmp.rad){
+                LOG && Gun.log("still cached", JSON.stringify(get), Object.keys(SPUT||{}).length);
                 return;
             }
         }

package.json

@@ -1,6 +1,6 @@
 {
   "name": "gun",
-  "version": "0.2020.115",
+  "version": "0.2020.116",
   "description": "A realtime, decentralized, offline-first, graph data synchronization engine.",
   "main": "index.js",
   "browser": "browser.js",