radix prototype

gun.js

@@ -74,7 +74,7 @@
 		Type.list.index = 1; // change this to 0 if you want non-logical, non-mathematical, non-matrix, non-convenient array notation
 		Type.obj = {is: function(o){ return o? (o instanceof Object && o.constructor === Object) || Object.prototype.toString.call(o).match(/^\[object (\w+)\]$/)[1] === 'Object' : false }}
 		Type.obj.put = function(o, f, v){ return (o||{})[f] = v, o }
-		Type.obj.has = function(o, f){ return o && Object.prototype.hasOwnProperty.call(o, f) }
+		Type.obj.has = function(o, f){ return o && Object.hasOwnProperty(o, f) }
 		Type.obj.del = function(o, k){
 			if(!o){ return }
 			o[k] = null;
@@ -126,7 +126,7 @@
 				var u, i = 0, x, r, ll, lle, f = fn_is(c);
 				t.r = null;
 				if(keys && obj_is(l)){
-					ll = Object.keys(l); lle = true;
+					ll = keys(l); lle = true;
 				}
 				if(list_is(l) || ll){
 					x = (ll || l).length;
@@ -995,6 +995,9 @@
 					if(!obj_is(at.opt.peers)){ at.opt.peers = {}}
 					at.opt.peers = obj_to(tmp, at.opt.peers);
 				}
+				at.opt.uuid = at.opt.uuid || function(){ 
+					return state().toString(36).replace('.','') + text_rand(12);
+				}
 				at.opt.wsc = at.opt.wsc || {protocols:[]} 
 				at.opt.peers = at.opt.peers || {};
 				obj_to(opt, at.opt); // copies options on to `at.opt` only if not already taken.
@@ -1003,10 +1006,10 @@
 			}
 		}());
 
-		var text_is = Gun.text.is;
 		var list_is = Gun.list.is;
+		var text = Gun.text, text_is = text.is, text_rand = text.random;
 		var obj = Gun.obj, obj_is = obj.is, obj_has = obj.has, obj_to = obj.to, obj_map = obj.map, obj_copy = obj.copy;
-		var _soul = Gun._.soul, _field = Gun._.field, rel_is = Gun.val.rel.is;
+		var state = Gun.state, _soul = Gun._.soul, _field = Gun._.field, rel_is = Gun.val.rel.is;
 		var empty = {}, u;
 
 		console.debug = function(i, s){ return (console.debug.i && i === console.debug.i && console.debug.i++) && (console.log.apply(console, arguments) || s) };
@@ -1066,7 +1069,7 @@
 		// language, consider implementing an easier API to build.
 		var Gun = require('./root');
 		Gun.chain.chain = function(){
-			var at = this._, chain = new this.constructor(this), cat = chain._;
+			var at = this._, chain = new this.constructor(this), cat = chain._, root;
 			cat.root = root = at.root;
 			cat.id = ++root._.once;
 			cat.back = this;
@@ -2003,12 +2006,11 @@
 
 		Gun.on('opt', function(at){
 			var opt = at.opt, peers = opt.peers;
-			var prefix = opt.file || opt.prefix || 'gun/';
+			opt.file = opt.file || opt.prefix || 'gun/'; // support old option name.
-			var wait = opt.wait || 1, batch = opt.batch || 10000;
 			this.to.next(at);
 			if(at.once){ return }
 			if(false === at.opt.file){ return }
-			var graph = at.graph, acks = {}, async = {}, count = 0, to;
+			var graph = at.graph, acks = {}, batch = {}, count = 0, to;
 			
 			at.on('put', function(at){
 				this.to.next(at);
@@ -2016,12 +2018,11 @@
 				if(!at['@']){ acks[at['#']] = 1; } // only ack non-acks.
 				count += 1;
 				console.log('lS PUT', Gun.obj.copy(async));
-				if(count >= batch){
+				if(count >= opt.batch){
 					return flush();
 				}
 				if(to){ return }
-				clearTimeout(to);
+				to = setTimeout(flush, opt.wait);
-				to = setTimeout(flush, wait);
 			});
 
 			function map(node, soul){
@@ -2042,7 +2043,7 @@
 					// the data that the user is currently interested in.
 					node = graph[soul] || all[soul] || node;
 					console.log("WRITE:", Gun.obj.copy(node));
-						try{store.setItem(prefix + soul, JSON.stringify(node));
+					try{store.setItem(opt.file + soul, JSON.stringify(node));
 					}catch(e){ err = e || "localStorage failure" }
 				});
 				if(!err && !Gun.obj.empty(peers)){ return } // only ack if there are no peers.
@@ -2062,7 +2063,7 @@
 				if(!lex || !(soul = lex[Gun._.soul])){ return }
 				//if(0 >= at.cap){ return }
 				var field = lex['.'];
-				data = async[soul] || Gun.obj.ify(store.getItem(prefix + soul) || null) || async[soul] || u;
+				data = async[soul] || Gun.obj.ify(store.getItem(opt.file + soul) || null) || async[soul] || u;
 				console.debug(2, 'lS GET', data, async[soul]);
 				if(data && field){
 					data = Gun.state.to(data, field);

lib/radisk.js

@@ -0,0 +1,196 @@
+var fs = require('fs');
+var Gun = require('../gun');
+var Radix = require('./radix');
+
+function Radisk(opt){
+	/*
+		Any and all storage adapters should...
+		1. If not busy, write to disk immediately.
+		2. If busy, batch to disk. (Improves performance, reduces potential disk corruption)
+		3. If a batch exceeds a certain number of writes, force to disk. (This caps total performance, but reduces potential loss)
+	*/
+	var radisk = function(key, val, cb){
+		if(val instanceof Function){
+			cb = val;
+			val = radisk.batch(key);
+			if(u !== val){
+				return cb(null, val);
+			}
+			if(radisk.was){
+				val = radisk.was(key);
+				if(u !== val){
+					return cb(null, val);
+				}
+			}
+			console.log("READ FROM DISK");
+			return cb(null, val);
+		}
+		radisk.batch(key, val);
+		if(cb){ radisk.batch.acks.push(cb) }
+		if(!count++){ return thrash() } // (1)
+		if(opt.batch <= count){ return thrash() } // (3)
+		clearTimeout(to); // (2)
+		to = setTimeout(thrash, opt.wait);
+	};
+	radisk.batch = Radix();
+	radisk.batch.acks = [];
+	var count = 0, wait, to, u;
+
+	opt = opt || {};
+	opt.file = opt.file || 'radata';
+	opt.size = opt.size || (1024 * 1024 * 10); // 10MB
+	opt.batch = opt.batch || 10 * 1000;
+	opt.wait = opt.wait || 1;
+	opt.nest = opt.nest || ' ';
+
+	console.log("Warning: Radix storage engine has not been tested with all types of values and keys yet.");
+  if(!fs.existsSync(opt.file)){ fs.mkdirSync(opt.file) }
+
+	var thrash = function(){
+		if(wait){ return }
+		clearTimeout(to);
+		wait = true;
+		var was = radisk.was = radisk.batch;
+		radisk.batch = null;
+		radisk.batch = Radix();
+		radisk.batch.acks = [];
+		chunk(radisk.was, function(err, ok){
+			radisk.was = null;
+			wait = false;
+			var tmp = count;
+			count = 0;
+			Gun.obj.map(was.acks, function(cb){cb(err, ok)});
+			if(1 < tmp){ thrash() }
+		});
+	}
+
+	/*
+		1. Find the first radix item in memory.
+		2. Use that as the starting index in the directory of files.
+		3. Find the first file that is lexically larger than it,
+		4. Read the previous file to that into memory
+		5. Scan through the in memory radix for all values lexically less than the limit.
+		6. Merge and write all of those to the in-memory file and back to disk.
+		7. If file to large, split. More details needed here.
+	*/
+	function chunk(radix, cb){
+		var step = {
+			check: function(tree, key){
+				if(key < step.start){ return }
+				step.start = key;
+				fs.readdir(opt.file, step.match);
+				return true;
+			},
+			match: function(err, dir){
+				step.dir = dir;
+				if(!dir.length){
+					step.file = '0';
+					return step.merge(null, Radix());
+				}
+				Gun.obj.map(dir, step.lex);
+				read(step.file, step.merge);
+			},
+			lex: function(file){
+				if(file > step.start){
+					return step.end = file;
+				}
+				step.file = file;
+			},
+			merge: function(err, disk){
+				if(err){ return console.log("ERROR!!!", err) }
+				step.disk = disk;
+				Radix.map(radix, step.add);
+				write(step.file, step.disk, step.done);
+			},
+			add: function(val, key){
+				if(key < step.start){ return }
+				if(step.end && step.end < key){ return step.next = key; }
+				step.disk(key, val);
+			},
+			done: function(err){
+				if(err){ console.log("ERROR!!!", err) }
+				if(!step.next){
+					return cb(err);
+				}
+				step.start = step.next;
+				step.end = step.next = step.file = u;
+				Radix.map(radix, step.check);
+			}
+		}
+		Radix.map(radix, step.check);
+	}
+
+	var write = function(file, radix, cb){
+		var step = {
+			rest: "",
+			count: 0,
+			file: file,
+			each: function(val, key, k, pre){
+				step.count++;
+				if(opt.size < step.rest.length){
+					step.rest = "";
+					step.limit = Math.ceil(step.count/2);
+					step.count = 0;
+					step.sub = Radix();
+					Radix.map(radix, step.slice);
+					return true;
+				}
+				var i = pre.length;
+				while(i--){ step.rest += opt.nest };
+				step.rest += k + (u === val? '' : '=' + val) + '\n';
+			},
+			dump: function(){
+				var rest = step.rest;
+				step.rest = "";
+				fs.writeFile(opt.file +'/'+ file, rest, cb);
+			},
+			slice: function(val, key){
+				if(key < step.file){ return }
+				if(step.limit < (++step.count)){
+					var name = step.file;
+					step.file = key;
+					step.count = 0;
+					write(name, step.sub, step.next);
+					return true;
+				}
+				step.sub(key, val);
+			},
+			next: function(err){
+				if(err){ console.log("ERR!!!!") }
+				step.sub = Radix();
+				if(!Radix.map(radix, step.slice)){
+					write(step.file, step.sub, cb);
+				}
+			}
+		};
+		if(!Radix.map(radix, step.each, true)){ step.dump() }
+	}
+
+	var read = function(file, cb){
+		var step = {
+			nest: 0,
+			rad: Radix(),
+			data: function(err, data){
+				if(err){ return console.log("ERROR READING FILE!", err) }
+				step.pre = [];
+				Gun.obj.map(data.toString().split('\n'), step.split); // TODO: Escape!
+				cb(null, step.rad);
+			},
+			split: function(line){ var LINE = line;
+				var nest = -1; while(opt.nest === line[++nest]){};
+				if(nest){ line = line.slice(nest) }
+				if(nest <= step.nest){ step.pre = step.pre.slice(0, nest - step.nest - 1) }
+				line = line.split('='); step.pre.push(line[0]);
+				if(1 < line.length){ step.rad(step.pre.join(''), line[1]) }
+				step.nest = nest;
+			}
+		}
+		fs.readFile(opt.file +'/'+ file, step.data);
+	}
+
+	radisk.read = read;
+
+	return radisk;
+}
+
+module.exports = Radisk;

lib/radix.js

@@ -1,10 +1,9 @@
-var fs = require('fs');
+var Gun = require('../gun');
-var Gun = require('gun');
 var gbm = Gun.obj.map, no = {}, u;
 
 function Radix(){
 	var radix = function(key, val, t){
-		t = t || tree;
+		t = t || radix._ || (radix._ = {});
 		var i = 0, l = key.length-1, k = key[i], at, tmp;
 		while(!(at = t[k]) && i < l){
 			k += key[++i];
@@ -36,75 +35,31 @@ function Radix(){
 			if(u === val){ return (u === (tmp = at.$))? at._ : tmp }
 			at.$ = val;
 		} else {
-			return radix(key.slice(++i), val, at._);
+			return radix(key.slice(++i), val, at._ || (at._ = {}));
 		}
 	}
-	var tree = radix._ = {};
 	return radix;
-}
-
-
-
-(function(){
-	var radix = Radix();
-	radix('user/marknadal', 'asdf');
-	radix('user/ambercazzell', 'dafs');
-	radix('user/taitforrest', 'sadf');
-	radix('user/taitveronika', 'fdsa');
-	radix('user/marknadal', 'foo');
-	radix('user/table', 'foo');
-	console.log("__________________");
-	console.log(radix._);
-	console.log(radix('user/table'));
-	//fs.writeFileSync('radix-data.json', JSON.stringify(radix._, null, 2));
-}());
-(function(){return;
-	var radix = Radix();
-	var gtr = Gun.text.random;
-	var c = 0, l = 10;
-	function bench(){
-		if(c > 1000){ return clearInterval(it), console.log(radix._, Object.keys(radix._).length), fs.writeFileSync('radix-data.json', JSON.stringify(radix._, null, 2)); }
-		for(var i = 0; i < l; i++){
-			radix(gtr(7), c++);
-		}
-		//if(c % 1000 === 0){ console.log(radix._) }
-	}
-	var it = setInterval(bench, 1);
-}());
-
-function Radisk(){
-	var radix = Radix();
-	var radisk = function(key, val, cb){
-		if(val instanceof Function){
-			cb = val;
-			val = radix(key);
-			if(u !== val){
-				return cb(null, val);
-			}
-			return console.log("read from disk");
-		}
-		radix(key, val);
-		if(!to){ return flush(to = true) }
-		clearTimeout(to);
-		to = setTimeout(flush, opt.wait)
 };
-	var opt = {file: 'radix', size: 1024 * 1024 * 2, wait: 10 * 1000, batch: 10 * 1000}, to;
+;(function(){
+	Radix.map = function map(radix, cb, opt, pre){ pre = pre || []; 
+		var _ = radix._ || radix, keys = Object.keys(_).sort(), i = 0, l = keys.length;
+		for(;i < l; i++){ var key = keys[i], tree = _[key], tmp;
+			if(u !== (tmp = tree.$)){
+				tmp = cb(tmp, pre.join('') + key, key, pre);
+				if(u !== tmp){ return tmp }
+			} else 
+			if(opt){
+				cb(u, pre.join(''), key, pre);
+			}
+			if(tmp = tree._){
+				pre.push(key);
+				tmp = map(tmp, cb, opt, pre);
+				if(u !== tmp){ return tmp }
+				pre.pop();
+			}
+		}
+	};
+	Object.keys = Object.keys || function(o){ return gbm(o, function(v,k,t){t(k)}) }
+}());
 
-	var flush = function(){
+module.exports = Radix;
-
-	}
-	var read = function(path){
-		return fs.readFileSync(nodePath.join(dir, path)).toString();
-	}
-
-	var write = function(path, data){
-		return fs.writeFileSync(nodePath.join(dir, path), data);
-	}
-
-	var mk = function(path){
-		path = nodePath.join(dir, path);
-	  if(fs.existsSync(path)){ return }
-		fs.mkdirSync(path);
-	}
-	return radisk;
-}

lib/radix_.js

@@ -1,104 +0,0 @@
-function radix(r){
-	r = r || {};
-	var u, n = null, c = 0;
-	function get(p){
-		var v = match(p, r);
-		return v;
-	}
-	function match(path, tree, v){
-		if(!Gun.obj.map(tree, function(val, key){
-			if(key[0] !== path[0]){ return }
-			var i = 1;
-			while(key[i] === path[i] && path[i]){ i++ }
-			if(key = key.slice(i)){ // recurse
-				console.log("match", key, i)
-				v = {sub: tree, pre: path.slice(0, i), val: val, post: key, path: path.slice(i) };
-			} else { // replace
-				console.log("matching", path, key, i);
-				v = match(path.slice(i), val);
-			}
-			return true;
-		})){ console.log("matched", tree, path); v = {sub: tree, path: path} } // insert
-		return v;
-	}
-	function rebalance(ctx, val){
-		console.log("rebalance", ctx, val);
-		if(!ctx.post){ return ctx.sub[ctx.path] = val }
-		ctx.sub[ctx.pre] = ctx.sub[ctx.pre] || (ctx.post? {} : ctx.val || {});
-		ctx.sub[ctx.pre][ctx.path] = val;
-		if(ctx.post){
-			ctx.sub[ctx.pre][ctx.post] = ctx.val;
-			delete ctx.sub[ctx.pre + ctx.post];
-		}
-	}
-	function set(p, val){
-		rebalance(match(p, r), val);
-		console.log('-------------------------');
-		return r;
-	}
-	return function(p, val){
-		return (1 < arguments.length)? set(p, val) : get(p || '');
-	}
-} // IT WORKS!!!!!!
-
-var rad = radix({});
-
-rad('user/marknadal', {'#': 'asdf'});
-rad('user/ambercazzell', {'#': 'dafs'});
-rad('user/taitforrest', {'#': 'sadf'});
-rad('user/taitveronika', {'#': 'fdsa'});
-rad('user/marknadal', {'#': 'foo'});
-
-/*
-
-function radix(r){
-	var u, n = null, c = 0;
-	r = r || {};
-	function get(){
-
-	}
-	function match(p, l, cb){
-		cb = cb || function(){};
-		console.log("LETS DO THIS", p, l);
-		if(!Gun.obj.map(l, function(v, k){
-			if(k[0] === p[0]){
-				var i = 1;
-				while(k[i] === p[i] && p[i]){ i++ }
-				k = k.slice(i);
-				if(k){
-					cb(p.slice(0, i), v, k, l, p.slice(i));
-				} else {
-					match(p.slice(i), v, cb);
-				}
-				return 1;
-			}
-		})){ cb(p, l, null, l) }
-	}
-	function set(p, val){
-		match(p, r, function(pre, data, f, rr, pp){
-			if(f === null){
-				rr[pre] = val;
-				return;
-			}
-			console.log("Match?", c, pre, data, f);
-			rr[pre] = r[pre] || (f? {} : data || {});
-			rr[pre][pp] = val;
-			if(f){
-				rr[pre][f] = data;
-				delete rr[pre + f];
-			}
-		});
-		return r;
-	}
-	return function(p, val){
-		return (1 < arguments.length)? set(p, val) : get(p || '');
-	}
-} // IT WORKS!!!!!!
-
-var rad = radix({});
-
-rad('user/marknadal', {'#': 'asdf'});
-//rad('user/ambercazzell', {'#': 'dafs'});
-//rad('user/taitforrest', {'#': 'sadf'});
-//rad('user/taitveronika', {'#': 'fdsa'});
-*/

test/ptsd/radisk.js

@@ -0,0 +1,71 @@
+var Radix = require('../../lib/radix');
+var Radisk = require('../../lib/radisk');
+var Gun = require('../../gun');
+var fs = require('fs');
+
+var TOTAL = 1000000;
+var c = 0;
+var acked = 0;
+var start;
+var diff;
+
+(function(){//return;
+	var radix = Radisk();
+	var gtr = Gun()._.opt.uuid;
+	var l = 500000;
+	var last = start;
+	var t = Gun.time.is;
+	var at = c;
+	var toc, alldone = function(){
+		acked++;
+		if(acked < TOTAL){ return }
+		diff = (Gun.time.is() - start) / 1000;
+		clearTimeout(toc);
+		toc = setTimeout(CHECK,1000);
+	}
+	function bench(){
+		if(c >= (TOTAL)){ return clearInterval(it); }
+		for(var i = 0; i < l; i++){
+			radix(gtr(), ++c, alldone);
+			if(c % 50000 === 0){
+				var now = t();
+				console.log(c);//, (now - last)/1000);
+				at = c;
+				last = now; 
+			}
+		}
+	}
+	start = Gun.time.is();
+	var it = setInterval(bench, 1);
+}());
+
+function CHECK(){
+	console.log(Math.floor(c / diff), 'disk writes per second');
+	var disk = Radisk();
+	var all = {};
+	var to;
+	var i = TOTAL;
+	/*while(--i){
+		all[i] = true;
+	}*/
+	var dir = fs.readdirSync('radata');
+	dir.forEach(function(file){
+		disk.read(file, function(err, rad){
+			Radix.map(rad, function(val, key){
+				all[key] = false;
+				clearTimeout(to);
+				to = setTimeout(function(){
+					var len = Object.keys(all).length;
+					console.log("how many?", len);
+					if(len < TOTAL){ return }
+					var missing = [];
+					var fail = Gun.obj.map(all, function(val, key){
+						if(val){ missing.push(key); return true }
+					});
+					//console.log(all);
+					console.log("DONE!", 'Verify ALL writes:', fail? '!!!FAIL!!!!' : 'YES');// '. Missing:', missing);
+				},1000);
+			})
+		})
+	})
+}