radix prototype

This commit is contained in:
Mark Nadal 2017-07-05 00:20:21 -07:00
parent 12d568ccf3
commit b004cc0635
5 changed files with 327 additions and 208 deletions

73
gun.js
View File

@ -74,7 +74,7 @@
Type.list.index = 1; // change this to 0 if you want non-logical, non-mathematical, non-matrix, non-convenient array notation
Type.obj = {is: function(o){ return o? (o instanceof Object && o.constructor === Object) || Object.prototype.toString.call(o).match(/^\[object (\w+)\]$/)[1] === 'Object' : false }}
Type.obj.put = function(o, f, v){ return (o||{})[f] = v, o }
Type.obj.has = function(o, f){ return o && Object.prototype.hasOwnProperty.call(o, f) }
Type.obj.has = function(o, f){ return o && Object.hasOwnProperty(o, f) }
Type.obj.del = function(o, k){
if(!o){ return }
o[k] = null;
@ -126,7 +126,7 @@
var u, i = 0, x, r, ll, lle, f = fn_is(c);
t.r = null;
if(keys && obj_is(l)){
ll = Object.keys(l); lle = true;
ll = keys(l); lle = true;
}
if(list_is(l) || ll){
x = (ll || l).length;
@ -995,6 +995,9 @@
if(!obj_is(at.opt.peers)){ at.opt.peers = {}}
at.opt.peers = obj_to(tmp, at.opt.peers);
}
at.opt.uuid = at.opt.uuid || function(){
return state().toString(36).replace('.','') + text_rand(12);
}
at.opt.wsc = at.opt.wsc || {protocols:[]}
at.opt.peers = at.opt.peers || {};
obj_to(opt, at.opt); // copies options on to `at.opt` only if not already taken.
@ -1003,10 +1006,10 @@
}
}());
var text_is = Gun.text.is;
var list_is = Gun.list.is;
var text = Gun.text, text_is = text.is, text_rand = text.random;
var obj = Gun.obj, obj_is = obj.is, obj_has = obj.has, obj_to = obj.to, obj_map = obj.map, obj_copy = obj.copy;
var _soul = Gun._.soul, _field = Gun._.field, rel_is = Gun.val.rel.is;
var state = Gun.state, _soul = Gun._.soul, _field = Gun._.field, rel_is = Gun.val.rel.is;
var empty = {}, u;
console.debug = function(i, s){ return (console.debug.i && i === console.debug.i && console.debug.i++) && (console.log.apply(console, arguments) || s) };
@ -1066,7 +1069,7 @@
// language, consider implementing an easier API to build.
var Gun = require('./root');
Gun.chain.chain = function(){
var at = this._, chain = new this.constructor(this), cat = chain._;
var at = this._, chain = new this.constructor(this), cat = chain._, root;
cat.root = root = at.root;
cat.id = ++root._.once;
cat.back = this;
@ -2003,12 +2006,11 @@
Gun.on('opt', function(at){
var opt = at.opt, peers = opt.peers;
var prefix = opt.file || opt.prefix || 'gun/';
var wait = opt.wait || 1, batch = opt.batch || 10000;
opt.file = opt.file || opt.prefix || 'gun/'; // support old option name.
this.to.next(at);
if(at.once){ return }
if(false === at.opt.file){ return }
var graph = at.graph, acks = {}, async = {}, count = 0, to;
var graph = at.graph, acks = {}, batch = {}, count = 0, to;
at.on('put', function(at){
this.to.next(at);
@ -2016,12 +2018,11 @@
if(!at['@']){ acks[at['#']] = 1; } // only ack non-acks.
count += 1;
console.log('lS PUT', Gun.obj.copy(async));
if(count >= batch){
if(count >= opt.batch){
return flush();
}
if(to){ return }
clearTimeout(to);
to = setTimeout(flush, wait);
to = setTimeout(flush, opt.wait);
});
function map(node, soul){
@ -2029,31 +2030,31 @@
}
function flush(){
var err;
count = 0;
clearTimeout(to);
to = false;
var ack = acks;
acks = {};
var all = async;
async = {};
Gun.obj.map(all, function(node, soul){
// Since localStorage only has 5MB, it is better that we keep only
// the data that the user is currently interested in.
node = graph[soul] || all[soul] || node;
console.log("WRITE:", Gun.obj.copy(node));
try{store.setItem(prefix + soul, JSON.stringify(node));
}catch(e){ err = e || "localStorage failure" }
var err;
count = 0;
clearTimeout(to);
to = false;
var ack = acks;
acks = {};
var all = async;
async = {};
Gun.obj.map(all, function(node, soul){
// Since localStorage only has 5MB, it is better that we keep only
// the data that the user is currently interested in.
node = graph[soul] || all[soul] || node;
console.log("WRITE:", Gun.obj.copy(node));
try{store.setItem(opt.file + soul, JSON.stringify(node));
}catch(e){ err = e || "localStorage failure" }
});
if(!err && !Gun.obj.empty(peers)){ return } // only ack if there are no peers.
Gun.obj.map(ack, function(yes, id){
at.on('in', {
'@': id,
err: err,
ok: 0 // localStorage isn't reliable, so make its `ok` code be a low number.
});
if(!err && !Gun.obj.empty(peers)){ return } // only ack if there are no peers.
Gun.obj.map(ack, function(yes, id){
at.on('in', {
'@': id,
err: err,
ok: 0 // localStorage isn't reliable, so make its `ok` code be a low number.
});
});
}
});
}
at.on('get', function(at){
this.to.next(at);
@ -2062,7 +2063,7 @@
if(!lex || !(soul = lex[Gun._.soul])){ return }
//if(0 >= at.cap){ return }
var field = lex['.'];
data = async[soul] || Gun.obj.ify(store.getItem(prefix + soul) || null) || async[soul] || u;
data = async[soul] || Gun.obj.ify(store.getItem(opt.file + soul) || null) || async[soul] || u;
console.debug(2, 'lS GET', data, async[soul]);
if(data && field){
data = Gun.state.to(data, field);

196
lib/radisk.js Normal file
View File

@ -0,0 +1,196 @@
var fs = require('fs');
var Gun = require('../gun');
var Radix = require('./radix');
function Radisk(opt){
/*
Any and all storage adapters should...
1. If not busy, write to disk immediately.
2. If busy, batch to disk. (Improves performance, reduces potential disk corruption)
3. If a batch exceeds a certain number of writes, force to disk. (This caps total performance, but reduces potential loss)
*/
var radisk = function(key, val, cb){
if(val instanceof Function){
cb = val;
val = radisk.batch(key);
if(u !== val){
return cb(null, val);
}
if(radisk.was){
val = radisk.was(key);
if(u !== val){
return cb(null, val);
}
}
console.log("READ FROM DISK");
return cb(null, val);
}
radisk.batch(key, val);
if(cb){ radisk.batch.acks.push(cb) }
if(!count++){ return thrash() } // (1)
if(opt.batch <= count){ return thrash() } // (3)
clearTimeout(to); // (2)
to = setTimeout(thrash, opt.wait);
};
radisk.batch = Radix();
radisk.batch.acks = [];
var count = 0, wait, to, u;
opt = opt || {};
opt.file = opt.file || 'radata';
opt.size = opt.size || (1024 * 1024 * 10); // 10MB
opt.batch = opt.batch || 10 * 1000;
opt.wait = opt.wait || 1;
opt.nest = opt.nest || ' ';
console.log("Warning: Radix storage engine has not been tested with all types of values and keys yet.");
if(!fs.existsSync(opt.file)){ fs.mkdirSync(opt.file) }
var thrash = function(){
if(wait){ return }
clearTimeout(to);
wait = true;
var was = radisk.was = radisk.batch;
radisk.batch = null;
radisk.batch = Radix();
radisk.batch.acks = [];
chunk(radisk.was, function(err, ok){
radisk.was = null;
wait = false;
var tmp = count;
count = 0;
Gun.obj.map(was.acks, function(cb){cb(err, ok)});
if(1 < tmp){ thrash() }
});
}
/*
1. Find the first radix item in memory.
2. Use that as the starting index in the directory of files.
3. Find the first file that is lexically larger than it,
4. Read the previous file to that into memory
5. Scan through the in memory radix for all values lexically less than the limit.
6. Merge and write all of those to the in-memory file and back to disk.
7. If file to large, split. More details needed here.
*/
function chunk(radix, cb){
var step = {
check: function(tree, key){
if(key < step.start){ return }
step.start = key;
fs.readdir(opt.file, step.match);
return true;
},
match: function(err, dir){
step.dir = dir;
if(!dir.length){
step.file = '0';
return step.merge(null, Radix());
}
Gun.obj.map(dir, step.lex);
read(step.file, step.merge);
},
lex: function(file){
if(file > step.start){
return step.end = file;
}
step.file = file;
},
merge: function(err, disk){
if(err){ return console.log("ERROR!!!", err) }
step.disk = disk;
Radix.map(radix, step.add);
write(step.file, step.disk, step.done);
},
add: function(val, key){
if(key < step.start){ return }
if(step.end && step.end < key){ return step.next = key; }
step.disk(key, val);
},
done: function(err){
if(err){ console.log("ERROR!!!", err) }
if(!step.next){
return cb(err);
}
step.start = step.next;
step.end = step.next = step.file = u;
Radix.map(radix, step.check);
}
}
Radix.map(radix, step.check);
}
var write = function(file, radix, cb){
var step = {
rest: "",
count: 0,
file: file,
each: function(val, key, k, pre){
step.count++;
if(opt.size < step.rest.length){
step.rest = "";
step.limit = Math.ceil(step.count/2);
step.count = 0;
step.sub = Radix();
Radix.map(radix, step.slice);
return true;
}
var i = pre.length;
while(i--){ step.rest += opt.nest };
step.rest += k + (u === val? '' : '=' + val) + '\n';
},
dump: function(){
var rest = step.rest;
step.rest = "";
fs.writeFile(opt.file +'/'+ file, rest, cb);
},
slice: function(val, key){
if(key < step.file){ return }
if(step.limit < (++step.count)){
var name = step.file;
step.file = key;
step.count = 0;
write(name, step.sub, step.next);
return true;
}
step.sub(key, val);
},
next: function(err){
if(err){ console.log("ERR!!!!") }
step.sub = Radix();
if(!Radix.map(radix, step.slice)){
write(step.file, step.sub, cb);
}
}
};
if(!Radix.map(radix, step.each, true)){ step.dump() }
}
var read = function(file, cb){
var step = {
nest: 0,
rad: Radix(),
data: function(err, data){
if(err){ return console.log("ERROR READING FILE!", err) }
step.pre = [];
Gun.obj.map(data.toString().split('\n'), step.split); // TODO: Escape!
cb(null, step.rad);
},
split: function(line){ var LINE = line;
var nest = -1; while(opt.nest === line[++nest]){};
if(nest){ line = line.slice(nest) }
if(nest <= step.nest){ step.pre = step.pre.slice(0, nest - step.nest - 1) }
line = line.split('='); step.pre.push(line[0]);
if(1 < line.length){ step.rad(step.pre.join(''), line[1]) }
step.nest = nest;
}
}
fs.readFile(opt.file +'/'+ file, step.data);
}
radisk.read = read;
return radisk;
}
module.exports = Radisk;

View File

@ -1,10 +1,9 @@
var fs = require('fs');
var Gun = require('gun');
var Gun = require('../gun');
var gbm = Gun.obj.map, no = {}, u;
function Radix(){
var radix = function(key, val, t){
t = t || tree;
t = t || radix._ || (radix._ = {});
var i = 0, l = key.length-1, k = key[i], at, tmp;
while(!(at = t[k]) && i < l){
k += key[++i];
@ -36,75 +35,31 @@ function Radix(){
if(u === val){ return (u === (tmp = at.$))? at._ : tmp }
at.$ = val;
} else {
return radix(key.slice(++i), val, at._);
return radix(key.slice(++i), val, at._ || (at._ = {}));
}
}
var tree = radix._ = {};
return radix;
}
(function(){
var radix = Radix();
radix('user/marknadal', 'asdf');
radix('user/ambercazzell', 'dafs');
radix('user/taitforrest', 'sadf');
radix('user/taitveronika', 'fdsa');
radix('user/marknadal', 'foo');
radix('user/table', 'foo');
console.log("__________________");
console.log(radix._);
console.log(radix('user/table'));
//fs.writeFileSync('radix-data.json', JSON.stringify(radix._, null, 2));
}());
(function(){return;
var radix = Radix();
var gtr = Gun.text.random;
var c = 0, l = 10;
function bench(){
if(c > 1000){ return clearInterval(it), console.log(radix._, Object.keys(radix._).length), fs.writeFileSync('radix-data.json', JSON.stringify(radix._, null, 2)); }
for(var i = 0; i < l; i++){
radix(gtr(7), c++);
}
//if(c % 1000 === 0){ console.log(radix._) }
}
var it = setInterval(bench, 1);
}());
function Radisk(){
var radix = Radix();
var radisk = function(key, val, cb){
if(val instanceof Function){
cb = val;
val = radix(key);
if(u !== val){
return cb(null, val);
};
;(function(){
Radix.map = function map(radix, cb, opt, pre){ pre = pre || [];
var _ = radix._ || radix, keys = Object.keys(_).sort(), i = 0, l = keys.length;
for(;i < l; i++){ var key = keys[i], tree = _[key], tmp;
if(u !== (tmp = tree.$)){
tmp = cb(tmp, pre.join('') + key, key, pre);
if(u !== tmp){ return tmp }
} else
if(opt){
cb(u, pre.join(''), key, pre);
}
if(tmp = tree._){
pre.push(key);
tmp = map(tmp, cb, opt, pre);
if(u !== tmp){ return tmp }
pre.pop();
}
return console.log("read from disk");
}
radix(key, val);
if(!to){ return flush(to = true) }
clearTimeout(to);
to = setTimeout(flush, opt.wait)
};
var opt = {file: 'radix', size: 1024 * 1024 * 2, wait: 10 * 1000, batch: 10 * 1000}, to;
Object.keys = Object.keys || function(o){ return gbm(o, function(v,k,t){t(k)}) }
}());
var flush = function(){
}
var read = function(path){
return fs.readFileSync(nodePath.join(dir, path)).toString();
}
var write = function(path, data){
return fs.writeFileSync(nodePath.join(dir, path), data);
}
var mk = function(path){
path = nodePath.join(dir, path);
if(fs.existsSync(path)){ return }
fs.mkdirSync(path);
}
return radisk;
}
module.exports = Radix;

View File

@ -1,104 +0,0 @@
function radix(r){
r = r || {};
var u, n = null, c = 0;
function get(p){
var v = match(p, r);
return v;
}
function match(path, tree, v){
if(!Gun.obj.map(tree, function(val, key){
if(key[0] !== path[0]){ return }
var i = 1;
while(key[i] === path[i] && path[i]){ i++ }
if(key = key.slice(i)){ // recurse
console.log("match", key, i)
v = {sub: tree, pre: path.slice(0, i), val: val, post: key, path: path.slice(i) };
} else { // replace
console.log("matching", path, key, i);
v = match(path.slice(i), val);
}
return true;
})){ console.log("matched", tree, path); v = {sub: tree, path: path} } // insert
return v;
}
function rebalance(ctx, val){
console.log("rebalance", ctx, val);
if(!ctx.post){ return ctx.sub[ctx.path] = val }
ctx.sub[ctx.pre] = ctx.sub[ctx.pre] || (ctx.post? {} : ctx.val || {});
ctx.sub[ctx.pre][ctx.path] = val;
if(ctx.post){
ctx.sub[ctx.pre][ctx.post] = ctx.val;
delete ctx.sub[ctx.pre + ctx.post];
}
}
function set(p, val){
rebalance(match(p, r), val);
console.log('-------------------------');
return r;
}
return function(p, val){
return (1 < arguments.length)? set(p, val) : get(p || '');
}
} // IT WORKS!!!!!!
var rad = radix({});
rad('user/marknadal', {'#': 'asdf'});
rad('user/ambercazzell', {'#': 'dafs'});
rad('user/taitforrest', {'#': 'sadf'});
rad('user/taitveronika', {'#': 'fdsa'});
rad('user/marknadal', {'#': 'foo'});
/*
function radix(r){
var u, n = null, c = 0;
r = r || {};
function get(){
}
function match(p, l, cb){
cb = cb || function(){};
console.log("LETS DO THIS", p, l);
if(!Gun.obj.map(l, function(v, k){
if(k[0] === p[0]){
var i = 1;
while(k[i] === p[i] && p[i]){ i++ }
k = k.slice(i);
if(k){
cb(p.slice(0, i), v, k, l, p.slice(i));
} else {
match(p.slice(i), v, cb);
}
return 1;
}
})){ cb(p, l, null, l) }
}
function set(p, val){
match(p, r, function(pre, data, f, rr, pp){
if(f === null){
rr[pre] = val;
return;
}
console.log("Match?", c, pre, data, f);
rr[pre] = r[pre] || (f? {} : data || {});
rr[pre][pp] = val;
if(f){
rr[pre][f] = data;
delete rr[pre + f];
}
});
return r;
}
return function(p, val){
return (1 < arguments.length)? set(p, val) : get(p || '');
}
} // IT WORKS!!!!!!
var rad = radix({});
rad('user/marknadal', {'#': 'asdf'});
//rad('user/ambercazzell', {'#': 'dafs'});
//rad('user/taitforrest', {'#': 'sadf'});
//rad('user/taitveronika', {'#': 'fdsa'});
*/

71
test/ptsd/radisk.js Normal file
View File

@ -0,0 +1,71 @@
var Radix = require('../../lib/radix');
var Radisk = require('../../lib/radisk');
var Gun = require('../../gun');
var fs = require('fs');
var TOTAL = 1000000;
var c = 0;
var acked = 0;
var start;
var diff;
(function(){//return;
var radix = Radisk();
var gtr = Gun()._.opt.uuid;
var l = 500000;
var last = start;
var t = Gun.time.is;
var at = c;
var toc, alldone = function(){
acked++;
if(acked < TOTAL){ return }
diff = (Gun.time.is() - start) / 1000;
clearTimeout(toc);
toc = setTimeout(CHECK,1000);
}
function bench(){
if(c >= (TOTAL)){ return clearInterval(it); }
for(var i = 0; i < l; i++){
radix(gtr(), ++c, alldone);
if(c % 50000 === 0){
var now = t();
console.log(c);//, (now - last)/1000);
at = c;
last = now;
}
}
}
start = Gun.time.is();
var it = setInterval(bench, 1);
}());
function CHECK(){
console.log(Math.floor(c / diff), 'disk writes per second');
var disk = Radisk();
var all = {};
var to;
var i = TOTAL;
/*while(--i){
all[i] = true;
}*/
var dir = fs.readdirSync('radata');
dir.forEach(function(file){
disk.read(file, function(err, rad){
Radix.map(rad, function(val, key){
all[key] = false;
clearTimeout(to);
to = setTimeout(function(){
var len = Object.keys(all).length;
console.log("how many?", len);
if(len < TOTAL){ return }
var missing = [];
var fail = Gun.obj.map(all, function(val, key){
if(val){ missing.push(key); return true }
});
//console.log(all);
console.log("DONE!", 'Verify ALL writes:', fail? '!!!FAIL!!!!' : 'YES');// '. Missing:', missing);
},1000);
})
})
})
}