because yeah!

This commit is contained in:
Mark Nadal 2020-02-05 01:57:11 -08:00
parent fa8a63606c
commit 89b8f01cda
8 changed files with 817 additions and 40 deletions

View File

@ -120,7 +120,7 @@
var w = 2;
function go(i){
var S = +new Date;
var ref = gun.get(Gun.text.random(12)).put({data: Math.random()}, function(ack){
var ref = gun.get(Gun.text.random(12)).put({data: Math.random(), ah: Math.random()}, function(ack){
console.log((+new Date - S)/1000, ack.err, ack.ok);
ref.off();
});

128
gun.js
View File

@ -359,7 +359,7 @@
/*if(perf){
t = start + perf.now(); // Danger: Accuracy decays significantly over time, even if precise.
} else {*/
t = time();
t = +new Date;
//}
if(last < t){
return N = 0, last = t + State.drift;
@ -368,10 +368,10 @@
}
var time = Type.time.is, last = -Infinity, N = 0, D = 1000; // WARNING! In the future, on machines that are D times faster than 2016AD machines, you will want to increase D by another several orders of magnitude so the processing speed never out paces the decimal resolution (increasing an integer effects the state accuracy).
var perf = (typeof performance !== 'undefined')? (performance.timing && performance) : false, start = (perf && perf.timing && perf.timing.navigationStart) || (perf = false);
State._ = '>';
var S_ = State._ = '>';
State.drift = 0;
State.is = function(n, k, o){ // convenience function to get the state on a key on a node and return it.
var tmp = (k && n && n[N_] && n[N_][State._]) || o;
var tmp = (k && n && n[N_] && n[N_][S_]) || o;
if(!tmp){ return }
return num_is(tmp = tmp[k])? tmp : -Infinity;
}
@ -383,7 +383,7 @@
}
n = Node.soul.ify(n, soul); // then make it so!
}
var tmp = obj_as(n[N_], State._); // grab the states data.
var tmp = obj_as(n[N_], S_); // grab the states data.
if(u !== k && k !== N_){
if(num_is(s)){
tmp[k] = s; // add the valid state.
@ -618,40 +618,37 @@
;USE(function(module){
var Type = USE('./type');
function Dup(opt){
var dup = {s:{}};
var dup = {s:{}}, s = dup.s;
opt = opt || {max: 1000, age: /*1000 * 9};//*/ 1000 * 9 * 3};
dup.check = function(id){ var tmp;
if(!(tmp = dup.s[id])){ return false }
if(tmp.pass){ return tmp.pass = false }
dup.check = function(id){
if(!s[id]){ return false }
return dup.track(id);
}
dup.track = function(id, pass){
var it = dup.s[id] || (dup.s[id] = {});
it.was = time_is();
if(pass){ it.pass = true }
dup.track = function(id){
var it = s[id] || (s[id] = {});
it.was = +new Date;
if(!dup.to){ dup.to = setTimeout(dup.drop, opt.age + 9) }
return it;
}
dup.drop = function(age){
var now = time_is();
Type.obj.map(dup.s, function(it, id){
var now = +new Date;
Type.obj.map(s, function(it, id){
if(it && (age || opt.age) > (now - it.was)){ return }
Type.obj.del(dup.s, id);
delete s[id];
});
dup.to = null;
}
return dup;
}
var time_is = Type.time.is;
module.exports = Dup;
})(USE, './dup');
;USE(function(module){
function Gun(o){
if(o instanceof Gun){ return (this._ = {gun: this, $: this}).$ }
if(o instanceof Gun){ return (this._ = {$: this}).$ }
if(!(this instanceof Gun)){ return new Gun(o) }
return Gun.create(this._ = {gun: this, $: this, opt: o});
return Gun.create(this._ = {$: this, opt: o});
}
Gun.is = function($){ return ($ instanceof Gun) || ($ && $._ && ($ === $._.$)) || false }
@ -682,7 +679,9 @@
var gun = at.$.opt(at.opt);
if(!at.once){
at.on('in', root, at);
at.on('in2', root2, at);
at.on('out', root, {at: at, out: root});
at.on('put2', cache, at);
Gun.on('create', at);
at.on('create', at);
}
@ -716,8 +715,89 @@
at.on('out', msg);
}
}
function root2(msg){
if(!msg){ return }
var eve = this, as = eve.as, at = as.at || as, gun = at.$, dup = at.dup, tmp;
if(!(tmp = msg['#'])){ tmp = msg['#'] = text_rand(9) }
if(dup.check(tmp)){ return } dup.track(tmp);
tmp = msg._; msg._ = ('function' == typeof msg._)? msg._ : function(){};
if(msg.get){ Gun.on.get(msg, gun) }
if(msg.put){ PUT(msg, gun) }
eve.to.next(msg);
}
}());
;(function(){
Gun.on.put2 = function(msg, gun){
var ctx = msg._, root = ctx.root = gun._, id = msg['#'];
var ack = ctx.ack = function(put){ put = put || {};
if(ack.err = ack.err || (put||{}).err){
if(ack.s){ root.on('in', {'@': id, err: Gun.log(ack.err)}) }
return ack.s = u;
}
var soul = put['#'], key = put['.'], val = put[':'], state = put['>'], tmp;
if((tmp = ack.s||{})[soul+key]){
delete tmp[soul+key];
root.on('put2', {put: put});
}
if(!obj_empty(ack.s)){ return } // keep waiting
console.log("I'm all done!!!");
}
ack.err = obj_map(msg.put, nodes, msg);
msg = ctx = u;
if(ack.err){ return ack() }
}
function nodes(node, soul){
if(!node){ return ERR+cut(soul)+"no node." }
var ctx = this._, tmp;
if(!(tmp = node._)){ return ERR+cut(soul)+"no meta." }
ctx.node = node;
if(soul !== tmp[_soul]){ return ERR+cut(soul)+"soul not same." }
ctx.soul = soul;
if(!(ctx.states = tmp[state_])){ return ERR+cut(soul)+"no state." }
return obj_map(node, souls, this);
}
function souls(val, key){
if(node_ === key){ return }
var ctx = this._, node = ctx.node, soul = ctx.soul, state = ctx.states[key];
if(u === state){ return ERR+cut(key)+"on"+cut(soul)+"no state." }
if(!val_is(val)){ return ERR+cut(key)+"on"+cut(soul)+"bad "+(typeof val)+cut(val) }
ham(ctx.root.graph, soul, key, val, state, ctx.ack); // TODO: HANDLE CALLBACK WHERE ALL DAY IS HISTORIC?
}
function ham(graph, soul, key, val, state, ack){
(ack.s || (ack.s = {}))[soul+key] = 1;
var vertex = graph[soul] || empty, was = state_is(vertex, key, 1), known = vertex[key];
var machine = State(), is = HAM(machine, state, was, val, known), u;
if(!is.incoming){
if(is.defer){
var to = is.defer - machine;
setTimeout(function(){
ham(graph, soul, key, val, state, cb);
}, to > MD? MD : to); // setTimeout Max Defer 32bit :(
return;
}
if(ack){ ack({'#': soul, '.': key}) }
return;
}
if(ack){ ack({'#': soul, '.': key, ':': val, '>': state}) }
}
var cut = function(s){ return " '"+(''+s).slice(0,9)+"...' " }
var ERR = "Error: Invalid graph!";
}());
var PUT = Gun.on.put2;
var HAM = Gun.HAM, MD = 2147483647, State = Gun.state;
function cache(msg){
var eve = this, root = eve.as, graph = root.graph;
var put = msg.put, soul = put['#'], key = put['.'], val = put[':'], state = put['>'];
graph[soul] = state_ify(graph[soul], key, state, val, soul);
// trigger chains // trigger after all individual values cached?
if(eve.to.last === eve.to.next){
console.log("THE END, reply back");
return;
}
eve.to.next(msg);
}
;(function(){
Gun.on.put = function(msg, gun){
var at = gun._, ctx = {$: gun, graph: at.graph, put: {}, map: {}, souls: {}, machine: Gun.state(), ack: msg['@'], cat: at, stop: {}};
@ -735,7 +815,6 @@
if(!ctx.diff){ return }
at.on('put', obj_to(msg, {put: ctx.diff}));
};
var MD = 2147483647;
function verify(val, key, node, soul){ var ctx = this;
var state = Gun.state.is(node, key), tmp;
if(!state){ return ctx.err = "Error: No state on '"+key+"' in node '"+soul+"'!" }
@ -860,8 +939,8 @@
var list_is = Gun.list.is;
var text = Gun.text, text_is = text.is, text_rand = text.random;
var obj = Gun.obj, obj_is = obj.is, obj_has = obj.has, obj_to = obj.to, obj_map = obj.map, obj_copy = obj.copy;
var state_lex = Gun.state.lex, _soul = Gun.val.link._, _has = '.', node_ = Gun.node._, rel_is = Gun.val.link.is;
var obj = Gun.obj, obj_empty = obj.empty, obj_is = obj.is, obj_has = obj.has, obj_to = obj.to, obj_map = obj.map, obj_copy = obj.copy;
var state_lex = Gun.state.lex, state_ify = Gun.state.ify, state_is = Gun.state.is, _soul = Gun.val.link._, _has = '.', node_ = Gun.node._, val_is = Gun.val.is, rel_is = Gun.val.link.is, state_ = Gun.state._;
var empty = {}, u;
console.only = function(i, s){ return (console.only.i && i === console.only.i && console.only.i++) && (console.log.apply(console, arguments) || s) };
@ -1331,7 +1410,7 @@
if(eve.seen && at.id && eve.seen[at.id]){ return eve.to.next(msg) }
//if((tmp = root.stop)){ if(tmp[at.id]){ return } tmp[at.id] = msg.root; } // temporary fix till a better solution?
if((tmp = data) && tmp[rel._] && (tmp = rel.is(tmp))){
tmp = ((msg.$$ = at.root.gun.get(tmp))._);
tmp = ((msg.$$ = at.root.$.get(tmp))._);
if(u !== tmp.put){
msg = obj_to(msg, {put: data = tmp.put});
}
@ -1995,10 +2074,9 @@
try{msg = msg || JSON.parse(raw);
}catch(e){return opt.log('DAM JSON parse error', e)}
if(!msg){ return }
if(msg.DBG_s){ opt.log(+new Date - msg.DBG_s, 'to hear', msg['#']) }
if(!(id = msg['#'])){ id = msg['#'] = Type.text.random(9) }
if(msg.DBG_s){ opt.log(+new Date - msg.DBG_s, 'to hear', id) }
if(dup.check(id)){ return }
dup.track(id, true); //.it = it(msg); // GUN core also dedups, so `true` is needed. // Does GUN core need to dedup anymore?
/*if(!(hash = msg['##']) && u !== msg.put){ hash = msg['##'] = Type.obj.hash(msg.put) }
if(hash && (tmp = msg['@'] || (msg.get && id))){ // Reduces backward daisy in case varying hashes at different daisy depths are the same.
if(dup.check(tmp+hash)){ return }
@ -2011,11 +2089,13 @@
if(tmp = mesh.hear[msg.dam]){
tmp(msg, peer, root);
}
dup.track(id);
return;
}
var S, ST; LOG && (S = +new Date); console.STAT = {};
//root.on('in', msg);
root.on('in2', msg);
dup.track(id);
if(LOG && !msg.nts && (ST = +new Date - S) > 9){ opt.log(S, ST, 'msg', msg['#'], JSON.stringify(console.STAT)); if(ST > 500){ try{ require('./lib/email').send({text: ""+ST+"ms "+JSON.stringify(msg)+" | "+JSON.stringify(console.STAT), from: "mark@gun.eco", to: "mark@gun.eco", subject: "GUN MSG"}, noop); }catch(e){} } } // this is ONLY turned on if ENV CONFIGS have email/password to send out from.
return;
}

528
lib/radiskip.js Normal file
View File

@ -0,0 +1,528 @@
;(function(){
function Radisk(opt){
opt = opt || {};
opt.log = opt.log || console.log;
opt.file = String(opt.file || 'radata');
var has = (Radisk.has || (Radisk.has = {}))[opt.file];
if(has){ return has }
opt.pack = opt.pack || (opt.memory? (opt.memory * 1000 * 1000) : 1399000000) * 0.3; // max_old_space_size defaults to 1400 MB.
opt.until = opt.until || opt.wait || 250;
opt.batch = opt.batch || (10 * 1000);
opt.chunk = opt.chunk || (1024 * 1024 * 1); // 1MB
opt.code = opt.code || {};
opt.code.from = opt.code.from || '!';
opt.jsonify = true;
function ename(t){ return encodeURIComponent(t).replace(/\*/g, '%2A') }
function atomic(v){ return u !== v && (!v || 'object' != typeof v) }
var map = Gun.obj.map;
var LOG = console.LOG;
var ST = 0;
if(!opt.store){
return opt.log("ERROR: Radisk needs `opt.store` interface with `{get: fn, put: fn (, list: fn)}`!");
}
if(!opt.store.put){
return opt.log("ERROR: Radisk needs `store.put` interface with `(file, data, cb)`!");
}
if(!opt.store.get){
return opt.log("ERROR: Radisk needs `store.get` interface with `(file, cb)`!");
}
if(!opt.store.list){
//opt.log("WARNING: `store.list` interface might be needed!");
}
/*
Any and all storage adapters should...
1. Because writing to disk takes time, we should batch data to disk. This improves performance, and reduces potential disk corruption.
2. If a batch exceeds a certain number of writes, we should immediately write to disk when physically possible. This caps total performance, but reduces potential loss.
*/
var r = function(key, data, cb){
if('function' === typeof data){
var o = cb || {};
cb = val;
return;
}
//var tmp = (tmp = r.batch = r.batch || {})[key] = tmp[key] || {};
//var tmp = (tmp = r.batch = r.batch || {})[key] = data;
r.save(key, data, cb);
}
r.save = function(key, data, cb){
var s = {key: key};
s.find = function(file){ var tmp;
s.file = file = file || opt.code.from;
if(tmp = r.disk[file]){ return s.mix(null, tmp) }
r.parse(file, s.mix);
}
s.mix = function(err, disk){
if(err){ return cb(err) }
s.file = (disk||noop).file || s.file;
((disk = r.disk[s.file] || disk)||noop).file = s.file;
if(!disk && s.file !== opt.code.from){ // corrupt file?
r.find.bad(s.file); // remove from dir list
r.save(key, data, cb); // try again
return;
}
(r.disk[s.file] = disk = disk || Radix()).file = s.file;
if(opt.compare){
data = opt.compare(disk(key), data, key, s.file);
if(u === data){ return cb(err, -1) }
}
// check if still in same r.find?
(s.disk = disk)(key, data);
if(disk.Q){ return disk.Q.push(cb) } disk.Q = [cb];
disk.to = setTimeout(s.write, opt.until);
}
s.write = function(){
var q = s.disk.Q;
delete s.disk.Q;
delete r.disk[s.file];
r.write(s.file, s.disk, function(err, ok){
Gun.obj.map(q, function(ack){ ack(err, ok) });
});
}
r.find(key, s.find);
}
r.disk = {};
/*
Any storage engine at some point will have to do a read in order to write.
This is true of even systems that use an append only log, if they support updates.
Therefore it is unavoidable that a read will have to happen,
the question is just how long you delay it.
*/
r.write = function(file, rad, cb, o){
if(!rad){ return cb('No radix!') }
o = ('object' == typeof o)? o : {force: o};
var f = function Fractal(){}, a, b;
f.text = '';
f.file = file = rad.file = rad.file || file;
if(!file){ return cb('What file?') }
f.write = function(){
var S; LOG && (S = +new Date);
r.disk[file = rad.file || f.file || file] = rad;
r.find.add(file, function(err){
if(err){ return cb(err) }
opt.store.put(ename(file), f.text, function(err, ok){
LOG && opt.log(S, ST = +new Date - S, "wrote disk", JSON.stringify(file));
cb(err, ok);
delete r.disk[file];
});
});
}
f.split = function(){
f.text = '';
if(!f.count){ f.count = 0;
Radix.map(rad, function(){ f.count++ }); // TODO: Perf? Any faster way to get total length?
}
f.limit = Math.ceil(f.count/2);
f.count = 0;
f.sub = Radix();
Radix.map(rad, f.slice, {reverse: 1}); // IMPORTANT: DO THIS IN REVERSE, SO LAST HALF OF DATA MOVED TO NEW FILE BEFORE DROPPING FROM CURRENT FILE.
r.write(f.end, f.sub, f.both, o);
f.hub = Radix();
Radix.map(rad, f.stop);
r.write(rad.file, f.hub, f.both, o);
return true;
}
f.slice = function(val, key){
f.sub(f.end = key, val);
if(f.limit <= (++f.count)){ return true }
}
f.stop = function(val, key){
if(key >= f.end){ return true }
f.hub(key, val);
}
f.both = function(err, ok){
if(b){ return cb(err || b) }
if(a){ return cb(err, ok) }
a = true;
b = err;
}
f.each = function(val, key, k, pre){
//console.log("RAD:::", JSON.stringify([val, key, k, pre]));
if(u !== val){ f.count++ }
if(opt.pack <= (val||'').length){ return cb("Record too big!"), true }
var enc = Radisk.encode(pre.length) +'#'+ Radisk.encode(k) + (u === val? '' : ':'+ Radisk.encode(val)) +'\n';
if((opt.chunk < f.text.length + enc.length) && (1 < f.count) && !o.force){
return f.split();
}
f.text += enc;
}
if(opt.jsonify){ return r.write.jsonify(f, rad, cb, o) } // temporary testing idea
if(!Radix.map(rad, f.each, true)){ f.write() }
}
r.write.jsonify = function(f, rad, cb, o){
var raw;
var S; LOG && (S = +new Date);
try{raw = JSON.stringify(rad.$);
}catch(e){ return cb("Cannot radisk!") }
LOG && opt.log(S, +new Date - S, "rad stringified JSON");
if(opt.chunk < raw.length && !o.force){
return f.split();
//if(Radix.map(rad, f.each, true)){ return }
}
f.text = raw;
f.write();
}
r.range = function(tree, o){
if(!tree || !o){ return }
if(u === o.start && u === o.end){ return tree }
if(atomic(tree)){ return tree }
var sub = Radix();
Radix.map(tree, function(v,k){ // ONLY PLACE THAT TAKES TREE, maybe reduce API for better perf?
sub(k,v);
}, o);
return sub('');
}
;(function(){
var Q = {};
r.read = function(key, cb, o){
o = o || {};
if(RAD && !o.next){ // cache
var S; LOG && (S = +new Date);
var val = RAD(key);
LOG && (ST = +new Date - S) > 9 && opt.log(S, ST, 'rad cached');
//if(u !== val){
//cb(u, val, o);
if(atomic(val)){ cb(u, val, o); return }
// if a node is requested and some of it is cached... the other parts might not be.
//}
}
o.span = (u !== o.start) || (u !== o.end); // is there a start or end?
var g = function Get(){};
g.lex = function(file){ var tmp; // // TODO: this had a out-of-memory crash!
file = (u === file)? u : decodeURIComponent(file);
tmp = o.next || key || (o.reverse? o.end || '\uffff' : o.start || '');
if(!file || (o.reverse? file < tmp : file > tmp)){
LOG && opt.log(S, +new Date - S, 'rad read lex'); S = +new Date;
if(o.next || o.reverse){ g.file = file }
if(tmp = Q[g.file]){
tmp.push({key: key, ack: cb, file: g.file, opt: o});
return true;
}
Q[g.file] = [{key: key, ack: cb, file: g.file, opt: o}];
if(!g.file){
g.it(null, u, {});
return true;
}
r.parse(g.file, g.check);
return true;
}
g.file = file;
}
g.it = function(err, disk, info){
if(g.err = err){ opt.log('err', err) }
if(!disk && g.file){ // corrupt file?
r.find.bad(g.file); // remove from dir list
r.read(key, cb, o); // look again
return;
}
g.info = info;
if(disk){ RAD = g.disk = disk }
disk = Q[g.file]; delete Q[g.file];
map(disk, g.ack);
}
g.ack = function(as){
if(!as.ack){ return }
var S; LOG && (S = +new Date);
var key = as.key, o = as.opt, info = g.info, rad = g.disk || noop, data = r.range(rad(key), o), last = rad.last || Radix.map(rad, rev, revo);
LOG && (ST = +new Date - S) > 9 && opt.log(S, ST, "rad range loaded");
o.parsed = (o.parsed || 0) + (info.parsed||0);
o.chunks = (o.chunks || 0) + 1;
o.more = true;
if((!as.file) // if no more places to look
|| (!o.span && last === key) // if our key exactly matches the very last atomic record
|| (!o.span && last && last > key && 0 != last.indexOf(key)) // 'zach' may be lexically larger than 'za', but there still might be more, like 'zane' in the 'za' prefix bucket so do not end here.
){
o.more = u;
as.ack(g.err, data, o);
return
}
if(u !== data){
as.ack(g.err, data, o); // more might be coming!
if(o.parsed >= o.limit){ return } // even if more, we've hit our limit, asking peer will need to make a new ask with a new starting point.
}
o.next = as.file;
r.read(key, as.ack, o);
}
g.check = function(err, disk, info){
g.it(err, disk, info);
var good = true;
Radix.map(disk, function(val, key){
// assume in memory for now, since both write/read already call r.find which will init it.
var go = function(file){
if(info.file !== file){
good = false
}
return true;
}
go.reverse = 1;
go.end = key;
r.list(go);
});
if(good){ return }
var id = Gun.text.random(3);
r.save(disk, function ack(err, ok){
if(err){ return r.save(disk, ack) } // ad infinitum???
console.log("MISLOCATED DATA CORRECTED", id);
});
}
/*g.check2 = function(err, disk, info){
if(err || !disk){ return g.it(err, disk, info) }
var good = true;
Radix.map(disk, function(val, key){
// assume in memory for now, since both write/read already call r.find which will init it.
var go = function(file){
if(info.file !== file){ good = false }
return true;
}
go.reverse = 1;
go.end = key;
r.list(go);
});
if(good){ return g.it(err, disk, info) }
var id = Gun.text.random(3); console.log("MISLOCATED DATA", id);
r.save(disk, function ack(err, ok){
if(err){ return r.save(disk, ack) } // ad infinitum???
console.log("MISLOCATED CORRECTED", id);
r.read(key, cb, o);
});
}*/
if(o.reverse){ g.lex.reverse = true }
LOG && (S = +new Date);
r.find(key, g.lex);
}
function rev(a,b){ return b }
var revo = {reverse: true};
}());
;(function(){
/*
Let us start by assuming we are the only process that is
changing the directory or bucket. Not because we do not want
to be multi-process/machine, but because we want to experiment
with how much performance and scale we can get out of only one.
Then we can work on the harder problem of being multi-process.
*/
var Q = {}, s = String.fromCharCode(31);
r.parse = function(file, cb, raw){ var q;
if(q = Q[file]){ return q.push(cb) } q = Q[file] = [cb];
var p = function Parse(){}, info = {file: file};
(p.disk = Radix()).file = file;
p.read = function(err, data){ var tmp;
LOG && opt.log(S, +new Date - S, 'read disk', JSON.stringify(file));
delete Q[file];
if((p.err = err) || (p.not = !data)){ return map(q, p.ack) }
if('string' !== typeof data){
try{
if(opt.pack <= data.length){
p.err = "Chunk too big!";
} else {
data = data.toString(); // If it crashes, it crashes here. How!?? We check size first!
}
}catch(e){ p.err = e }
if(p.err){ return map(q, p.ack) }
}
info.parsed = data.length;
LOG && (S = +new Date);
if(opt.jsonify || '{' === data[0]){
try{
var json = JSON.parse(data); // TODO: this caused a out-of-memory crash!
p.disk.$ = json;
LOG && (ST = +new Date - S) > 9 && opt.log(S, ST, 'rad parsed JSON');
map(q, p.ack);
return;
}catch(e){ tmp = e }
if('{' === data[0]){
p.err = tmp || "JSON error!";
return map(q, p.ack);
}
}
return p.radec(err, data);
}
p.ack = function(cb){
if(!cb){ return }
if(p.err || p.not){ return cb(p.err, u, info) }
cb(u, p.disk, info);
}
p.radec = function(err, data){
LOG && (S = +new Date);
var tmp = p.split(data), pre = [], i, k, v;
if(!tmp || 0 !== tmp[1]){
p.err = "File '"+file+"' does not have root radix! ";
return map(q, p.ack);
}
while(tmp){
k = v = u;
i = tmp[1];
tmp = p.split(tmp[2])||'';
if('#' == tmp[0]){
k = tmp[1];
pre = pre.slice(0,i);
if(i <= pre.length){
pre.push(k);
}
}
tmp = p.split(tmp[2])||'';
if('\n' == tmp[0]){ continue }
if('=' == tmp[0] || ':' == tmp[0]){ v = tmp[1] }
if(u !== k && u !== v){ p.disk(pre.join(''), v) }
tmp = p.split(tmp[2]);
}
LOG && opt.log(S, +new Date - S, 'parsed RAD');
map(q, p.ack);
};
p.split = function(t){
if(!t){ return }
var l = [], o = {}, i = -1, a = '', b, c;
i = t.indexOf(s);
if(!t[i]){ return }
a = t.slice(0, i);
l[0] = a;
l[1] = b = Radisk.decode(t.slice(i), o);
l[2] = t.slice(i + o.i);
return l;
}
var S; LOG && (S = +new Date);
if(raw){ return p.read(null, raw) }
opt.store.get(ename(file), p.read);
}
}());
;(function(){
var dir, f = String.fromCharCode(28), Q;
r.find = function(key, cb){
if(!dir){
if(Q){ return Q.push([key, cb]) } Q = [[key, cb]];
return r.parse(f, init);
}
Radix.map(dir, function(val, key){
if(!val){ return }
return cb(key) || true;
}, {reverse: 1, end: key}) || cb();
}
r.find.add = function(file, cb){
var has = dir(file);
if(has || file === f){ return cb(u, 1) }
dir(file, 1);
cb.found = (cb.found || 0) + 1;
r.write(f, dir, function(err, ok){
if(err){ return cb(err) }
cb.found = (cb.found || 0) - 1;
if(0 !== cb.found){ return }
cb(u, 1);
}, true);
}
r.find.bad = function(file, cb){
dir(file, 0);
r.write(f, dir, cb||noop);
}
function init(err, disk){
if(err){
opt.log('list', err);
setTimeout(function(){ r.parse(f, init) }, 1000);
return;
}
if(disk){ return drain(disk) }
if(!opt.store.list){ return drain(disk || dir || Radix()) }
// import directory.
opt.store.list(function(file){
dir = dir || Radix();
if(!file){ return drain(dir) }
r.find.add(file, noop);
});
}
function drain(rad, tmp){
dir = dir || rad;
dir.file = f;
tmp = Q; Q = null;
Gun.list.map(tmp, function(arg){
r.find(arg[0], arg[1]);
});
}
}());
var noop = function(){}, RAD, u;
Radisk.has[opt.file] = r;
return r;
}
;(function(){
var _ = String.fromCharCode(31), u;
Radisk.encode = function(d, o, s){ s = s || _;
var t = s, tmp;
if(typeof d == 'string'){
var i = d.indexOf(s);
while(i != -1){ t += s; i = d.indexOf(s, i+1) }
return t + '"' + d + s;
} else
if(d && d['#'] && (tmp = Gun.val.link.is(d))){
return t + '#' + tmp + t;
} else
if(Gun.num.is(d)){
return t + '+' + (d||0) + t;
} else
if(null === d){
return t + ' ' + t;
} else
if(true === d){
return t + '+' + t;
} else
if(false === d){
return t + '-' + t;
}// else
//if(binary){}
}
Radisk.decode = function(t, o, s){ s = s || _;
var d = '', i = -1, n = 0, c, p;
if(s !== t[0]){ return }
while(s === t[++i]){ ++n }
p = t[c = n] || true;
while(--n >= 0){ i = t.indexOf(s, i+1) }
if(i == -1){ i = t.length }
d = t.slice(c+1, i);
if(o){ o.i = i+1 }
if('"' === p){
return d;
} else
if('#' === p){
return Gun.val.link.ify(d);
} else
if('+' === p){
if(0 === d.length){
return true;
}
return parseFloat(d);
} else
if(' ' === p){
return null;
} else
if('-' === p){
return false;
}
}
}());
if(typeof window !== "undefined"){
var Gun = window.Gun;
var Radix = window.Radix;
window.Radisk = Radisk;
} else {
var Gun = require('../gun');
var Radix = require('./radix');
//var Radix = require('./radix2'); Radisk = require('./radisk2');
try{ module.exports = Radisk }catch(e){}
}
Radisk.Radix = Radix;
}());

View File

@ -2,13 +2,13 @@
function Radix(){
var radix = function(key, val, t){
key = ''+key;
if(!t && u !== val){
radix.last = (key < radix.last)? radix.last : key;
radix.last = (''+key < radix.last)? radix.last : ''+key;
delete (radix.$||{})[_];
}
t = t || radix.$ || (radix.$ = {});
if(!key && Object.keys(t).length){ return t }
key = ''+key;
var i = 0, l = key.length-1, k = key[i], at, tmp;
while(!(at = t[k]) && i < l){
k += key[++i];
@ -22,19 +22,24 @@
if(kk){
if(u === val){
if(ii <= l){ return }
return (tmp || (tmp = {}))[s.slice(ii)] = r;
(tmp || (tmp = {}))[s.slice(ii)] = r;
//(tmp[_] = function $(){ $.sort = Object.keys(tmp).sort(); return $ }()); // get rid of this one, cause it is on read?
return r;
}
var __ = {};
__[s.slice(ii)] = r;
ii = key.slice(ii);
('' === ii)? (__[''] = val) : ((__[ii] = {})[''] = val);
//(__[_] = function $(){ $.sort = Object.keys(__).sort(); return $ }());
t[kk] = __;
delete t[s];
//(t[_] = function $(){ $.sort = Object.keys(t).sort(); return $ }());
return true;
}
})){
if(u === val){ return; }
(t[k] || (t[k] = {}))[''] = val;
//(t[_] = function $(){ $.sort = Object.keys(t).sort(); return $ }());
}
if(u === val){
return tmp;
@ -43,8 +48,10 @@
if(i == l){
if(u === val){ return (u === (tmp = at['']))? at : tmp }
at[''] = val;
//(at[_] = function $(){ $.sort = Object.keys(at).sort(); return $ }());
} else {
if(u !== val){ delete at[_] }
//at && (at[_] = function $(){ $.sort = Object.keys(at).sort(); return $ }());
return radix(key.slice(++i), val, at || (at = {}));
}
}

View File

@ -6,12 +6,22 @@ Gun.on('create', function(root){
var opt = root.opt, empty = {}, u;
if(false === opt.radisk){ return }
var Radisk = (Gun.window && Gun.window.Radisk) || require('./radisk');
var Radiskip = (Gun.window && Gun.window.Radisk) || require('./radiskip');
var Radix = Radisk.Radix;
var LOG = console.LOG, ST = 0;
opt.store = opt.store || (!Gun.window && require('./rfs')(opt));
var rad = Radisk(opt), esc = String.fromCharCode(27);
var dare = Radiskip(opt);
root.on('put2', function(msg){
this.to.next(msg);
var id = msg['#'], put = msg.put, soul = put['#'], key = put['.'], val = put[':'], state = put['>'], tmp;
dare(soul+key, {':': val, '>': state}, function(err, ok){
console.log("SAVED", soul, key, val, id);
});
});
root.on('put', function(msg){
this.to.next(msg);
var id = msg['#'] || Gun.text.random(3), track = !msg['@'], acks = track? 0 : u; // only ack non-acks.

110
sea.js
View File

@ -275,13 +275,13 @@
cb = salt;
salt = u;
}
salt = salt || shim.random(9);
data = (typeof data == 'string')? data : JSON.stringify(data);
if('sha' === (opt.name||'').toLowerCase().slice(0,3)){
var rsha = shim.Buffer.from(await sha(data, opt.name), 'binary').toString(opt.encode || 'base64')
if(cb){ try{ cb(rsha) }catch(e){console.log(e)} }
return rsha;
}
salt = salt || shim.random(9);
var key = await (shim.ossl || shim.subtle).importKey('raw', new shim.TextEncoder().encode(data), {name: opt.name || 'PBKDF2'}, false, ['deriveBits']);
var work = await (shim.ossl || shim.subtle).deriveBits({
name: opt.name || 'PBKDF2',
@ -1090,8 +1090,8 @@
})(USE, './create');
;USE(function(module){
const SEA = USE('./sea')
const Gun = SEA.Gun;
var SEA = USE('./sea')
var Gun = SEA.Gun;
// After we have a GUN extension to make user registration/login easy, we then need to handle everything else.
// We do this with a GUN adapter, we first listen to when a gun instance is created (and when its options change)
@ -1101,6 +1101,7 @@
at.on('in', security, at); // now listen to all input data, acting as a firewall.
at.on('out', signature, at); // and output listeners, to encrypt outgoing data.
at.on('node', each, at);
at.on('put2', check, at);
}
this.to.next(at); // make sure to call the "next" middleware adapter.
});
@ -1149,6 +1150,102 @@
security.call(this, msg);
}
var u;
function check(msg){
var eve = this, at = eve.as, put = msg.put, soul = put['#'], key = put['.'], val = put[':'], state = put['>'], id = msg['#'], tmp;
//console.log("security check", msg);
var no = function(why){ at.on('in', {'@': id, err: why}) }
if('#' === soul){ // special case for content addressing immutable hashed data.
check.hash(eve, msg, val, key, soul, at, no); return;
}
if('~@' === soul){ // special case for shared system data, the list of aliases.
check.alias(eve, msg, val, key, soul, at, no); return;
}
if('~@' === soul.slice(0,2)){ // special case for shared system data, the list of public keys for an alias.
check.pubs(eve, msg, val, key, soul, at, no); return;
}
if('~' === soul.slice(0,1) && 2 === (tmp = soul.slice(1)).split('.').length){ // special case, account data for a public key.
check.pub(eve, msg, val, key, soul, at, no, (msg._||noop).user, tmp); return;
}
check.any(eve, msg, val, key, soul, at, no, (msg._||noop).user); return;
eve.to.next(msg); // not handled
}
check.hash = function(eve, msg, val, key, soul, at, no){
SEA.work(val, null, function(data){
if(data === key){ return eve.to.next(msg) }
no("Data hash not same as hash!");
}, {name: 'SHA-256'});
}
check.alias = function(eve, msg, val, key, soul, at, no){ // Example: {_:#~@, ~@alice: {#~@alice}}
if(!val){ return no("Data must exist!") } // data MUST exist
if('~@'+key === link_is(val)){ return eve.to.next(msg) } // in fact, it must be EXACTLY equal to itself
no("Alias not same!"); // if it isn't, reject.
};
check.pubs = function(eve, msg, val, key, soul, at, no){ // Example: {_:#~@alice, ~asdf: {#~asdf}}
if(!val){ return no("Alias must exist!") } // data MUST exist
if(key === link_is(val)){ return eve.to.next(msg) } // and the ID must be EXACTLY equal to its property
no("Alias not same!"); // that way nobody can tamper with the list of public keys.
};
check.pub = function(eve, msg, val, key, soul, at, no, user, pub){ var tmp; // Example: {_:#~asdf, hello:'world'~fdsa}}
if('pub' === key){
if(val === pub){ return eve.to.next(msg) } // the account MUST match `pub` property that equals the ID of the public key.
return no("Account not same!");
}
check['user'+soul+key] = 1;
if(Gun.is(msg.$) && user && user.is && pub === user.is.pub){
SEA.sign(msg.put, (user._).sea, function(data){ var rel;
if(u === data){ return no(SEA.err || 'Signature fail.') }
if(rel = link_is(val)){ (at.sea.own[rel] = at.sea.own[rel] || {})[pub] = 1 }
console.log("WHAT HAPPENS HERE?", data.m, SEA.opt.unpack(data.m), key, soul);
msg.put[':'] = JSON.stringify({':': SEA.opt.unpack(data.m), '~': data.s});
//node[key] = JSON.stringify({':': SEA.opt.unpack(data.m), '~': data.s});
eve.to.next(msg);
}, {check: msg.put, raw: 1});
return;
}
SEA.verify(msg.put, pub, function(data){ var rel, tmp;
console.log("WHAT VERIFIES HERE?", data, SEA.opt.unpack(data, key), key, soul);
data = SEA.opt.unpack(data, key);
if(u === data){ return no("Unverified data.") } // make sure the signature matches the account it claims to be on. // reject any updates that are signed with a mismatched account.
if((rel = link_is(data)) && pub === SEA.opt.pub(rel)){
(at.sea.own[rel] = at.sea.own[rel] || {})[pub] = 1;
}
eve.to.next(msg);
});
};
check.any = function(eve, msg, val, key, soul, at, no, user){ var tmp, pub;
if(!(pub = SEA.opt.pub(soul))){
if(at.opt.secure){ return no("Soul missing public key at '" + key + "'.") }
// TODO: Ask community if should auto-sign non user-graph data.
at.on('secure', function(msg){ this.off();
if(!at.opt.secure){ return eve.to.next(msg) }
no("Data cannot be changed.");
}).on.on('secure', msg);
return;
}
// TODO: DEDUP WITH check.pub ???
if(Gun.is(msg.$) && user && user.is && pub === user.is.pub){
SEA.sign(mgs.put, (user._).sea, function(data){
if(u === data){ return no('User signature fail.') }
console.log("WHAT HAPPENS HERE??", data.m, SEA.opt.unpack(data.m), key, soul);
msg.put[':'] = JSON.stringify({':': SEA.opt.unpack(data.m), '~': data.s});
//node[key] = JSON.stringify({':': SEA.opt.unpack(data.m), '~': data.s});
eve.to.next(msg);
}, {check: msg.put, raw: 1});
return;
}
SEA.verify(msg.put, pub, function(data){ var rel;
console.log("WHAT VERIFIES HERE?", data, SEA.opt.unpack(data, key), key, soul);
data = SEA.opt.unpack(data, key);
if(u === data){ return no("Not owner on '" + key + "'.") } // thanks @rogowski !
if((rel = link_is(data)) && pub === SEA.opt.pub(rel)){
(at.sea.own[rel] = at.sea.own[rel] || {})[pub] = 1;
}
eve.to.next(msg);
});
}
var link_is = Gun.val.link.is;
// okay! The security function handles all the heavy lifting.
// It needs to deal read and write of input and output of system data, account/public key data, and regular data.
// This is broken down into some pretty clear edge cases, let's go over them:
@ -1174,6 +1271,11 @@
}
}
if(msg.put){
/*
NOTICE: THIS IS OLD AND GETTING DEPRECATED.
ANY SECURITY CHANGES SHOULD HAPPEN ABOVE FIRST
THEN PORTED TO HERE.
*/
// potentially parallel async operations!!!
var check = {}, each = {}, u;
each.node = function(node, soul){
@ -1260,9 +1362,11 @@
return;
}*/
check['any'+soul+key] = 1;
console.log(val, key, node, soul, '...', SEA.opt.prep(tmp = SEA.opt.parse(val), key, node, soul));
SEA.sign(SEA.opt.prep(tmp = SEA.opt.parse(val), key, node, soul), (user._).sea, function(data){
if(u === data){ return each.end({err: 'My signature fail.'}) }
node[key] = JSON.stringify({':': SEA.opt.unpack(data.m), '~': data.s});
console.log(key, node[key], '...', data);
check['any'+soul+key] = 0;
each.end({ok: 1});
}, {check: SEA.opt.pack(tmp, key, node, soul), raw: 1});

View File

@ -22,34 +22,62 @@
//window.radText = Radisk.encode(window.BigText);
window.namez = ["Adalard","Adora","Aia","Albertina","Alfie","Allyn","Amabil","Ammamaria","Andy","Anselme","Ardeen","Armand","Ashelman","Aube","Averyl","Baker","Barger","Baten","Bee","Benia","Bernat","Bevers","Bittner","Bobbe","Bonny","Boyce","Breech","Brittaney","Bryn","Burkitt","Cadmann","Campagna","Carlee","Carver","Cavallaro","Chainey","Chaunce","Ching","Cianca","Claudina","Clyve","Colon","Cooke","Corrina","Crawley","Cullie","Dacy","Daniela","Daryn","Deedee","Denie","Devland","Dimitri","Dolphin","Dorinda","Dream","Dunham","Eachelle","Edina","Eisenstark","Elish","Elvis","Eng","Erland","Ethan","Evelyn","Fairman","Faus","Fenner","Fillander","Flip","Foskett","Fredette","Fullerton","Gamali","Gaspar","Gemina","Germana","Gilberto","Giuditta","Goer","Gotcher","Greenstein","Grosvenor","Guthrey","Haldane","Hankins","Harriette","Hayman","Heise","Hepsiba","Hewie","Hiroshi","Holtorf","Howlond","Hurless","Ieso","Ingold","Isidora","Jacoba","Janelle","Jaye","Jennee","Jillana","Johnson","Josy","Justinian","Kannan","Kast","Keeley","Kennett","Kho","Kiran","Knowles","Koser","Kroll","LaMori","Lanctot","Lasky","Laverna","Leff","Leonanie","Lewert","Lilybel","Lissak","Longerich","Lou","Ludeman","Lyman","Madai","Maia","Malvina","Marcy","Maris","Martens","Mathilda","Maye","McLain","Melamie","Meras","Micco","Millburn","Mittel","Montfort","Moth","Mutz","Nananne","Nazler","Nesta","Nicolina","Noellyn","Nuli","Ody","Olympie","Orlena","Other","Pain","Parry","Paynter","Pentheas","Pettifer","Phyllida","Plath","Posehn","Proulx","Quinlan","Raimes","Ras","Redmer","Renelle","Ricard","Rior","Rocky","Ron","Rosetta","Rubia","Ruttger","Salbu","Sandy","Saw","Scholz","Secor","September","Shanleigh","Shenan","Sholes","Sig","Sisely","Soble","Spanos","Stanwinn","Stevie","Stu","Suzanne","Tacy","Tanney","Tekla","Thackeray","Thomasin","Tilla","Tomas","Tracay","Tristis","Ty","Urana","Valdis","Vasta","Vezza","Vitoria","Wait","Warring","Weissmann","Whetstone","Williamson","Wittenburg","Wymore","Yoho","Zamir","Zimmermann"];
window.radiz = window.radiz || Radix();
window.radix = window.radix || Radix();
window.arr = []; var i = 1000; while(--i){ arr.push(Math.random()) }
window.arrs = arr.slice(0).sort();
window.ALLZ = window.ALLZ || {};
window.namez.forEach(function(v,i){ ALLZ[v] = i });
});
/* TEMPORARY COPY OF RADIX UNIT TESTS TO BOOST SPEED */
/* THESE ARE PROBABLY STALE AND NEED TO BE COPIED FROM UNIT TESTS AGAIN */
/*stool.add('map', function(){
Gun.obj.map(ALLZ, function(v,i){
v;
});
});
stool.add('for', function(){
for(var k in ALLZ){
ALLZ[k];
}
});
stool.add('for', function(){
Object.keys(ALLZ).forEach(function(k){
ALLZ[k];
})
});
return;*/
stool.add('1', function(){
var rad = Radix();
rad('asdf.pub', 'yum');
rad('ablah', 'cool');
rad('ab', {yes: 1});
rad('node/circle.bob', 'awesome');
(rad('asdf.').pub[''] !== 'yum') && bad1;
(rad('nv/foo.bar') !== undefined) && bad2;
(JSON.stringify(rad('asdf.')) !== JSON.stringify({pub: {'': 'yum'}})) && bada;
(rad('nv/foo.bar') != undefined) && badb;
(JSON.stringify(rad('ab')) != JSON.stringify({yes: 1})) && badc
(JSON.stringify(rad()) != JSON.stringify({"a":{"sdf.pub":{"":"yum"},"b":{"lah":{"":"cool"},"":{"yes":1}}},"node/circle.bob":{"":"awesome"}})) && badd;
});
stool.add('2', function(){
var all = {};
namez.forEach(function(v,i){
v = v.toLowerCase();
all[v] = v;
radiz(v, i)
ALLZ[v] = i;
radix(v, i)
});
(Gun.obj.empty(all) === true) && bad3;
Radix.map(radiz, function(v,k){
Radix.map(radix, function(v,k){
delete all[k];
});
(Gun.obj.empty(all) !== true) && bad4;
});
stool.add('fast?', function(){
ALLZ['rubia'];
});
stool.add('fastest?', function(){
namez.indexOf('Rubia');
});
stool.add('3', function(){
var all = {};
namez.forEach(function(v,i){
@ -58,7 +86,7 @@
//rad(v, i)
});
(Gun.obj.empty(all) === true) && bad5;
Radix.map(radiz, function(v,k){
Radix.map(radix, function(v,k){
delete all[k];
});
(Gun.obj.empty(all) !== true) && bad6;
@ -73,7 +101,7 @@
//rad(v, i)
});
(Gun.obj.empty(all) === true) && bad7;
Radix.map(radiz, function(v,k, a,b){
Radix.map(radix, function(v,k, a,b){
//if(!all[k]){ throw "out of range!" }
delete all[k];
}, {start: start, end: end});
@ -89,12 +117,19 @@
//rad(v, i)
});
(Gun.obj.empty(all) === true) && bad9;
Radix.map(radiz, function(v,k, a,b){
Radix.map(radix, function(v,k, a,b){
//if(!all[k]){ throw "out of range!" }
delete all[k];
}, {start: start, end: end});
(Gun.obj.empty(all) !== true) && bad10;
});
stool.add('reverse item', function(){
Radix.map(radix, function(v,k, a,b){
(k !== 'ieso') && badri;
(v !== 96) && badri2;
return true;
}, {reverse: 1, end: 'iesogon'});
});
stool.add('6', function(){
var r = Radix(), tmp;
r('alice', 1);r('bob', 2);r('carl', 3);r('carlo',4);

View File

@ -50,10 +50,13 @@ var names = ["Adalard","Adora","Aia","Albertina","Alfie","Allyn","Amabil","Ammam
var rad = Radix();
rad('asdf.pub', 'yum');
rad('ablah', 'cool');
rad('ab', {yes: 1});
rad('node/circle.bob', 'awesome');
expect(rad('asdf.')).to.be.eql({pub: {'': 'yum'}});
expect(Gun.obj.copy(rad('asdf.'))).to.be.eql({pub: {'': 'yum'}});
expect(rad('nv/foo.bar')).to.be(undefined);
expect(rad('ab')).to.eql({yes: 1});
expect(Gun.obj.copy(rad())).to.be.eql({"a":{"sdf.pub":{"":"yum"},"b":{"lah":{"":"cool"},"":{"yes":1}}},"node/circle.bob":{"":"awesome"}});
});
it('radix write read', function(done){
@ -121,6 +124,16 @@ var names = ["Adalard","Adora","Aia","Albertina","Alfie","Allyn","Amabil","Ammam
expect(Gun.obj.empty(all)).to.be.ok();
done();
});
it('radix reverse item', function(done){
var opt = {reverse: 1, end: 'iesogon'};
Radix.map(radix, function(v,k, a,b){
expect(k).to.be('ieso');
expect(v).to.be(96);
return true;
}, opt);
done();
});
it('radix reverse', function(done){
var r = Radix(), tmp;