diff --git a/examples/todo/index.html b/examples/todo/index.html index e1061c57..c8a5d305 100644 --- a/examples/todo/index.html +++ b/examples/todo/index.html @@ -120,7 +120,7 @@ var w = 2; function go(i){ var S = +new Date; - var ref = gun.get(Gun.text.random(12)).put({data: Math.random()}, function(ack){ + var ref = gun.get(Gun.text.random(12)).put({data: Math.random(), ah: Math.random()}, function(ack){ console.log((+new Date - S)/1000, ack.err, ack.ok); ref.off(); }); diff --git a/gun.js b/gun.js index b4f81ef9..7831f053 100644 --- a/gun.js +++ b/gun.js @@ -359,7 +359,7 @@ /*if(perf){ t = start + perf.now(); // Danger: Accuracy decays significantly over time, even if precise. } else {*/ - t = time(); + t = +new Date; //} if(last < t){ return N = 0, last = t + State.drift; @@ -368,10 +368,10 @@ } var time = Type.time.is, last = -Infinity, N = 0, D = 1000; // WARNING! In the future, on machines that are D times faster than 2016AD machines, you will want to increase D by another several orders of magnitude so the processing speed never out paces the decimal resolution (increasing an integer effects the state accuracy). var perf = (typeof performance !== 'undefined')? (performance.timing && performance) : false, start = (perf && perf.timing && perf.timing.navigationStart) || (perf = false); - State._ = '>'; + var S_ = State._ = '>'; State.drift = 0; State.is = function(n, k, o){ // convenience function to get the state on a key on a node and return it. - var tmp = (k && n && n[N_] && n[N_][State._]) || o; + var tmp = (k && n && n[N_] && n[N_][S_]) || o; if(!tmp){ return } return num_is(tmp = tmp[k])? tmp : -Infinity; } @@ -383,7 +383,7 @@ } n = Node.soul.ify(n, soul); // then make it so! } - var tmp = obj_as(n[N_], State._); // grab the states data. + var tmp = obj_as(n[N_], S_); // grab the states data. if(u !== k && k !== N_){ if(num_is(s)){ tmp[k] = s; // add the valid state. @@ -618,40 +618,37 @@ ;USE(function(module){ var Type = USE('./type'); function Dup(opt){ - var dup = {s:{}}; + var dup = {s:{}}, s = dup.s; opt = opt || {max: 1000, age: /*1000 * 9};//*/ 1000 * 9 * 3}; - dup.check = function(id){ var tmp; - if(!(tmp = dup.s[id])){ return false } - if(tmp.pass){ return tmp.pass = false } + dup.check = function(id){ + if(!s[id]){ return false } return dup.track(id); } - dup.track = function(id, pass){ - var it = dup.s[id] || (dup.s[id] = {}); - it.was = time_is(); - if(pass){ it.pass = true } + dup.track = function(id){ + var it = s[id] || (s[id] = {}); + it.was = +new Date; if(!dup.to){ dup.to = setTimeout(dup.drop, opt.age + 9) } return it; } dup.drop = function(age){ - var now = time_is(); - Type.obj.map(dup.s, function(it, id){ + var now = +new Date; + Type.obj.map(s, function(it, id){ if(it && (age || opt.age) > (now - it.was)){ return } - Type.obj.del(dup.s, id); + delete s[id]; }); dup.to = null; } return dup; } - var time_is = Type.time.is; module.exports = Dup; })(USE, './dup'); ;USE(function(module){ function Gun(o){ - if(o instanceof Gun){ return (this._ = {gun: this, $: this}).$ } + if(o instanceof Gun){ return (this._ = {$: this}).$ } if(!(this instanceof Gun)){ return new Gun(o) } - return Gun.create(this._ = {gun: this, $: this, opt: o}); + return Gun.create(this._ = {$: this, opt: o}); } Gun.is = function($){ return ($ instanceof Gun) || ($ && $._ && ($ === $._.$)) || false } @@ -682,7 +679,9 @@ var gun = at.$.opt(at.opt); if(!at.once){ at.on('in', root, at); + at.on('in2', root2, at); at.on('out', root, {at: at, out: root}); + at.on('put2', cache, at); Gun.on('create', at); at.on('create', at); } @@ -716,8 +715,89 @@ at.on('out', msg); } } + function root2(msg){ + if(!msg){ return } + var eve = this, as = eve.as, at = as.at || as, gun = at.$, dup = at.dup, tmp; + if(!(tmp = msg['#'])){ tmp = msg['#'] = text_rand(9) } + if(dup.check(tmp)){ return } dup.track(tmp); + tmp = msg._; msg._ = ('function' == typeof msg._)? msg._ : function(){}; + if(msg.get){ Gun.on.get(msg, gun) } + if(msg.put){ PUT(msg, gun) } + eve.to.next(msg); + } }()); + ;(function(){ + Gun.on.put2 = function(msg, gun){ + var ctx = msg._, root = ctx.root = gun._, id = msg['#']; + var ack = ctx.ack = function(put){ put = put || {}; + if(ack.err = ack.err || (put||{}).err){ + if(ack.s){ root.on('in', {'@': id, err: Gun.log(ack.err)}) } + return ack.s = u; + } + var soul = put['#'], key = put['.'], val = put[':'], state = put['>'], tmp; + if((tmp = ack.s||{})[soul+key]){ + delete tmp[soul+key]; + root.on('put2', {put: put}); + } + if(!obj_empty(ack.s)){ return } // keep waiting + console.log("I'm all done!!!"); + } + ack.err = obj_map(msg.put, nodes, msg); + msg = ctx = u; + if(ack.err){ return ack() } + } + function nodes(node, soul){ + if(!node){ return ERR+cut(soul)+"no node." } + var ctx = this._, tmp; + if(!(tmp = node._)){ return ERR+cut(soul)+"no meta." } + ctx.node = node; + if(soul !== tmp[_soul]){ return ERR+cut(soul)+"soul not same." } + ctx.soul = soul; + if(!(ctx.states = tmp[state_])){ return ERR+cut(soul)+"no state." } + return obj_map(node, souls, this); + } + function souls(val, key){ + if(node_ === key){ return } + var ctx = this._, node = ctx.node, soul = ctx.soul, state = ctx.states[key]; + if(u === state){ return ERR+cut(key)+"on"+cut(soul)+"no state." } + if(!val_is(val)){ return ERR+cut(key)+"on"+cut(soul)+"bad "+(typeof val)+cut(val) } + ham(ctx.root.graph, soul, key, val, state, ctx.ack); // TODO: HANDLE CALLBACK WHERE ALL DAY IS HISTORIC? + } + function ham(graph, soul, key, val, state, ack){ + (ack.s || (ack.s = {}))[soul+key] = 1; + var vertex = graph[soul] || empty, was = state_is(vertex, key, 1), known = vertex[key]; + var machine = State(), is = HAM(machine, state, was, val, known), u; + if(!is.incoming){ + if(is.defer){ + var to = is.defer - machine; + setTimeout(function(){ + ham(graph, soul, key, val, state, cb); + }, to > MD? MD : to); // setTimeout Max Defer 32bit :( + return; + } + if(ack){ ack({'#': soul, '.': key}) } + return; + } + if(ack){ ack({'#': soul, '.': key, ':': val, '>': state}) } + } + var cut = function(s){ return " '"+(''+s).slice(0,9)+"...' " } + var ERR = "Error: Invalid graph!"; + }()); + var PUT = Gun.on.put2; + var HAM = Gun.HAM, MD = 2147483647, State = Gun.state; + function cache(msg){ + var eve = this, root = eve.as, graph = root.graph; + var put = msg.put, soul = put['#'], key = put['.'], val = put[':'], state = put['>']; + graph[soul] = state_ify(graph[soul], key, state, val, soul); + // trigger chains // trigger after all individual values cached? + if(eve.to.last === eve.to.next){ + console.log("THE END, reply back"); + return; + } + eve.to.next(msg); + } + ;(function(){ Gun.on.put = function(msg, gun){ var at = gun._, ctx = {$: gun, graph: at.graph, put: {}, map: {}, souls: {}, machine: Gun.state(), ack: msg['@'], cat: at, stop: {}}; @@ -735,7 +815,6 @@ if(!ctx.diff){ return } at.on('put', obj_to(msg, {put: ctx.diff})); }; - var MD = 2147483647; function verify(val, key, node, soul){ var ctx = this; var state = Gun.state.is(node, key), tmp; if(!state){ return ctx.err = "Error: No state on '"+key+"' in node '"+soul+"'!" } @@ -860,8 +939,8 @@ var list_is = Gun.list.is; var text = Gun.text, text_is = text.is, text_rand = text.random; - var obj = Gun.obj, obj_is = obj.is, obj_has = obj.has, obj_to = obj.to, obj_map = obj.map, obj_copy = obj.copy; - var state_lex = Gun.state.lex, _soul = Gun.val.link._, _has = '.', node_ = Gun.node._, rel_is = Gun.val.link.is; + var obj = Gun.obj, obj_empty = obj.empty, obj_is = obj.is, obj_has = obj.has, obj_to = obj.to, obj_map = obj.map, obj_copy = obj.copy; + var state_lex = Gun.state.lex, state_ify = Gun.state.ify, state_is = Gun.state.is, _soul = Gun.val.link._, _has = '.', node_ = Gun.node._, val_is = Gun.val.is, rel_is = Gun.val.link.is, state_ = Gun.state._; var empty = {}, u; console.only = function(i, s){ return (console.only.i && i === console.only.i && console.only.i++) && (console.log.apply(console, arguments) || s) }; @@ -1331,7 +1410,7 @@ if(eve.seen && at.id && eve.seen[at.id]){ return eve.to.next(msg) } //if((tmp = root.stop)){ if(tmp[at.id]){ return } tmp[at.id] = msg.root; } // temporary fix till a better solution? if((tmp = data) && tmp[rel._] && (tmp = rel.is(tmp))){ - tmp = ((msg.$$ = at.root.gun.get(tmp))._); + tmp = ((msg.$$ = at.root.$.get(tmp))._); if(u !== tmp.put){ msg = obj_to(msg, {put: data = tmp.put}); } @@ -1995,10 +2074,9 @@ try{msg = msg || JSON.parse(raw); }catch(e){return opt.log('DAM JSON parse error', e)} if(!msg){ return } + if(msg.DBG_s){ opt.log(+new Date - msg.DBG_s, 'to hear', msg['#']) } if(!(id = msg['#'])){ id = msg['#'] = Type.text.random(9) } - if(msg.DBG_s){ opt.log(+new Date - msg.DBG_s, 'to hear', id) } if(dup.check(id)){ return } - dup.track(id, true); //.it = it(msg); // GUN core also dedups, so `true` is needed. // Does GUN core need to dedup anymore? /*if(!(hash = msg['##']) && u !== msg.put){ hash = msg['##'] = Type.obj.hash(msg.put) } if(hash && (tmp = msg['@'] || (msg.get && id))){ // Reduces backward daisy in case varying hashes at different daisy depths are the same. if(dup.check(tmp+hash)){ return } @@ -2011,11 +2089,13 @@ if(tmp = mesh.hear[msg.dam]){ tmp(msg, peer, root); } + dup.track(id); return; } var S, ST; LOG && (S = +new Date); console.STAT = {}; //root.on('in', msg); root.on('in2', msg); + dup.track(id); if(LOG && !msg.nts && (ST = +new Date - S) > 9){ opt.log(S, ST, 'msg', msg['#'], JSON.stringify(console.STAT)); if(ST > 500){ try{ require('./lib/email').send({text: ""+ST+"ms "+JSON.stringify(msg)+" | "+JSON.stringify(console.STAT), from: "mark@gun.eco", to: "mark@gun.eco", subject: "GUN MSG"}, noop); }catch(e){} } } // this is ONLY turned on if ENV CONFIGS have email/password to send out from. return; } diff --git a/lib/radiskip.js b/lib/radiskip.js new file mode 100644 index 00000000..70c86c7b --- /dev/null +++ b/lib/radiskip.js @@ -0,0 +1,528 @@ +;(function(){ + + function Radisk(opt){ + + opt = opt || {}; + opt.log = opt.log || console.log; + opt.file = String(opt.file || 'radata'); + var has = (Radisk.has || (Radisk.has = {}))[opt.file]; + if(has){ return has } + + opt.pack = opt.pack || (opt.memory? (opt.memory * 1000 * 1000) : 1399000000) * 0.3; // max_old_space_size defaults to 1400 MB. + opt.until = opt.until || opt.wait || 250; + opt.batch = opt.batch || (10 * 1000); + opt.chunk = opt.chunk || (1024 * 1024 * 1); // 1MB + opt.code = opt.code || {}; + opt.code.from = opt.code.from || '!'; + opt.jsonify = true; + + function ename(t){ return encodeURIComponent(t).replace(/\*/g, '%2A') } + function atomic(v){ return u !== v && (!v || 'object' != typeof v) } + var map = Gun.obj.map; + var LOG = console.LOG; + var ST = 0; + + if(!opt.store){ + return opt.log("ERROR: Radisk needs `opt.store` interface with `{get: fn, put: fn (, list: fn)}`!"); + } + if(!opt.store.put){ + return opt.log("ERROR: Radisk needs `store.put` interface with `(file, data, cb)`!"); + } + if(!opt.store.get){ + return opt.log("ERROR: Radisk needs `store.get` interface with `(file, cb)`!"); + } + if(!opt.store.list){ + //opt.log("WARNING: `store.list` interface might be needed!"); + } + + /* + Any and all storage adapters should... + 1. Because writing to disk takes time, we should batch data to disk. This improves performance, and reduces potential disk corruption. + 2. If a batch exceeds a certain number of writes, we should immediately write to disk when physically possible. This caps total performance, but reduces potential loss. + */ + var r = function(key, data, cb){ + if('function' === typeof data){ + var o = cb || {}; + cb = val; + return; + } + //var tmp = (tmp = r.batch = r.batch || {})[key] = tmp[key] || {}; + //var tmp = (tmp = r.batch = r.batch || {})[key] = data; + r.save(key, data, cb); + } + r.save = function(key, data, cb){ + var s = {key: key}; + s.find = function(file){ var tmp; + s.file = file = file || opt.code.from; + if(tmp = r.disk[file]){ return s.mix(null, tmp) } + r.parse(file, s.mix); + } + s.mix = function(err, disk){ + if(err){ return cb(err) } + s.file = (disk||noop).file || s.file; + ((disk = r.disk[s.file] || disk)||noop).file = s.file; + if(!disk && s.file !== opt.code.from){ // corrupt file? + r.find.bad(s.file); // remove from dir list + r.save(key, data, cb); // try again + return; + } + (r.disk[s.file] = disk = disk || Radix()).file = s.file; + if(opt.compare){ + data = opt.compare(disk(key), data, key, s.file); + if(u === data){ return cb(err, -1) } + } + // check if still in same r.find? + (s.disk = disk)(key, data); + if(disk.Q){ return disk.Q.push(cb) } disk.Q = [cb]; + disk.to = setTimeout(s.write, opt.until); + } + s.write = function(){ + var q = s.disk.Q; + delete s.disk.Q; + delete r.disk[s.file]; + r.write(s.file, s.disk, function(err, ok){ + Gun.obj.map(q, function(ack){ ack(err, ok) }); + }); + } + r.find(key, s.find); + } + r.disk = {}; + + /* + Any storage engine at some point will have to do a read in order to write. + This is true of even systems that use an append only log, if they support updates. + Therefore it is unavoidable that a read will have to happen, + the question is just how long you delay it. + */ + r.write = function(file, rad, cb, o){ + if(!rad){ return cb('No radix!') } + o = ('object' == typeof o)? o : {force: o}; + var f = function Fractal(){}, a, b; + f.text = ''; + f.file = file = rad.file = rad.file || file; + if(!file){ return cb('What file?') } + f.write = function(){ + var S; LOG && (S = +new Date); + r.disk[file = rad.file || f.file || file] = rad; + r.find.add(file, function(err){ + if(err){ return cb(err) } + opt.store.put(ename(file), f.text, function(err, ok){ + LOG && opt.log(S, ST = +new Date - S, "wrote disk", JSON.stringify(file)); + cb(err, ok); + delete r.disk[file]; + }); + }); + } + f.split = function(){ + f.text = ''; + if(!f.count){ f.count = 0; + Radix.map(rad, function(){ f.count++ }); // TODO: Perf? Any faster way to get total length? + } + f.limit = Math.ceil(f.count/2); + f.count = 0; + f.sub = Radix(); + Radix.map(rad, f.slice, {reverse: 1}); // IMPORTANT: DO THIS IN REVERSE, SO LAST HALF OF DATA MOVED TO NEW FILE BEFORE DROPPING FROM CURRENT FILE. + r.write(f.end, f.sub, f.both, o); + f.hub = Radix(); + Radix.map(rad, f.stop); + r.write(rad.file, f.hub, f.both, o); + return true; + } + f.slice = function(val, key){ + f.sub(f.end = key, val); + if(f.limit <= (++f.count)){ return true } + } + f.stop = function(val, key){ + if(key >= f.end){ return true } + f.hub(key, val); + } + f.both = function(err, ok){ + if(b){ return cb(err || b) } + if(a){ return cb(err, ok) } + a = true; + b = err; + } + f.each = function(val, key, k, pre){ + //console.log("RAD:::", JSON.stringify([val, key, k, pre])); + if(u !== val){ f.count++ } + if(opt.pack <= (val||'').length){ return cb("Record too big!"), true } + var enc = Radisk.encode(pre.length) +'#'+ Radisk.encode(k) + (u === val? '' : ':'+ Radisk.encode(val)) +'\n'; + if((opt.chunk < f.text.length + enc.length) && (1 < f.count) && !o.force){ + return f.split(); + } + f.text += enc; + } + if(opt.jsonify){ return r.write.jsonify(f, rad, cb, o) } // temporary testing idea + if(!Radix.map(rad, f.each, true)){ f.write() } + } + + r.write.jsonify = function(f, rad, cb, o){ + var raw; + var S; LOG && (S = +new Date); + try{raw = JSON.stringify(rad.$); + }catch(e){ return cb("Cannot radisk!") } + LOG && opt.log(S, +new Date - S, "rad stringified JSON"); + if(opt.chunk < raw.length && !o.force){ + return f.split(); + //if(Radix.map(rad, f.each, true)){ return } + } + f.text = raw; + f.write(); + } + + r.range = function(tree, o){ + if(!tree || !o){ return } + if(u === o.start && u === o.end){ return tree } + if(atomic(tree)){ return tree } + var sub = Radix(); + Radix.map(tree, function(v,k){ // ONLY PLACE THAT TAKES TREE, maybe reduce API for better perf? + sub(k,v); + }, o); + return sub(''); + } + + ;(function(){ + var Q = {}; + r.read = function(key, cb, o){ + o = o || {}; + if(RAD && !o.next){ // cache + var S; LOG && (S = +new Date); + var val = RAD(key); + LOG && (ST = +new Date - S) > 9 && opt.log(S, ST, 'rad cached'); + //if(u !== val){ + //cb(u, val, o); + if(atomic(val)){ cb(u, val, o); return } + // if a node is requested and some of it is cached... the other parts might not be. + //} + } + o.span = (u !== o.start) || (u !== o.end); // is there a start or end? + var g = function Get(){}; + g.lex = function(file){ var tmp; // // TODO: this had a out-of-memory crash! + file = (u === file)? u : decodeURIComponent(file); + tmp = o.next || key || (o.reverse? o.end || '\uffff' : o.start || ''); + if(!file || (o.reverse? file < tmp : file > tmp)){ + LOG && opt.log(S, +new Date - S, 'rad read lex'); S = +new Date; + if(o.next || o.reverse){ g.file = file } + if(tmp = Q[g.file]){ + tmp.push({key: key, ack: cb, file: g.file, opt: o}); + return true; + } + Q[g.file] = [{key: key, ack: cb, file: g.file, opt: o}]; + if(!g.file){ + g.it(null, u, {}); + return true; + } + r.parse(g.file, g.check); + return true; + } + g.file = file; + } + g.it = function(err, disk, info){ + if(g.err = err){ opt.log('err', err) } + if(!disk && g.file){ // corrupt file? + r.find.bad(g.file); // remove from dir list + r.read(key, cb, o); // look again + return; + } + g.info = info; + if(disk){ RAD = g.disk = disk } + disk = Q[g.file]; delete Q[g.file]; + map(disk, g.ack); + } + g.ack = function(as){ + if(!as.ack){ return } + var S; LOG && (S = +new Date); + var key = as.key, o = as.opt, info = g.info, rad = g.disk || noop, data = r.range(rad(key), o), last = rad.last || Radix.map(rad, rev, revo); + LOG && (ST = +new Date - S) > 9 && opt.log(S, ST, "rad range loaded"); + o.parsed = (o.parsed || 0) + (info.parsed||0); + o.chunks = (o.chunks || 0) + 1; + o.more = true; + if((!as.file) // if no more places to look + || (!o.span && last === key) // if our key exactly matches the very last atomic record + || (!o.span && last && last > key && 0 != last.indexOf(key)) // 'zach' may be lexically larger than 'za', but there still might be more, like 'zane' in the 'za' prefix bucket so do not end here. + ){ + o.more = u; + as.ack(g.err, data, o); + return + } + if(u !== data){ + as.ack(g.err, data, o); // more might be coming! + if(o.parsed >= o.limit){ return } // even if more, we've hit our limit, asking peer will need to make a new ask with a new starting point. + } + o.next = as.file; + r.read(key, as.ack, o); + } + g.check = function(err, disk, info){ + g.it(err, disk, info); + var good = true; + Radix.map(disk, function(val, key){ + // assume in memory for now, since both write/read already call r.find which will init it. + var go = function(file){ + if(info.file !== file){ + good = false + } + return true; + } + go.reverse = 1; + go.end = key; + r.list(go); + }); + if(good){ return } + var id = Gun.text.random(3); + r.save(disk, function ack(err, ok){ + if(err){ return r.save(disk, ack) } // ad infinitum??? + console.log("MISLOCATED DATA CORRECTED", id); + }); + } + /*g.check2 = function(err, disk, info){ + if(err || !disk){ return g.it(err, disk, info) } + var good = true; + Radix.map(disk, function(val, key){ + // assume in memory for now, since both write/read already call r.find which will init it. + var go = function(file){ + if(info.file !== file){ good = false } + return true; + } + go.reverse = 1; + go.end = key; + r.list(go); + }); + if(good){ return g.it(err, disk, info) } + var id = Gun.text.random(3); console.log("MISLOCATED DATA", id); + r.save(disk, function ack(err, ok){ + if(err){ return r.save(disk, ack) } // ad infinitum??? + console.log("MISLOCATED CORRECTED", id); + r.read(key, cb, o); + }); + }*/ + if(o.reverse){ g.lex.reverse = true } + LOG && (S = +new Date); + r.find(key, g.lex); + } + function rev(a,b){ return b } + var revo = {reverse: true}; + }()); + + ;(function(){ + /* + Let us start by assuming we are the only process that is + changing the directory or bucket. Not because we do not want + to be multi-process/machine, but because we want to experiment + with how much performance and scale we can get out of only one. + Then we can work on the harder problem of being multi-process. + */ + var Q = {}, s = String.fromCharCode(31); + r.parse = function(file, cb, raw){ var q; + if(q = Q[file]){ return q.push(cb) } q = Q[file] = [cb]; + var p = function Parse(){}, info = {file: file}; + (p.disk = Radix()).file = file; + p.read = function(err, data){ var tmp; + LOG && opt.log(S, +new Date - S, 'read disk', JSON.stringify(file)); + delete Q[file]; + if((p.err = err) || (p.not = !data)){ return map(q, p.ack) } + if('string' !== typeof data){ + try{ + if(opt.pack <= data.length){ + p.err = "Chunk too big!"; + } else { + data = data.toString(); // If it crashes, it crashes here. How!?? We check size first! + } + }catch(e){ p.err = e } + if(p.err){ return map(q, p.ack) } + } + info.parsed = data.length; + LOG && (S = +new Date); + if(opt.jsonify || '{' === data[0]){ + try{ + var json = JSON.parse(data); // TODO: this caused a out-of-memory crash! + p.disk.$ = json; + LOG && (ST = +new Date - S) > 9 && opt.log(S, ST, 'rad parsed JSON'); + map(q, p.ack); + return; + }catch(e){ tmp = e } + if('{' === data[0]){ + p.err = tmp || "JSON error!"; + return map(q, p.ack); + } + } + return p.radec(err, data); + } + p.ack = function(cb){ + if(!cb){ return } + if(p.err || p.not){ return cb(p.err, u, info) } + cb(u, p.disk, info); + } + p.radec = function(err, data){ + LOG && (S = +new Date); + var tmp = p.split(data), pre = [], i, k, v; + if(!tmp || 0 !== tmp[1]){ + p.err = "File '"+file+"' does not have root radix! "; + return map(q, p.ack); + } + while(tmp){ + k = v = u; + i = tmp[1]; + tmp = p.split(tmp[2])||''; + if('#' == tmp[0]){ + k = tmp[1]; + pre = pre.slice(0,i); + if(i <= pre.length){ + pre.push(k); + } + } + tmp = p.split(tmp[2])||''; + if('\n' == tmp[0]){ continue } + if('=' == tmp[0] || ':' == tmp[0]){ v = tmp[1] } + if(u !== k && u !== v){ p.disk(pre.join(''), v) } + tmp = p.split(tmp[2]); + } + LOG && opt.log(S, +new Date - S, 'parsed RAD'); + map(q, p.ack); + }; + p.split = function(t){ + if(!t){ return } + var l = [], o = {}, i = -1, a = '', b, c; + i = t.indexOf(s); + if(!t[i]){ return } + a = t.slice(0, i); + l[0] = a; + l[1] = b = Radisk.decode(t.slice(i), o); + l[2] = t.slice(i + o.i); + return l; + } + var S; LOG && (S = +new Date); + if(raw){ return p.read(null, raw) } + opt.store.get(ename(file), p.read); + } + }()); + + ;(function(){ + var dir, f = String.fromCharCode(28), Q; + r.find = function(key, cb){ + if(!dir){ + if(Q){ return Q.push([key, cb]) } Q = [[key, cb]]; + return r.parse(f, init); + } + Radix.map(dir, function(val, key){ + if(!val){ return } + return cb(key) || true; + }, {reverse: 1, end: key}) || cb(); + } + r.find.add = function(file, cb){ + var has = dir(file); + if(has || file === f){ return cb(u, 1) } + dir(file, 1); + cb.found = (cb.found || 0) + 1; + r.write(f, dir, function(err, ok){ + if(err){ return cb(err) } + cb.found = (cb.found || 0) - 1; + if(0 !== cb.found){ return } + cb(u, 1); + }, true); + } + r.find.bad = function(file, cb){ + dir(file, 0); + r.write(f, dir, cb||noop); + } + function init(err, disk){ + if(err){ + opt.log('list', err); + setTimeout(function(){ r.parse(f, init) }, 1000); + return; + } + if(disk){ return drain(disk) } + if(!opt.store.list){ return drain(disk || dir || Radix()) } + // import directory. + opt.store.list(function(file){ + dir = dir || Radix(); + if(!file){ return drain(dir) } + r.find.add(file, noop); + }); + } + function drain(rad, tmp){ + dir = dir || rad; + dir.file = f; + tmp = Q; Q = null; + Gun.list.map(tmp, function(arg){ + r.find(arg[0], arg[1]); + }); + } + }()); + + var noop = function(){}, RAD, u; + Radisk.has[opt.file] = r; + return r; + } + + + + ;(function(){ + var _ = String.fromCharCode(31), u; + Radisk.encode = function(d, o, s){ s = s || _; + var t = s, tmp; + if(typeof d == 'string'){ + var i = d.indexOf(s); + while(i != -1){ t += s; i = d.indexOf(s, i+1) } + return t + '"' + d + s; + } else + if(d && d['#'] && (tmp = Gun.val.link.is(d))){ + return t + '#' + tmp + t; + } else + if(Gun.num.is(d)){ + return t + '+' + (d||0) + t; + } else + if(null === d){ + return t + ' ' + t; + } else + if(true === d){ + return t + '+' + t; + } else + if(false === d){ + return t + '-' + t; + }// else + //if(binary){} + } + Radisk.decode = function(t, o, s){ s = s || _; + var d = '', i = -1, n = 0, c, p; + if(s !== t[0]){ return } + while(s === t[++i]){ ++n } + p = t[c = n] || true; + while(--n >= 0){ i = t.indexOf(s, i+1) } + if(i == -1){ i = t.length } + d = t.slice(c+1, i); + if(o){ o.i = i+1 } + if('"' === p){ + return d; + } else + if('#' === p){ + return Gun.val.link.ify(d); + } else + if('+' === p){ + if(0 === d.length){ + return true; + } + return parseFloat(d); + } else + if(' ' === p){ + return null; + } else + if('-' === p){ + return false; + } + } + }()); + + if(typeof window !== "undefined"){ + var Gun = window.Gun; + var Radix = window.Radix; + window.Radisk = Radisk; + } else { + var Gun = require('../gun'); + var Radix = require('./radix'); + //var Radix = require('./radix2'); Radisk = require('./radisk2'); + try{ module.exports = Radisk }catch(e){} + } + + Radisk.Radix = Radix; + +}()); \ No newline at end of file diff --git a/lib/radix.js b/lib/radix.js index e338e3fb..ffd13c72 100644 --- a/lib/radix.js +++ b/lib/radix.js @@ -2,13 +2,13 @@ function Radix(){ var radix = function(key, val, t){ - key = ''+key; if(!t && u !== val){ - radix.last = (key < radix.last)? radix.last : key; + radix.last = (''+key < radix.last)? radix.last : ''+key; delete (radix.$||{})[_]; } t = t || radix.$ || (radix.$ = {}); if(!key && Object.keys(t).length){ return t } + key = ''+key; var i = 0, l = key.length-1, k = key[i], at, tmp; while(!(at = t[k]) && i < l){ k += key[++i]; @@ -22,19 +22,24 @@ if(kk){ if(u === val){ if(ii <= l){ return } - return (tmp || (tmp = {}))[s.slice(ii)] = r; + (tmp || (tmp = {}))[s.slice(ii)] = r; + //(tmp[_] = function $(){ $.sort = Object.keys(tmp).sort(); return $ }()); // get rid of this one, cause it is on read? + return r; } var __ = {}; __[s.slice(ii)] = r; ii = key.slice(ii); ('' === ii)? (__[''] = val) : ((__[ii] = {})[''] = val); + //(__[_] = function $(){ $.sort = Object.keys(__).sort(); return $ }()); t[kk] = __; delete t[s]; + //(t[_] = function $(){ $.sort = Object.keys(t).sort(); return $ }()); return true; } })){ if(u === val){ return; } (t[k] || (t[k] = {}))[''] = val; + //(t[_] = function $(){ $.sort = Object.keys(t).sort(); return $ }()); } if(u === val){ return tmp; @@ -43,8 +48,10 @@ if(i == l){ if(u === val){ return (u === (tmp = at['']))? at : tmp } at[''] = val; + //(at[_] = function $(){ $.sort = Object.keys(at).sort(); return $ }()); } else { if(u !== val){ delete at[_] } + //at && (at[_] = function $(){ $.sort = Object.keys(at).sort(); return $ }()); return radix(key.slice(++i), val, at || (at = {})); } } diff --git a/lib/store.js b/lib/store.js index 80c26c36..260fd77b 100644 --- a/lib/store.js +++ b/lib/store.js @@ -6,12 +6,22 @@ Gun.on('create', function(root){ var opt = root.opt, empty = {}, u; if(false === opt.radisk){ return } var Radisk = (Gun.window && Gun.window.Radisk) || require('./radisk'); + var Radiskip = (Gun.window && Gun.window.Radisk) || require('./radiskip'); var Radix = Radisk.Radix; var LOG = console.LOG, ST = 0; opt.store = opt.store || (!Gun.window && require('./rfs')(opt)); var rad = Radisk(opt), esc = String.fromCharCode(27); + var dare = Radiskip(opt); + root.on('put2', function(msg){ + this.to.next(msg); + var id = msg['#'], put = msg.put, soul = put['#'], key = put['.'], val = put[':'], state = put['>'], tmp; + dare(soul+key, {':': val, '>': state}, function(err, ok){ + console.log("SAVED", soul, key, val, id); + }); + }); + root.on('put', function(msg){ this.to.next(msg); var id = msg['#'] || Gun.text.random(3), track = !msg['@'], acks = track? 0 : u; // only ack non-acks. diff --git a/sea.js b/sea.js index 213fe57f..c653daf1 100644 --- a/sea.js +++ b/sea.js @@ -275,13 +275,13 @@ cb = salt; salt = u; } - salt = salt || shim.random(9); data = (typeof data == 'string')? data : JSON.stringify(data); if('sha' === (opt.name||'').toLowerCase().slice(0,3)){ var rsha = shim.Buffer.from(await sha(data, opt.name), 'binary').toString(opt.encode || 'base64') if(cb){ try{ cb(rsha) }catch(e){console.log(e)} } return rsha; } + salt = salt || shim.random(9); var key = await (shim.ossl || shim.subtle).importKey('raw', new shim.TextEncoder().encode(data), {name: opt.name || 'PBKDF2'}, false, ['deriveBits']); var work = await (shim.ossl || shim.subtle).deriveBits({ name: opt.name || 'PBKDF2', @@ -1090,8 +1090,8 @@ })(USE, './create'); ;USE(function(module){ - const SEA = USE('./sea') - const Gun = SEA.Gun; + var SEA = USE('./sea') + var Gun = SEA.Gun; // After we have a GUN extension to make user registration/login easy, we then need to handle everything else. // We do this with a GUN adapter, we first listen to when a gun instance is created (and when its options change) @@ -1101,6 +1101,7 @@ at.on('in', security, at); // now listen to all input data, acting as a firewall. at.on('out', signature, at); // and output listeners, to encrypt outgoing data. at.on('node', each, at); + at.on('put2', check, at); } this.to.next(at); // make sure to call the "next" middleware adapter. }); @@ -1149,6 +1150,102 @@ security.call(this, msg); } + var u; + function check(msg){ + var eve = this, at = eve.as, put = msg.put, soul = put['#'], key = put['.'], val = put[':'], state = put['>'], id = msg['#'], tmp; + //console.log("security check", msg); + var no = function(why){ at.on('in', {'@': id, err: why}) } + if('#' === soul){ // special case for content addressing immutable hashed data. + check.hash(eve, msg, val, key, soul, at, no); return; + } + if('~@' === soul){ // special case for shared system data, the list of aliases. + check.alias(eve, msg, val, key, soul, at, no); return; + } + if('~@' === soul.slice(0,2)){ // special case for shared system data, the list of public keys for an alias. + check.pubs(eve, msg, val, key, soul, at, no); return; + } + if('~' === soul.slice(0,1) && 2 === (tmp = soul.slice(1)).split('.').length){ // special case, account data for a public key. + check.pub(eve, msg, val, key, soul, at, no, (msg._||noop).user, tmp); return; + } + check.any(eve, msg, val, key, soul, at, no, (msg._||noop).user); return; + eve.to.next(msg); // not handled + } + check.hash = function(eve, msg, val, key, soul, at, no){ + SEA.work(val, null, function(data){ + if(data === key){ return eve.to.next(msg) } + no("Data hash not same as hash!"); + }, {name: 'SHA-256'}); + } + check.alias = function(eve, msg, val, key, soul, at, no){ // Example: {_:#~@, ~@alice: {#~@alice}} + if(!val){ return no("Data must exist!") } // data MUST exist + if('~@'+key === link_is(val)){ return eve.to.next(msg) } // in fact, it must be EXACTLY equal to itself + no("Alias not same!"); // if it isn't, reject. + }; + check.pubs = function(eve, msg, val, key, soul, at, no){ // Example: {_:#~@alice, ~asdf: {#~asdf}} + if(!val){ return no("Alias must exist!") } // data MUST exist + if(key === link_is(val)){ return eve.to.next(msg) } // and the ID must be EXACTLY equal to its property + no("Alias not same!"); // that way nobody can tamper with the list of public keys. + }; + check.pub = function(eve, msg, val, key, soul, at, no, user, pub){ var tmp; // Example: {_:#~asdf, hello:'world'~fdsa}} + if('pub' === key){ + if(val === pub){ return eve.to.next(msg) } // the account MUST match `pub` property that equals the ID of the public key. + return no("Account not same!"); + } + check['user'+soul+key] = 1; + if(Gun.is(msg.$) && user && user.is && pub === user.is.pub){ + SEA.sign(msg.put, (user._).sea, function(data){ var rel; + if(u === data){ return no(SEA.err || 'Signature fail.') } + if(rel = link_is(val)){ (at.sea.own[rel] = at.sea.own[rel] || {})[pub] = 1 } + console.log("WHAT HAPPENS HERE?", data.m, SEA.opt.unpack(data.m), key, soul); + msg.put[':'] = JSON.stringify({':': SEA.opt.unpack(data.m), '~': data.s}); + //node[key] = JSON.stringify({':': SEA.opt.unpack(data.m), '~': data.s}); + eve.to.next(msg); + }, {check: msg.put, raw: 1}); + return; + } + SEA.verify(msg.put, pub, function(data){ var rel, tmp; + console.log("WHAT VERIFIES HERE?", data, SEA.opt.unpack(data, key), key, soul); + data = SEA.opt.unpack(data, key); + if(u === data){ return no("Unverified data.") } // make sure the signature matches the account it claims to be on. // reject any updates that are signed with a mismatched account. + if((rel = link_is(data)) && pub === SEA.opt.pub(rel)){ + (at.sea.own[rel] = at.sea.own[rel] || {})[pub] = 1; + } + eve.to.next(msg); + }); + }; + check.any = function(eve, msg, val, key, soul, at, no, user){ var tmp, pub; + if(!(pub = SEA.opt.pub(soul))){ + if(at.opt.secure){ return no("Soul missing public key at '" + key + "'.") } + // TODO: Ask community if should auto-sign non user-graph data. + at.on('secure', function(msg){ this.off(); + if(!at.opt.secure){ return eve.to.next(msg) } + no("Data cannot be changed."); + }).on.on('secure', msg); + return; + } + // TODO: DEDUP WITH check.pub ??? + if(Gun.is(msg.$) && user && user.is && pub === user.is.pub){ + SEA.sign(mgs.put, (user._).sea, function(data){ + if(u === data){ return no('User signature fail.') } + console.log("WHAT HAPPENS HERE??", data.m, SEA.opt.unpack(data.m), key, soul); + msg.put[':'] = JSON.stringify({':': SEA.opt.unpack(data.m), '~': data.s}); + //node[key] = JSON.stringify({':': SEA.opt.unpack(data.m), '~': data.s}); + eve.to.next(msg); + }, {check: msg.put, raw: 1}); + return; + } + SEA.verify(msg.put, pub, function(data){ var rel; + console.log("WHAT VERIFIES HERE?", data, SEA.opt.unpack(data, key), key, soul); + data = SEA.opt.unpack(data, key); + if(u === data){ return no("Not owner on '" + key + "'.") } // thanks @rogowski ! + if((rel = link_is(data)) && pub === SEA.opt.pub(rel)){ + (at.sea.own[rel] = at.sea.own[rel] || {})[pub] = 1; + } + eve.to.next(msg); + }); + } + var link_is = Gun.val.link.is; + // okay! The security function handles all the heavy lifting. // It needs to deal read and write of input and output of system data, account/public key data, and regular data. // This is broken down into some pretty clear edge cases, let's go over them: @@ -1174,6 +1271,11 @@ } } if(msg.put){ + /* + NOTICE: THIS IS OLD AND GETTING DEPRECATED. + ANY SECURITY CHANGES SHOULD HAPPEN ABOVE FIRST + THEN PORTED TO HERE. + */ // potentially parallel async operations!!! var check = {}, each = {}, u; each.node = function(node, soul){ @@ -1260,9 +1362,11 @@ return; }*/ check['any'+soul+key] = 1; + console.log(val, key, node, soul, '...', SEA.opt.prep(tmp = SEA.opt.parse(val), key, node, soul)); SEA.sign(SEA.opt.prep(tmp = SEA.opt.parse(val), key, node, soul), (user._).sea, function(data){ if(u === data){ return each.end({err: 'My signature fail.'}) } node[key] = JSON.stringify({':': SEA.opt.unpack(data.m), '~': data.s}); + console.log(key, node[key], '...', data); check['any'+soul+key] = 0; each.end({ok: 1}); }, {check: SEA.opt.pack(tmp, key, node, soul), raw: 1}); diff --git a/test/ptsd/radix.js b/test/ptsd/radix.js index 4e4d5f79..c0514b76 100644 --- a/test/ptsd/radix.js +++ b/test/ptsd/radix.js @@ -22,34 +22,62 @@ //window.radText = Radisk.encode(window.BigText); window.namez = ["Adalard","Adora","Aia","Albertina","Alfie","Allyn","Amabil","Ammamaria","Andy","Anselme","Ardeen","Armand","Ashelman","Aube","Averyl","Baker","Barger","Baten","Bee","Benia","Bernat","Bevers","Bittner","Bobbe","Bonny","Boyce","Breech","Brittaney","Bryn","Burkitt","Cadmann","Campagna","Carlee","Carver","Cavallaro","Chainey","Chaunce","Ching","Cianca","Claudina","Clyve","Colon","Cooke","Corrina","Crawley","Cullie","Dacy","Daniela","Daryn","Deedee","Denie","Devland","Dimitri","Dolphin","Dorinda","Dream","Dunham","Eachelle","Edina","Eisenstark","Elish","Elvis","Eng","Erland","Ethan","Evelyn","Fairman","Faus","Fenner","Fillander","Flip","Foskett","Fredette","Fullerton","Gamali","Gaspar","Gemina","Germana","Gilberto","Giuditta","Goer","Gotcher","Greenstein","Grosvenor","Guthrey","Haldane","Hankins","Harriette","Hayman","Heise","Hepsiba","Hewie","Hiroshi","Holtorf","Howlond","Hurless","Ieso","Ingold","Isidora","Jacoba","Janelle","Jaye","Jennee","Jillana","Johnson","Josy","Justinian","Kannan","Kast","Keeley","Kennett","Kho","Kiran","Knowles","Koser","Kroll","LaMori","Lanctot","Lasky","Laverna","Leff","Leonanie","Lewert","Lilybel","Lissak","Longerich","Lou","Ludeman","Lyman","Madai","Maia","Malvina","Marcy","Maris","Martens","Mathilda","Maye","McLain","Melamie","Meras","Micco","Millburn","Mittel","Montfort","Moth","Mutz","Nananne","Nazler","Nesta","Nicolina","Noellyn","Nuli","Ody","Olympie","Orlena","Other","Pain","Parry","Paynter","Pentheas","Pettifer","Phyllida","Plath","Posehn","Proulx","Quinlan","Raimes","Ras","Redmer","Renelle","Ricard","Rior","Rocky","Ron","Rosetta","Rubia","Ruttger","Salbu","Sandy","Saw","Scholz","Secor","September","Shanleigh","Shenan","Sholes","Sig","Sisely","Soble","Spanos","Stanwinn","Stevie","Stu","Suzanne","Tacy","Tanney","Tekla","Thackeray","Thomasin","Tilla","Tomas","Tracay","Tristis","Ty","Urana","Valdis","Vasta","Vezza","Vitoria","Wait","Warring","Weissmann","Whetstone","Williamson","Wittenburg","Wymore","Yoho","Zamir","Zimmermann"]; - window.radiz = window.radiz || Radix(); + window.radix = window.radix || Radix(); window.arr = []; var i = 1000; while(--i){ arr.push(Math.random()) } window.arrs = arr.slice(0).sort(); + window.ALLZ = window.ALLZ || {}; + window.namez.forEach(function(v,i){ ALLZ[v] = i }); }); /* TEMPORARY COPY OF RADIX UNIT TESTS TO BOOST SPEED */ /* THESE ARE PROBABLY STALE AND NEED TO BE COPIED FROM UNIT TESTS AGAIN */ + /*stool.add('map', function(){ + Gun.obj.map(ALLZ, function(v,i){ + v; + }); + }); + stool.add('for', function(){ + for(var k in ALLZ){ + ALLZ[k]; + } + }); + stool.add('for', function(){ + Object.keys(ALLZ).forEach(function(k){ + ALLZ[k]; + }) + }); + return;*/ stool.add('1', function(){ var rad = Radix(); rad('asdf.pub', 'yum'); rad('ablah', 'cool'); + rad('ab', {yes: 1}); rad('node/circle.bob', 'awesome'); - (rad('asdf.').pub[''] !== 'yum') && bad1; - (rad('nv/foo.bar') !== undefined) && bad2; + (JSON.stringify(rad('asdf.')) !== JSON.stringify({pub: {'': 'yum'}})) && bada; + (rad('nv/foo.bar') != undefined) && badb; + (JSON.stringify(rad('ab')) != JSON.stringify({yes: 1})) && badc + (JSON.stringify(rad()) != JSON.stringify({"a":{"sdf.pub":{"":"yum"},"b":{"lah":{"":"cool"},"":{"yes":1}}},"node/circle.bob":{"":"awesome"}})) && badd; }); stool.add('2', function(){ var all = {}; namez.forEach(function(v,i){ v = v.toLowerCase(); all[v] = v; - radiz(v, i) + ALLZ[v] = i; + radix(v, i) }); (Gun.obj.empty(all) === true) && bad3; - Radix.map(radiz, function(v,k){ + Radix.map(radix, function(v,k){ delete all[k]; }); (Gun.obj.empty(all) !== true) && bad4; }); + stool.add('fast?', function(){ + ALLZ['rubia']; + }); + stool.add('fastest?', function(){ + namez.indexOf('Rubia'); + }); stool.add('3', function(){ var all = {}; namez.forEach(function(v,i){ @@ -58,7 +86,7 @@ //rad(v, i) }); (Gun.obj.empty(all) === true) && bad5; - Radix.map(radiz, function(v,k){ + Radix.map(radix, function(v,k){ delete all[k]; }); (Gun.obj.empty(all) !== true) && bad6; @@ -73,7 +101,7 @@ //rad(v, i) }); (Gun.obj.empty(all) === true) && bad7; - Radix.map(radiz, function(v,k, a,b){ + Radix.map(radix, function(v,k, a,b){ //if(!all[k]){ throw "out of range!" } delete all[k]; }, {start: start, end: end}); @@ -89,12 +117,19 @@ //rad(v, i) }); (Gun.obj.empty(all) === true) && bad9; - Radix.map(radiz, function(v,k, a,b){ + Radix.map(radix, function(v,k, a,b){ //if(!all[k]){ throw "out of range!" } delete all[k]; }, {start: start, end: end}); (Gun.obj.empty(all) !== true) && bad10; }); + stool.add('reverse item', function(){ + Radix.map(radix, function(v,k, a,b){ + (k !== 'ieso') && badri; + (v !== 96) && badri2; + return true; + }, {reverse: 1, end: 'iesogon'}); + }); stool.add('6', function(){ var r = Radix(), tmp; r('alice', 1);r('bob', 2);r('carl', 3);r('carlo',4); diff --git a/test/rad/rad.js b/test/rad/rad.js index 28ccda24..c367def3 100644 --- a/test/rad/rad.js +++ b/test/rad/rad.js @@ -50,10 +50,13 @@ var names = ["Adalard","Adora","Aia","Albertina","Alfie","Allyn","Amabil","Ammam var rad = Radix(); rad('asdf.pub', 'yum'); rad('ablah', 'cool'); + rad('ab', {yes: 1}); rad('node/circle.bob', 'awesome'); - expect(rad('asdf.')).to.be.eql({pub: {'': 'yum'}}); + expect(Gun.obj.copy(rad('asdf.'))).to.be.eql({pub: {'': 'yum'}}); expect(rad('nv/foo.bar')).to.be(undefined); + expect(rad('ab')).to.eql({yes: 1}); + expect(Gun.obj.copy(rad())).to.be.eql({"a":{"sdf.pub":{"":"yum"},"b":{"lah":{"":"cool"},"":{"yes":1}}},"node/circle.bob":{"":"awesome"}}); }); it('radix write read', function(done){ @@ -121,6 +124,16 @@ var names = ["Adalard","Adora","Aia","Albertina","Alfie","Allyn","Amabil","Ammam expect(Gun.obj.empty(all)).to.be.ok(); done(); }); + + it('radix reverse item', function(done){ + var opt = {reverse: 1, end: 'iesogon'}; + Radix.map(radix, function(v,k, a,b){ + expect(k).to.be('ieso'); + expect(v).to.be(96); + return true; + }, opt); + done(); + }); it('radix reverse', function(done){ var r = Radix(), tmp;