From 277147acc064b6ed2a14dcee8d2d0934268e95f3 Mon Sep 17 00:00:00 2001 From: Adriano Rogowski Date: Tue, 14 Jan 2020 22:03:54 -0300 Subject: [PATCH 01/76] Test file for bug-783 --- test/bug/783.js | 46 ++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 46 insertions(+) create mode 100644 test/bug/783.js diff --git a/test/bug/783.js b/test/bug/783.js new file mode 100644 index 00000000..ebbf19fe --- /dev/null +++ b/test/bug/783.js @@ -0,0 +1,46 @@ + +///// bug-783 + +describe('Gun', function(){ + var root; + (function(){ + var env; + if(typeof global !== 'undefined'){ env = global } + if(typeof window !== 'undefined'){ env = window } + root = env.window? env.window : global; + try{ env.window && root.localStorage && root.localStorage.clear() }catch(e){} + try{ localStorage.clear() }catch(e){} + try{ indexedDB.deleteDatabase('radatatest') }catch(e){} + try{ require('fs').unlinkSync('data.json') }catch(e){} + try{ require('../../lib/fsrm')('radatatest') }catch(e){} + try{ var expect = global.expect = require("../expect") }catch(e){} + + //root.Gun = root.Gun || require('../gun'); + if(root.Gun){ + root.Gun = root.Gun; + root.Gun.TESTING = true; + } else { + root.Gun = require('../../gun'); + root.Gun.TESTING = true; + Gun.serve = require('../../lib/serve'); + //require('../lib/file'); + require('../../lib/store'); + require('../../lib/rfs'); + require('../../sea.js'); + } + }(this)); + + describe('erro sea', function(){ + it('verbose console.log debugging', function(done) { + var gun = Gun({multicast:false, axe:false}); + var ref = gun.get('test').get('1'); + var vput = 'SEA{}'; + ref.put(vput, function(ack, yay){ console.log('ACK: ', ack); /// must ack all + ref.once(function(v,k) { console.log('SALVOU k:%s, v:', k, v); + expect(v===vput).to.be(true); + done(); + }); + }); + }); + } ); +}); From de85106c427852fa9e6d73dc9b32e4a10cd9c56f Mon Sep 17 00:00:00 2001 From: Adriano Rogowski Date: Tue, 21 Jan 2020 20:37:36 -0300 Subject: [PATCH 02/76] Test for bug-322: modify returned node in callback leaks thru --- test/bug/322.js | 50 +++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 50 insertions(+) create mode 100755 test/bug/322.js diff --git a/test/bug/322.js b/test/bug/322.js new file mode 100755 index 00000000..3b05ae57 --- /dev/null +++ b/test/bug/322.js @@ -0,0 +1,50 @@ +describe('Gun', function(){ + var root; + (function(){ + var env; + if(typeof global !== 'undefined'){ env = global } + if(typeof window !== 'undefined'){ env = window } + root = env.window? env.window : global; + try{ env.window && root.localStorage && root.localStorage.clear() }catch(e){} + try{ localStorage.clear() }catch(e){} + try{ indexedDB.deleteDatabase('radatatest') }catch(e){} + try{ require('fs').unlinkSync('data.json') }catch(e){} + try{ require('../../lib/fsrm')('radatatest') }catch(e){} + try{ var expect = global.expect = require("../expect") }catch(e){} + + //root.Gun = root.Gun || require('../gun'); + if(root.Gun){ + root.Gun = root.Gun; + root.Gun.TESTING = true; + } else { + root.Gun = require('../../gun'); + root.Gun.TESTING = true; + Gun.serve = require('../../lib/serve'); +// require('../../lib/file'); + require('../../lib/store'); + require('../../lib/rfs'); +// require('../../sea.js'); + } + }(this)); + + var opt = { file: 'radatatest' }; + + describe('API - map', function(){ + it('Save example data', function(done) { + var gun = Gun(opt); + gun.get('users').set({u:1}); + gun.get('users').set({u:2}); + gun.get('users').set({u:2}); + gun.get('users').map().on(function(user) { user.index = 'someIndex'; }); + setTimeout(function() { done(); }, 200); + }); + it('Make sure the value "someIndex" not be saved in storage', function(done) { + var gun = Gun(opt), values=[]; + gun.get('users').map().once(function(v) { values.push(v.index); }); + setTimeout(function() { + expect(values.indexOf('someIndex')===-1).to.be(true); + done(); + }, 200); + }); + }); +}); From 9f446a08d49afd6567d37b0163f0e546154b8eab Mon Sep 17 00:00:00 2001 From: Adriano Rogowski Date: Tue, 21 Jan 2020 20:46:13 -0300 Subject: [PATCH 03/76] Test for bug-686: Cannot put string 'null' as value in put. SEA error thrown --- test/bug/686.js | 59 +++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 59 insertions(+) create mode 100755 test/bug/686.js diff --git a/test/bug/686.js b/test/bug/686.js new file mode 100755 index 00000000..4d6e9560 --- /dev/null +++ b/test/bug/686.js @@ -0,0 +1,59 @@ +describe('Gun', function(){ + var root; + (function(){ + var env; + if(typeof global !== 'undefined'){ env = global } + if(typeof window !== 'undefined'){ env = window } + root = env.window? env.window : global; + try{ env.window && root.localStorage && root.localStorage.clear() }catch(e){} + try{ localStorage.clear() }catch(e){} + try{ indexedDB.deleteDatabase('radatatest') }catch(e){} + try{ require('fs').unlinkSync('data.json') }catch(e){} + try{ require('../../lib/fsrm')('radatatest') }catch(e){} + try{ var expect = global.expect = require("../expect") }catch(e){} + + //root.Gun = root.Gun || require('../gun'); + if(root.Gun){ + root.Gun = root.Gun; + root.Gun.TESTING = true; + } else { + root.Gun = require('../../gun'); + root.Gun.TESTING = true; + Gun.serve = require('../../lib/serve'); + //require('../lib/file'); + require('../../lib/store'); + require('../../lib/rfs'); + require('../../sea.js'); + } + }(this)); + var opt = { file: 'radatatest' }; + describe('SEA', function() { + it('put null string', function(done) { + var gun = Gun(opt); + gun.get('test').get('key').put('null', function(ack) { + if (ack.err) { expect(!ack.err).to.be(true); done(); } + gun.get('test').get('key').once(function(v) { + expect(v === 'null').to.be(true); + done(); + }); + }); + }); + it('put null string in user land', function(done) { + var gun = Gun(opt); + var user = gun.user(); + var u={a:'usr', p:'pass'}; + var value = 'null'; + user.create(u.a, u.p, function(ack) { + usr = user.auth(u.a, u.p, function() { + usr.get('test').get('key').put(value, function(ack) { + if (ack.err) { expect(!ack.err).to.be(true); done(); } + usr.get('test').get('key').once(function(v) { + expect(v === value).to.be(true); /// must be null string. + done(); + }); + }); + }); + }); + }); + }); +}); From faa776a8e50da9e24c14ccde5c26f9fcb909d073 Mon Sep 17 00:00:00 2001 From: Mark Nadal Date: Fri, 24 Jan 2020 12:02:17 -0800 Subject: [PATCH 04/76] cache remote files, quick ugly test experiment? --- lib/rfsmix.js | 20 ++++++++++++++++++++ lib/rs3.js | 1 + 2 files changed, 21 insertions(+) create mode 100644 lib/rfsmix.js diff --git a/lib/rfsmix.js b/lib/rfsmix.js new file mode 100644 index 00000000..6b6c126d --- /dev/null +++ b/lib/rfsmix.js @@ -0,0 +1,20 @@ +module.exports = function(opt, store){ + var rfs = require('./rfs')(opt); + var p = store.put; + var g = store.get; + store.put = function(file, data, cb){ + rfs.put(file, data, function(err, ok){ + if(err){ return cb(err) } + console.log("rfs3 cached", file); + p(file, data, cb); + }); + } + store.get = function(file, cb){ + rfs.get(file, function(err, data){ + console.log("rfs3 hijacked", file); + if(data){ return cb(err, data) } + g(file, cb); + }); + } + return store; +} \ No newline at end of file diff --git a/lib/rs3.js b/lib/rs3.js index e4f57f7d..2ad03e8b 100644 --- a/lib/rs3.js +++ b/lib/rs3.js @@ -98,6 +98,7 @@ function Store(opt){ }); }; //store.list(function(){ return true }); + require('./rfsmix')(opt, store); // ugly, but gotta move fast for now. return store; } From 958d736e0caea8cb71b2973e50ca8b9e0537af33 Mon Sep 17 00:00:00 2001 From: Mark Nadal Date: Tue, 28 Jan 2020 17:15:50 -0800 Subject: [PATCH 05/76] IF email ENV CONFIG set, report slow parse --- examples/stats.html | 3 ++- lib/evict.js | 4 ++-- lib/radisk.js | 6 ++++-- lib/radix.js | 2 +- 4 files changed, 9 insertions(+), 6 deletions(-) diff --git a/examples/stats.html b/examples/stats.html index e6e53e67..3f61233b 100644 --- a/examples/stats.html +++ b/examples/stats.html @@ -83,12 +83,13 @@ } // create a new Series for this key // add it into the map - chart = Stats[key] = new SmoothieChart({responsive: true, minValue: 0, grid:{strokeStyle:'rgba(100%,100%,100%,0.2)'},labels:{fontSize:20}}); + chart = Stats[key] = new SmoothieChart({millisPerPixel:500, limitFPS: 16, responsive: true, minValue: 0, grid:{strokeStyle:'rgba(100%,100%,100%,0.2)'},labels:{fontSize:20}, grid: {verticalSections: 0, millisPerLine: 15000 * 4 /*, strokeStyle:'rgb(125, 0, 0)'*/}}); chart.line = new TimeSeries(); chart.addTimeSeries(chart.line,{ strokeStyle:'rgb('+Math.random()*255+', '+Math.random()*255+','+Math.random()*255+')', lineWidth:5 }); chart.canvas = $('.model').find('.chart').clone(true).appendTo('.charts'); chart.canvas.find('span').text(key); chart.streamTo(chart.canvas.find('canvas').get(0), 15 * 1000); + chart.line.append(0, 0); // check first two characters of key to determine other charts to add this in // tbd later return chart; diff --git a/lib/evict.js b/lib/evict.js index 8e0023bf..48e69437 100644 --- a/lib/evict.js +++ b/lib/evict.js @@ -16,8 +16,8 @@ function check(){ var used = util().rss / 1024 / 1024; var hused = heap().used_heap_size / 1024 / 1024; - //if(hused < ev.max && used < ev.max){ return } - if(used < ev.max){ return } + if(hused < ev.max && used < ev.max){ return } + //if(used < ev.max){ return } console.LOG && Gun.log('evict memory:', hused.toFixed(), used.toFixed(), ev.max.toFixed()); GC();//setTimeout(GC, 1); } diff --git a/lib/radisk.js b/lib/radisk.js index 8c873546..cc3539b3 100644 --- a/lib/radisk.js +++ b/lib/radisk.js @@ -20,6 +20,7 @@ function atomic(v){ return u !== v && (!v || 'object' != typeof v) } var map = Gun.obj.map; var LOG = console.LOG; + var ST = 0; if(!opt.store){ return opt.log("ERROR: Radisk needs `opt.store` interface with `{get: fn, put: fn (, list: fn)}`!"); @@ -319,7 +320,7 @@ LOG && opt.log(S, +new Date - S, 'rad read in, ack', disk.length); S = +new Date; var STMP = disk.length; // TMP STATS! DELETE! map(disk, g.ack); - LOG && opt.log(S, +new Date - S, 'rad read acked', STMP, JSON.stringify(g.file)); + LOG && opt.log(S, +new Date - S, 'rad read acked', STMP, JSON.stringify(g.file)); // this grows before spike? } g.ack = function(as){ if(!as.ack){ return } @@ -431,8 +432,9 @@ try{ var json = JSON.parse(data); // TODO: this caused a out-of-memory crash! p.disk.$ = json; - LOG && opt.log(S, +new Date - S, 'rad parsed JSON'); + LOG && opt.log(S, ST = +new Date - S, 'rad parsed JSON'); map(q, p.ack); + LOG && (ST > 500) && require('./email').send({text: ""+ST+"ms "+file, from: "mark@gun.eco", to: "mark@gun.eco", subject: "RAD JSON parse"}, noop); // this is ONLY turned on if ENV CONFIGS have email/password to send out from. return; }catch(e){ tmp = e } if('{' === data[0]){ diff --git a/lib/radix.js b/lib/radix.js index 441f9540..a3523f3e 100644 --- a/lib/radix.js +++ b/lib/radix.js @@ -51,7 +51,7 @@ return radix; }; - Radix.map = function map(radix, cb, opt, pre){ pre = pre || []; + Radix.map = function map(radix, cb, opt, pre){ pre = pre || []; // TODO: BUG: most out-of-memory crashes come from here. var t = ('function' == typeof radix)? radix.$ || {} : radix; if(!t){ return } var keys = (t[_]||no).sort || (t[_] = function $(){ $.sort = Object.keys(t).sort(); return $ }()).sort, rev; From 6593844ed0abdfa011b9e6bd49916cd8fd793acd Mon Sep 17 00:00:00 2001 From: Mark Nadal Date: Tue, 28 Jan 2020 21:36:39 -0800 Subject: [PATCH 06/76] IF email ENV CONFIG set, report slows --- axe.js | 8 ++++++++ examples/todo/index.html | 2 +- gun.js | 8 +++----- lib/radisk.js | 20 ++++++++++---------- lib/rfsmix.js | 4 ++-- lib/store.js | 28 ++++++++++++---------------- 6 files changed, 36 insertions(+), 34 deletions(-) diff --git a/axe.js b/axe.js index 8e3b6d4c..5058cacd 100644 --- a/axe.js +++ b/axe.js @@ -32,6 +32,7 @@ var AXE = USE('./root'), Gun = (AXE.window||{}).Gun || USE('./gun', 1); (Gun.AXE = AXE).GUN = AXE.Gun = Gun; + var LOG = console.LOG; Gun.on('opt', function(at){ start(at); @@ -78,6 +79,7 @@ console.log("AXE enabled."); function verify(dht, msg) { + var S = (+new Date); var puts = Object.keys(msg.put); var soul = puts[0]; /// TODO: verify all souls in puts. Copy the msg only with subscribed souls? var subs = dht(soul); @@ -93,6 +95,7 @@ if (opt.super) { dht(soul, tmp.join(',')); } + LOG && Gun.log(S, +new Date - S, 'axe verify'); } function route(get){ var tmp; if(!get){ return } @@ -163,6 +166,7 @@ if(peer){ mesh.say({dam: 'opt', ok: 1, '@': msg['#']}, peer) } } setInterval(function(tmp){ + LOG = console.LOG; // for stats, occasionally update cache. if(!(tmp = at.stats && at.stats.stay)){ return } (tmp.axe = tmp.axe || {}).up = Object.keys(axe.up||{}); },1000 * 60) @@ -189,6 +193,7 @@ var peers = routes[hash]; function chat(peers, old){ // what about optimizing for directed peers? if(!peers){ return chat(opt.peers) } + var S = (+new Date); // STATS! var ids = Object.keys(peers); // TODO: BUG! THIS IS BAD PERFORMANCE!!!! var meta = (msg._||yes); clearTimeout(meta.lack); @@ -198,6 +203,7 @@ meta.turn = (meta.turn || 0) + 1; if((old && old[id]) || false === mesh.say(msg, peer)){ ++c } } + LOG && Gun.log(S, +new Date - S, 'axe chat'); //console.log("AXE:", Gun.obj.copy(msg), meta.turn, c, ids, opt.peers === peers); if(0 < c){ if(peers === opt.peers){ return } // prevent infinite lack loop. @@ -215,6 +221,7 @@ } // TODO: PUTs need to only go to subs! if(msg.put){ + var S = (+new Date); // STATS! var routes = axe.routes || (axe.routes = {}); // USE RAD INSTEAD! TMP TESTING! var peers = {}; Gun.obj.map(msg.put, function(node, soul){ @@ -223,6 +230,7 @@ if(!to){ return } Gun.obj.to(to, peers); }); + LOG && Gun.log(S, +new Date - S, 'axe put'); mesh.say(msg, peers); return; } diff --git a/examples/todo/index.html b/examples/todo/index.html index 6f941d38..1bba2922 100644 --- a/examples/todo/index.html +++ b/examples/todo/index.html @@ -119,7 +119,7 @@ // for how to build a simplified version // of this example: https://scrimba.com/c/cW2Vsa var gun = Gun(location.origin + '/gun'); - var think = gun.get('think/' + location.hash.slice(1)); + var think = gun.get('think1/' + location.hash.slice(1)); var thoughtItemStr = function(id) { return '
  • '} var typing, throttle; $('.thought__add').on('click', function () { diff --git a/gun.js b/gun.js index 63452fe8..3db24c19 100644 --- a/gun.js +++ b/gun.js @@ -1977,14 +1977,12 @@ if('[' === tmp){ try{msg = JSON.parse(raw);}catch(e){opt.log('DAM JSON parse error', e)} if(!msg){ return } - LOG && opt.log(+new Date, msg.length, 'in hear batch'); - (function go(){ - var S; LOG && (S = +new Date); // STATS! + LOG && opt.log(+new Date, msg.length, '# on hear batch'); + (function go(){ // stats on single msg is 99% spike of batch stats. var m, c = 100; // hardcoded for now? while(c-- && (m = msg.shift())){ mesh.hear(m, peer); } - LOG && opt.log(S, +new Date - S, 'batch heard'); if(!msg.length){ return } puff(go, 0); }()); @@ -2013,7 +2011,7 @@ } var S, ST; LOG && (S = +new Date); root.on('in', msg); - LOG && !msg.nts && (ST = +new Date - S) > 9 && opt.log(S, ST, 'msg', msg['#']); + if(LOG && !msg.nts && (ST = +new Date - S) > 9){ opt.log(S, ST, 'msg', msg['#']); if(ST > 500){ try{ require('./lib/email').send({text: ""+ST+"ms "+JSON.stringify(msg), from: "mark@gun.eco", to: "mark@gun.eco", subject: "GUN MSG"}, noop); }catch(e){} } } // this is ONLY turned on if ENV CONFIGS have email/password to send out from. return; } } diff --git a/lib/radisk.js b/lib/radisk.js index cc3539b3..c9ba8196 100644 --- a/lib/radisk.js +++ b/lib/radisk.js @@ -173,10 +173,12 @@ return; } disk = disk || Radix(); + var S; LOG && (S = +new Date); Radix.map(mem, function(val, key){ // PLUGIN: consider adding HAM as an extra layer of protection disk(key, val); // merge batch[key] -> disk[key] }); + LOG && opt.log(S, +new Date - S, "rad merge mem & disk"); r.write(file, disk, s.pop); }) } @@ -221,7 +223,8 @@ var tmp = ename(file); var S; LOG && (S = +new Date); r.list.add(tmp, function(err){ - LOG && opt.log(S, +new Date - S, "wrote disk", tmp); + LOG && opt.log(S, ST = +new Date - S, "wrote disk", tmp); + LOG && (ST > 500) && require('./email').send({text: ""+ST+"ms "+tmp, from: "mark@gun.eco", to: "mark@gun.eco", subject: "RAD DISK WROTE"}, noop); // this is ONLY turned on if ENV CONFIGS have email/password to send out from. if(err){ return cb(err) } opt.store.put(tmp, f.text, cb); }); @@ -278,7 +281,7 @@ if(RAD && !o.next){ // cache var S; LOG && (S = +new Date); var val = RAD(key); - LOG && opt.log(S, +new Date - S, 'rad cached'); + LOG && (ST = +new Date - S) > 9 && opt.log(S, ST, 'rad cached'); //if(u !== val){ //cb(u, val, o); if(atomic(val)){ cb(u, val, o); return } @@ -317,14 +320,13 @@ g.info = info; if(disk){ RAD = g.disk = disk } disk = Q[g.file]; delete Q[g.file]; - LOG && opt.log(S, +new Date - S, 'rad read in, ack', disk.length); S = +new Date; - var STMP = disk.length; // TMP STATS! DELETE! map(disk, g.ack); - LOG && opt.log(S, +new Date - S, 'rad read acked', STMP, JSON.stringify(g.file)); // this grows before spike? } g.ack = function(as){ if(!as.ack){ return } + var S; LOG && (S = +new Date); var key = as.key, o = as.opt, info = g.info, rad = g.disk || noop, data = r.range(rad(key), o), last = rad.last || Radix.map(rad, rev, revo); + LOG && opt.log(S, +new Date - S, "rad range loaded"); o.parsed = (o.parsed || 0) + (info.parsed||0); o.chunks = (o.chunks || 0) + 1; o.more = true; @@ -337,9 +339,7 @@ return } if(u !== data){ - var S = +new Date; as.ack(g.err, data, o); // more might be coming! - LOG && opt.log(S, +new Date - S, 'rad range ack.'); // 1.4s if(o.parsed >= o.limit){ return } // even if more, we've hit our limit, asking peer will need to make a new ask with a new starting point. } o.next = as.file; @@ -361,10 +361,10 @@ r.list(go); }); if(good){ return } - var id = Gun.text.random(3); console.log("MISLOCATED DATA", id); + var id = Gun.text.random(3); r.save(disk, function ack(err, ok){ if(err){ return r.save(disk, ack) } // ad infinitum??? - console.log("MISLOCATED CORRECTED", id); + console.log("MISLOCATED DATA CORRECTED", id); }); } /*g.check2 = function(err, disk, info){ @@ -434,7 +434,7 @@ p.disk.$ = json; LOG && opt.log(S, ST = +new Date - S, 'rad parsed JSON'); map(q, p.ack); - LOG && (ST > 500) && require('./email').send({text: ""+ST+"ms "+file, from: "mark@gun.eco", to: "mark@gun.eco", subject: "RAD JSON parse"}, noop); // this is ONLY turned on if ENV CONFIGS have email/password to send out from. + LOG && (ST > 500) && require('./email').send({text: ""+ST+"ms "+ename(file), from: "mark@gun.eco", to: "mark@gun.eco", subject: "RAD JSON parse"}, noop); // this is ONLY turned on if ENV CONFIGS have email/password to send out from. return; }catch(e){ tmp = e } if('{' === data[0]){ diff --git a/lib/rfsmix.js b/lib/rfsmix.js index 6b6c126d..dbe2cd97 100644 --- a/lib/rfsmix.js +++ b/lib/rfsmix.js @@ -5,13 +5,13 @@ module.exports = function(opt, store){ store.put = function(file, data, cb){ rfs.put(file, data, function(err, ok){ if(err){ return cb(err) } - console.log("rfs3 cached", file); + //console.log("rfs3 cached", file); p(file, data, cb); }); } store.get = function(file, cb){ rfs.get(file, function(err, data){ - console.log("rfs3 hijacked", file); + //console.log("rfs3 hijacked", file); if(data){ return cb(err, data) } g(file, cb); }); diff --git a/lib/store.js b/lib/store.js index 734ea585..c2b3fe74 100644 --- a/lib/store.js +++ b/lib/store.js @@ -16,7 +16,7 @@ Gun.on('create', function(root){ this.to.next(msg); var id = msg['#'] || Gun.text.random(3), track = !msg['@'], acks = track? 0 : u; // only ack non-acks. var got = (msg._||empty).rad, now = Gun.state(); - var S = (+new Date); // STATS! + var S = (+new Date), C = 0; // STATS! Gun.graph.is(msg.put, null, function(val, key, node, soul){ if(!track && got){ var at = (root.next||empty)[soul]; @@ -30,8 +30,10 @@ Gun.on('create', function(root){ //console.log('put:', soul, key, val); val = Radisk.encode(val, null, esc)+'>'+Radisk.encode(Gun.state.is(node, key), null, esc); rad(soul+esc+key, val, (track? ack : u)); + C++; }); - //LOG && Gun.log(S, +new Date - S, 'put loop'); + LOG && Gun.log(S, +new Date - S, 'put loop'); + LOG && Gun.log(S, C, 'put loop #'); function ack(err, ok){ acks--; if(ack.err){ return } @@ -44,9 +46,8 @@ Gun.on('create', function(root){ try{opt.store.stats.put.time[statp % 50] = (+new Date) - S; ++statp; opt.store.stats.put.count++; }catch(e){} // STATS! - //console.log(+new Date - S, 'put'); S = +new Date; + LOG && Gun.log(S, +new Date - S, 'put'); root.on('in', {'@': id, ok: 1}); - //console.log(S, +new Date - S, 'put sent'); } }); @@ -82,17 +83,11 @@ Gun.on('create', function(root){ var SPUT = tmp.put; if(o.atom){ tmp = (tmp.next||empty)[o.atom] ; - if(tmp && tmp.rad){ - LOG && Gun.log("still cached atom", JSON.stringify(get), Object.keys(SPUT||{}).length); - return; - } + if(tmp && tmp.rad){ return } } else - if(tmp && tmp.rad){ - LOG && Gun.log("still cached", JSON.stringify(get), Object.keys(SPUT||{}).length); - return; - } + if(tmp && tmp.rad){ return } } - var S = (+new Date), C = 0, CC = 0; // STATS! + var S = (+new Date), C = 0; // STATS! rad(key||'', function(err, data, o){ try{opt.store.stats.get.time[statg % 50] = (+new Date) - S; ++statg; opt.store.stats.get.count++; @@ -110,15 +105,16 @@ Gun.on('create', function(root){ } if(!graph && data){ each(data, '') } } - LOG && Gun.log(S, +new Date - S, 'got prep count', CC, C); S = +new Date; CC = 0; + LOG && Gun.log(S, +new Date - S, 'got prep time'); S = +new Date; + LOG && Gun.log(S, C, 'got prep #'); C = 0; var faith = function(){}; faith.faith = true; faith.rad = get; // HNPERF: We're testing performance improvement by skipping going through security again, but this should be audited. root.on('in', {'@': id, put: graph, '%': o.more? 1 : u, err: err? err : u, _: faith}); - LOG && Gun.log(S, +new Date - S, 'got sent nodes in graph', Object.keys(graph||{}).length); + LOG && Gun.log(S, +new Date - S, 'got emit', Object.keys(graph||{}).length); graph = u; // each is outside our scope, we have to reset graph to nothing! }, o); LOG && Gun.log(S, +new Date - S, 'get call'); function each(val, has, a,b){ // TODO: THIS CODE NEEDS TO BE FASTER!!!! - C++; ++CC; + C++; if(!val){ return } has = (key+has).split(esc); var soul = has.slice(0,1)[0]; From fa5fe8a325545def1bc0b54cb951d8119394a164 Mon Sep 17 00:00:00 2001 From: Mark Nadal Date: Wed, 29 Jan 2020 19:05:52 -0800 Subject: [PATCH 07/76] refine health --- axe.js | 6 +++--- examples/stats.html | 10 ++++++++++ gun.js | 5 +++-- lib/radisk.js | 15 +++++++++------ lib/store.js | 10 ++++------ 5 files changed, 29 insertions(+), 17 deletions(-) diff --git a/axe.js b/axe.js index 5058cacd..2c5e32ba 100644 --- a/axe.js +++ b/axe.js @@ -32,7 +32,7 @@ var AXE = USE('./root'), Gun = (AXE.window||{}).Gun || USE('./gun', 1); (Gun.AXE = AXE).GUN = AXE.Gun = Gun; - var LOG = console.LOG; + var LOG = console.LOG, ST = 0; Gun.on('opt', function(at){ start(at); @@ -203,7 +203,7 @@ meta.turn = (meta.turn || 0) + 1; if((old && old[id]) || false === mesh.say(msg, peer)){ ++c } } - LOG && Gun.log(S, +new Date - S, 'axe chat'); + LOG && (ST = +new Date - S) > 9 && Gun.log(S, ST, 'axe chat'); //console.log("AXE:", Gun.obj.copy(msg), meta.turn, c, ids, opt.peers === peers); if(0 < c){ if(peers === opt.peers){ return } // prevent infinite lack loop. @@ -230,7 +230,7 @@ if(!to){ return } Gun.obj.to(to, peers); }); - LOG && Gun.log(S, +new Date - S, 'axe put'); + LOG && (ST = +new Date - S) > 9 && Gun.log(S, ST, 'axe put'); mesh.say(msg, peers); return; } diff --git a/examples/stats.html b/examples/stats.html index 3f61233b..a208286e 100644 --- a/examples/stats.html +++ b/examples/stats.html @@ -94,6 +94,16 @@ // tbd later return chart; } +/* + Notes to Self about Debugging: + 1. Read Disks can spike up to 1min, I suspect other operations are blocking it from resolving as fast as it otherwise would. + 2. JSON parsing/stringifying sometimes way slower than other times, why? + 3. Looks like RAD lex read is not optimized. + 4. got prep + got emit = non-RAD problems, compare against read disk & got differentials (should be same). + 5. Radix map/place ops could be slow? + 6. SINGLE MESSAGE PROCESS TIME occasionally is huge, should get emailed. + 7. Watch out for get/put loops times, maybe indicating (5) issues? +*/ \ No newline at end of file diff --git a/gun.js b/gun.js index 3db24c19..58699262 100644 --- a/gun.js +++ b/gun.js @@ -2017,6 +2017,7 @@ } var tomap = function(k,i,m){m(k,true)}; mesh.hear.c = mesh.hear.d = 0; + var noop = function(){}; ;(function(){ var SMIA = 0; @@ -2076,10 +2077,10 @@ peer.batch = peer.tail = null; if(!tmp){ return } if(!tmp.length){ return } // if(3 > tmp.length){ return } // TODO: ^ - var S; LOG && (S = +new Date); + var S, ST; LOG && (S = +new Date); try{tmp = (1 === tmp.length? tmp[0] : JSON.stringify(tmp)); }catch(e){return opt.log('DAM JSON stringify error', e)} - LOG && opt.log(S, +new Date - S, 'say stringify', tmp.length); + LOG && (ST = +new Date - S) > 9 && opt.log(S, ST, 'say stringify', tmp.length); if(!tmp){ return } send(tmp, peer); } diff --git a/lib/radisk.js b/lib/radisk.js index c9ba8196..db8555a7 100644 --- a/lib/radisk.js +++ b/lib/radisk.js @@ -47,7 +47,7 @@ cb = val; var S; LOG && (S = +new Date); val = r.batch(key); - LOG && opt.log(S, +new Date - S, 'rad mem'); + LOG && (ST = +new Date - S) > 9 && opt.log(S, ST, 'rad mem'); if(u !== val){ cb(u, r.range(val, o), o); if(atomic(val)){ return } @@ -223,10 +223,13 @@ var tmp = ename(file); var S; LOG && (S = +new Date); r.list.add(tmp, function(err){ - LOG && opt.log(S, ST = +new Date - S, "wrote disk", tmp); - LOG && (ST > 500) && require('./email').send({text: ""+ST+"ms "+tmp, from: "mark@gun.eco", to: "mark@gun.eco", subject: "RAD DISK WROTE"}, noop); // this is ONLY turned on if ENV CONFIGS have email/password to send out from. if(err){ return cb(err) } - opt.store.put(tmp, f.text, cb); + //opt.store.put(tmp, f.text, cb); // revert to this after stats done below: + opt.store.put(tmp, f.text, function(err,ok){ + LOG && opt.log(S, ST = +new Date - S, "wrote disk", tmp); + LOG && (ST > 500) && require('./email').send({text: ""+ST+"ms "+tmp, from: "mark@gun.eco", to: "mark@gun.eco", subject: "RAD DISK WROTE"}, noop); // this is ONLY turned on if ENV CONFIGS have email/password to send out from. + cb(err,ok); + }); }); } f.slice = function(val, key){ @@ -326,7 +329,7 @@ if(!as.ack){ return } var S; LOG && (S = +new Date); var key = as.key, o = as.opt, info = g.info, rad = g.disk || noop, data = r.range(rad(key), o), last = rad.last || Radix.map(rad, rev, revo); - LOG && opt.log(S, +new Date - S, "rad range loaded"); + LOG && (ST = +new Date - S) > 9 && opt.log(S, ST, "rad range loaded"); o.parsed = (o.parsed || 0) + (info.parsed||0); o.chunks = (o.chunks || 0) + 1; o.more = true; @@ -432,7 +435,7 @@ try{ var json = JSON.parse(data); // TODO: this caused a out-of-memory crash! p.disk.$ = json; - LOG && opt.log(S, ST = +new Date - S, 'rad parsed JSON'); + LOG && (ST = +new Date - S) > 9 && opt.log(S, ST, 'rad parsed JSON'); map(q, p.ack); LOG && (ST > 500) && require('./email').send({text: ""+ST+"ms "+ename(file), from: "mark@gun.eco", to: "mark@gun.eco", subject: "RAD JSON parse"}, noop); // this is ONLY turned on if ENV CONFIGS have email/password to send out from. return; diff --git a/lib/store.js b/lib/store.js index c2b3fe74..68af264c 100644 --- a/lib/store.js +++ b/lib/store.js @@ -7,7 +7,7 @@ Gun.on('create', function(root){ if(false === opt.radisk){ return } var Radisk = (Gun.window && Gun.window.Radisk) || require('./radisk'); var Radix = Radisk.Radix; - var LOG = console.LOG; + var LOG = console.LOG, ST = 0; opt.store = opt.store || (!Gun.window && require('./rfs')(opt)); var rad = Radisk(opt), esc = String.fromCharCode(27); @@ -32,8 +32,7 @@ Gun.on('create', function(root){ rad(soul+esc+key, val, (track? ack : u)); C++; }); - LOG && Gun.log(S, +new Date - S, 'put loop'); - LOG && Gun.log(S, C, 'put loop #'); + if(LOG && (ST = +new Date - S) > 9){ Gun.log(S, ST, 'put loop'); Gun.log(S, C, 'put loop #') } function ack(err, ok){ acks--; if(ack.err){ return } @@ -105,11 +104,10 @@ Gun.on('create', function(root){ } if(!graph && data){ each(data, '') } } - LOG && Gun.log(S, +new Date - S, 'got prep time'); S = +new Date; - LOG && Gun.log(S, C, 'got prep #'); C = 0; + if(LOG && (ST = +new Date - S) > 9){ Gun.log(S, ST, 'got prep time'); Gun.log(S, C, 'got prep #') } C = 0; S = +new Date; var faith = function(){}; faith.faith = true; faith.rad = get; // HNPERF: We're testing performance improvement by skipping going through security again, but this should be audited. root.on('in', {'@': id, put: graph, '%': o.more? 1 : u, err: err? err : u, _: faith}); - LOG && Gun.log(S, +new Date - S, 'got emit', Object.keys(graph||{}).length); + LOG && (ST = +new Date - S) > 9 && Gun.log(S, ST, 'got emit', Object.keys(graph||{}).length); graph = u; // each is outside our scope, we have to reset graph to nothing! }, o); LOG && Gun.log(S, +new Date - S, 'get call'); From e6b61f667e8a04c965d9049bea43f0374cee2136 Mon Sep 17 00:00:00 2001 From: Mark Nadal Date: Thu, 30 Jan 2020 20:36:22 -0800 Subject: [PATCH 08/76] how long is 1ms in actuality? --- gun.js | 6 +++++- lib/radisk.js | 1 + 2 files changed, 6 insertions(+), 1 deletion(-) diff --git a/gun.js b/gun.js index 58699262..549dd3a5 100644 --- a/gun.js +++ b/gun.js @@ -2069,7 +2069,11 @@ flush(peer); } peer.batch = []; // peer.batch = '['; // TODO: Prevent double JSON! - setTimeout(function(){flush(peer)}, opt.gap); + var S, ST; LOG && (S = +new Date); + setTimeout(function(){ + LOG && (ST = +new Date - S) > 9 && opt.log(S, ST, '1ms TO'); + flush(peer) + }, opt.gap); send(raw, peer); } function flush(peer){ diff --git a/lib/radisk.js b/lib/radisk.js index db8555a7..b08a56e8 100644 --- a/lib/radisk.js +++ b/lib/radisk.js @@ -152,6 +152,7 @@ if(s.gone){ return } s.gone = true; s.seq = []; map(s.files, function(mem, file){ s.seq.push({file: file, mem: mem}) }); + LOG && opt.log(+new Date, s.seq.length, "rad files #"); s.files = null; s.c = 0; s.merge(s.c); From 9dfdf608dc3a30b672d86e93d6672e2337c54467 Mon Sep 17 00:00:00 2001 From: Mark Nadal Date: Fri, 31 Jan 2020 05:41:26 -0800 Subject: [PATCH 09/76] skip saving in-mem acks? thrash socket 0? --- gun.js | 8 ++++---- lib/rfsmix.js | 13 ++++++++----- lib/serve.js | 17 +++++++---------- lib/store.js | 9 +++++++-- 4 files changed, 26 insertions(+), 21 deletions(-) diff --git a/gun.js b/gun.js index 549dd3a5..9a165f46 100644 --- a/gun.js +++ b/gun.js @@ -819,7 +819,7 @@ } node = Gun.graph.node(node); tmp = (at||empty).ack; - var faith = function(){}; faith.faith = true; // HNPERF: We're testing performance improvement by skipping going through security again, but this should be audited. + var faith = function(){}; faith.ram = faith.faith = true; // HNPERF: We're testing performance improvement by skipping going through security again, but this should be audited. root.on('in', { '@': msg['#'], how: 'mem', @@ -1963,7 +1963,7 @@ var mesh = function(){}; var opt = root.opt || {}; opt.log = opt.log || console.log; - opt.gap = opt.gap || opt.wait || 1; + opt.gap = opt.gap || opt.wait || 0; opt.pack = opt.pack || (opt.memory? (opt.memory * 1000 * 1000) : 1399000000) * 0.3; // max_old_space_size defaults to 1400 MB. var dup = root.dup; @@ -2009,9 +2009,9 @@ } return; } - var S, ST; LOG && (S = +new Date); + var S, ST; LOG && (S = +new Date); console.STAT = {}; root.on('in', msg); - if(LOG && !msg.nts && (ST = +new Date - S) > 9){ opt.log(S, ST, 'msg', msg['#']); if(ST > 500){ try{ require('./lib/email').send({text: ""+ST+"ms "+JSON.stringify(msg), from: "mark@gun.eco", to: "mark@gun.eco", subject: "GUN MSG"}, noop); }catch(e){} } } // this is ONLY turned on if ENV CONFIGS have email/password to send out from. + if(LOG && !msg.nts && (ST = +new Date - S) > 9){ opt.log(S, ST, 'msg', msg['#'], JSON.stringify(console.STAT)); if(ST > 500){ try{ require('./lib/email').send({text: ""+ST+"ms "+JSON.stringify(msg)+" | "+JSON.stringify(console.STAT), from: "mark@gun.eco", to: "mark@gun.eco", subject: "GUN MSG"}, noop); }catch(e){} } } // this is ONLY turned on if ENV CONFIGS have email/password to send out from. return; } } diff --git a/lib/rfsmix.js b/lib/rfsmix.js index dbe2cd97..1c06c7b5 100644 --- a/lib/rfsmix.js +++ b/lib/rfsmix.js @@ -3,11 +3,14 @@ module.exports = function(opt, store){ var p = store.put; var g = store.get; store.put = function(file, data, cb){ - rfs.put(file, data, function(err, ok){ - if(err){ return cb(err) } - //console.log("rfs3 cached", file); - p(file, data, cb); - }); + var a, b, c = function(err, ok){ + if(b){ return cb(err || b) } + if(a){ return cb(err, ok) } + a = true; + b = err; + } + p(file, data, c); // parallel + rfs.put(file, data, c); // parallel } store.get = function(file, cb){ rfs.get(file, function(err, data){ diff --git a/lib/serve.js b/lib/serve.js index b315cf88..55d36257 100644 --- a/lib/serve.js +++ b/lib/serve.js @@ -8,10 +8,10 @@ function CDN(dir){ req.url = (req.url||'').replace(dot,'').replace(slash,'/'); if(serve(req, res)){ return } // filters GUN requests! fs.createReadStream(path.join(dir, req.url)).on('error',function(tmp){ // static files! - try{ tmp = fs.readFileSync(path.join(dir, 'index.html')) }catch(e){} - try{ res.writeHead(200, {'Content-Type': 'text/html'}); - res.end(tmp+''); }catch(e){} // or default to index - }).pipe(res); // stream + fs.readFile(path.join(dir, 'index.html'), function(err, tmp){ + try{ res.writeHead(200, {'Content-Type': 'text/html'}); + res.end(tmp+''); }catch(e){} // or default to index + })}).pipe(res); // stream } } @@ -28,12 +28,9 @@ function serve(req, res, next){ var tmp; } if(0 <= req.url.indexOf('gun/')){ res.writeHead(200, {'Content-Type': 'text/javascript'}); - var path = __dirname + '/../' + req.url.split('/').slice(2).join('/'), file; - try{file = require('fs').readFileSync(path)}catch(e){} - if(file){ - res.end(file); - return true; - } + var path = __dirname + '/../' + req.url.split('/').slice(2).join('/'); + fs.readFile(path, function(err, file){ res.end((file || (err && 404))+'') }); + return true; } if((tmp = req.socket) && (tmp = tmp.server) && (tmp = tmp.route)){ var url; if(tmp = tmp[(((req.url||'').slice(1)).split('/')[0]||'').split('.')[0]]){ diff --git a/lib/store.js b/lib/store.js index 68af264c..55a32487 100644 --- a/lib/store.js +++ b/lib/store.js @@ -15,8 +15,10 @@ Gun.on('create', function(root){ root.on('put', function(msg){ this.to.next(msg); var id = msg['#'] || Gun.text.random(3), track = !msg['@'], acks = track? 0 : u; // only ack non-acks. - var got = (msg._||empty).rad, now = Gun.state(); + var _ = (msg._||empty), got = _.rad; + if(_.ram){ return } // in-memory ACKs to GETs do not need to be written to disk again. var S = (+new Date), C = 0; // STATS! + var now = Gun.state(); Gun.graph.is(msg.put, null, function(val, key, node, soul){ if(!track && got){ var at = (root.next||empty)[soul]; @@ -33,6 +35,7 @@ Gun.on('create', function(root){ C++; }); if(LOG && (ST = +new Date - S) > 9){ Gun.log(S, ST, 'put loop'); Gun.log(S, C, 'put loop #') } + console.STAT && (console.STAT.radputloop = ST) && (console.STAT.radputcount = C); function ack(err, ok){ acks--; if(ack.err){ return } @@ -104,13 +107,15 @@ Gun.on('create', function(root){ } if(!graph && data){ each(data, '') } } + console.STAT && (console.STAT.radgetcount = C); if(LOG && (ST = +new Date - S) > 9){ Gun.log(S, ST, 'got prep time'); Gun.log(S, C, 'got prep #') } C = 0; S = +new Date; var faith = function(){}; faith.faith = true; faith.rad = get; // HNPERF: We're testing performance improvement by skipping going through security again, but this should be audited. root.on('in', {'@': id, put: graph, '%': o.more? 1 : u, err: err? err : u, _: faith}); LOG && (ST = +new Date - S) > 9 && Gun.log(S, ST, 'got emit', Object.keys(graph||{}).length); graph = u; // each is outside our scope, we have to reset graph to nothing! }, o); - LOG && Gun.log(S, +new Date - S, 'get call'); + LOG && (ST = +new Date - S) > 9 && Gun.log(S, ST, 'get call'); + console.STAT && (console.STAT.radget = ST); function each(val, has, a,b){ // TODO: THIS CODE NEEDS TO BE FASTER!!!! C++; if(!val){ return } From 328ae52c2c42f7be7e2c2cb9764237e2eca70f02 Mon Sep 17 00:00:00 2001 From: Mark Nadal Date: Sun, 2 Feb 2020 01:09:48 -0800 Subject: [PATCH 10/76] begin --- gun.js | 21 +++++++++++++-------- lib/evict.js | 2 +- lib/radisk.js | 6 ++++-- lib/store.js | 2 +- 4 files changed, 19 insertions(+), 12 deletions(-) diff --git a/gun.js b/gun.js index 9a165f46..b4f81ef9 100644 --- a/gun.js +++ b/gun.js @@ -1957,7 +1957,7 @@ ;USE(function(module){ var Type = USE('../type'); - var puff = (typeof setImmediate !== "undefined")? setImmediate : setTimeout; + var puff = setTimeout; //(typeof setImmediate !== "undefined")? setImmediate : setTimeout; function Mesh(root){ var mesh = function(){}; @@ -1976,15 +1976,18 @@ if('{' != raw[2]){ mesh.hear.d += raw.length||0; ++mesh.hear.c; } // STATS! // ugh, stupid double JSON encoding if('[' === tmp){ try{msg = JSON.parse(raw);}catch(e){opt.log('DAM JSON parse error', e)} + raw = ''; if(!msg){ return } LOG && opt.log(+new Date, msg.length, '# on hear batch'); (function go(){ // stats on single msg is 99% spike of batch stats. - var m, c = 100; // hardcoded for now? - while(c-- && (m = msg.shift())){ + var m, i = 0, c = 100; // hardcoded for now? + while(c-- && (m = msg[i++])){ mesh.hear(m, peer); } + msg = msg.slice(i); + // TODO: consider triggering peer flush to response times speedier! if(!msg.length){ return } - puff(go, 0); + puff(go, 1); }()); return; } @@ -1995,14 +1998,15 @@ if(!(id = msg['#'])){ id = msg['#'] = Type.text.random(9) } if(msg.DBG_s){ opt.log(+new Date - msg.DBG_s, 'to hear', id) } if(dup.check(id)){ return } - dup.track(id, true).it = it(msg); // GUN core also dedups, so `true` is needed. // Does GUN core need to dedup anymore? - if(!(hash = msg['##']) && u !== msg.put){ hash = msg['##'] = Type.obj.hash(msg.put) } + dup.track(id, true); //.it = it(msg); // GUN core also dedups, so `true` is needed. // Does GUN core need to dedup anymore? + /*if(!(hash = msg['##']) && u !== msg.put){ hash = msg['##'] = Type.obj.hash(msg.put) } if(hash && (tmp = msg['@'] || (msg.get && id))){ // Reduces backward daisy in case varying hashes at different daisy depths are the same. if(dup.check(tmp+hash)){ return } dup.track(tmp+hash, true).it = it(msg); // GUN core also dedups, so `true` is needed. // Does GUN core need to dedup anymore? } - (msg._ = function(){}).via = peer; if(tmp = msg['><']){ (msg._).to = Type.obj.map(tmp.split(','), tomap) } + */ // TOOD: COME BACK TO THIS LATER!!! IMPORTANT MESH STUFF!! + (msg._ = function(){}).via = peer; if(msg.dam){ if(tmp = mesh.hear[msg.dam]){ tmp(msg, peer, root); @@ -2010,7 +2014,8 @@ return; } var S, ST; LOG && (S = +new Date); console.STAT = {}; - root.on('in', msg); + //root.on('in', msg); + root.on('in2', msg); if(LOG && !msg.nts && (ST = +new Date - S) > 9){ opt.log(S, ST, 'msg', msg['#'], JSON.stringify(console.STAT)); if(ST > 500){ try{ require('./lib/email').send({text: ""+ST+"ms "+JSON.stringify(msg)+" | "+JSON.stringify(console.STAT), from: "mark@gun.eco", to: "mark@gun.eco", subject: "GUN MSG"}, noop); }catch(e){} } } // this is ONLY turned on if ENV CONFIGS have email/password to send out from. return; } diff --git a/lib/evict.js b/lib/evict.js index 48e69437..8647f95e 100644 --- a/lib/evict.js +++ b/lib/evict.js @@ -10,7 +10,7 @@ try{ heap = require('v8').getHeapStatistics }catch(e){} if(!heap){ return } - ev.max = parseFloat(root.opt.memory || (heap().heap_size_limit / 1024 / 1024) || process.env.WEB_MEMORY || 1399) * 0.8; // max_old_space_size defaults to 1400 MB. Note: old space !== memory space though. + ev.max = parseFloat(root.opt.memory || (heap().heap_size_limit / 1024 / 1024) || process.env.WEB_MEMORY || 1399) * 0.8; // max_old_space_size defaults to 1400 MB. Note: old space !== memory space though. // KEEPING USED_HEA_SIZE < HEAP_SIZE_LIMIT ONLY THING TO BE BELOW TO PREVENT CRASH! setInterval(check, 1000); function check(){ diff --git a/lib/radisk.js b/lib/radisk.js index b08a56e8..3e7ac2f0 100644 --- a/lib/radisk.js +++ b/lib/radisk.js @@ -174,12 +174,14 @@ return; } disk = disk || Radix(); - var S; LOG && (S = +new Date); + var S, C = 0; LOG && (S = +new Date); Radix.map(mem, function(val, key){ + C++; // PLUGIN: consider adding HAM as an extra layer of protection disk(key, val); // merge batch[key] -> disk[key] }); - LOG && opt.log(S, +new Date - S, "rad merge mem & disk"); + LOG && opt.log(S, +new Date - S, "rad merge"); + LOG && opt.log(S, C, "rad merge #"); r.write(file, disk, s.pop); }) } diff --git a/lib/store.js b/lib/store.js index 55a32487..be5c5b0a 100644 --- a/lib/store.js +++ b/lib/store.js @@ -96,7 +96,7 @@ Gun.on('create', function(root){ if(err){ opt.store.stats.get.err = err } }catch(e){} // STATS! //if(u === data && o.chunks > 1){ return } // if we already sent a chunk, ignore ending empty responses. // this causes tests to fail. - LOG && Gun.log(S, +new Date - S, 'got'); S = +new Date; + LOG && Gun.log(S, +new Date - S, 'got', JSON.stringify(key)); S = +new Date; if(data){ if(typeof data !== 'string'){ if(o.atom){ From d3e89c2adc24e56ca6c707306ba080eee1821535 Mon Sep 17 00:00:00 2001 From: Mark Nadal Date: Sun, 2 Feb 2020 01:13:47 -0800 Subject: [PATCH 11/76] live debug ? --- Aptfile | 2 ++ Procfile | 2 +- 2 files changed, 3 insertions(+), 1 deletion(-) create mode 100644 Aptfile diff --git a/Aptfile b/Aptfile new file mode 100644 index 00000000..1b55cba9 --- /dev/null +++ b/Aptfile @@ -0,0 +1,2 @@ +linux-tools-common +linux-tools-generic \ No newline at end of file diff --git a/Procfile b/Procfile index 875fe822..f814922f 100644 --- a/Procfile +++ b/Procfile @@ -1 +1 @@ -web: node --optimize_for_size --gc_interval=100 examples/http.js +web: node --inspect examples/http.js From c5d3ef47e723e6e584eeb7d5910aaa1bbbb6b063 Mon Sep 17 00:00:00 2001 From: Mark Nadal Date: Sun, 2 Feb 2020 09:17:43 -0800 Subject: [PATCH 12/76] try live debug (chrome inspector) --- Procfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Procfile b/Procfile index f814922f..cbb01468 100644 --- a/Procfile +++ b/Procfile @@ -1 +1 @@ -web: node --inspect examples/http.js +web: node --debug=9090 --inspect --perf-basic-prof-only-functions examples/http.js From cc6415fa444f56190802659bf9c00e50b46024f7 Mon Sep 17 00:00:00 2001 From: Mark Nadal Date: Sun, 2 Feb 2020 09:24:37 -0800 Subject: [PATCH 13/76] Update Aptfile --- Aptfile | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/Aptfile b/Aptfile index 1b55cba9..8bac94ca 100644 --- a/Aptfile +++ b/Aptfile @@ -1,2 +1,6 @@ linux-tools-common -linux-tools-generic \ No newline at end of file +linux-tools-generic +linux-tools-4.4.0-1057-aws +linux-cloud-tools-4.4.0-1057-aws +linux-tools-aws +linux-cloud-tools-aws \ No newline at end of file From ebfd82eac6d7a8adb2e114317cd53e059f0d4625 Mon Sep 17 00:00:00 2001 From: Mark Nadal Date: Sun, 2 Feb 2020 09:27:51 -0800 Subject: [PATCH 14/76] Update Aptfile --- Aptfile | 2 -- 1 file changed, 2 deletions(-) diff --git a/Aptfile b/Aptfile index 8bac94ca..0d99b744 100644 --- a/Aptfile +++ b/Aptfile @@ -1,6 +1,4 @@ linux-tools-common linux-tools-generic -linux-tools-4.4.0-1057-aws -linux-cloud-tools-4.4.0-1057-aws linux-tools-aws linux-cloud-tools-aws \ No newline at end of file From 6281fed309509724595cbed4e044120a447f52f5 Mon Sep 17 00:00:00 2001 From: Mark Nadal Date: Sun, 2 Feb 2020 09:31:26 -0800 Subject: [PATCH 15/76] Update Aptfile --- Aptfile | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/Aptfile b/Aptfile index 0d99b744..844cac60 100644 --- a/Aptfile +++ b/Aptfile @@ -1,4 +1,3 @@ linux-tools-common linux-tools-generic -linux-tools-aws -linux-cloud-tools-aws \ No newline at end of file +linux-tools-aws \ No newline at end of file From cae82648321662d625fdbe3696ffaa4dfb1e528f Mon Sep 17 00:00:00 2001 From: Mark Nadal Date: Sun, 2 Feb 2020 09:40:49 -0800 Subject: [PATCH 16/76] Update Procfile --- Procfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Procfile b/Procfile index cbb01468..92d3c2f2 100644 --- a/Procfile +++ b/Procfile @@ -1 +1 @@ -web: node --debug=9090 --inspect --perf-basic-prof-only-functions examples/http.js +web: node --inspect --perf-basic-prof-only-functions examples/http.js From c97526a36cea2ca1d46e4c1b94496be28338dbb2 Mon Sep 17 00:00:00 2001 From: Mark Nadal Date: Sun, 2 Feb 2020 09:40:59 -0800 Subject: [PATCH 17/76] Update Procfile --- Procfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Procfile b/Procfile index 92d3c2f2..8006d00b 100644 --- a/Procfile +++ b/Procfile @@ -1 +1 @@ -web: node --inspect --perf-basic-prof-only-functions examples/http.js +web: node --inspect=9090 --perf-basic-prof-only-functions examples/http.js From d8f3ed7edbb913a78f7220ec2518ea6ad99c59d5 Mon Sep 17 00:00:00 2001 From: Mark Nadal Date: Sun, 2 Feb 2020 10:05:10 -0800 Subject: [PATCH 18/76] Update Procfile --- Procfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Procfile b/Procfile index 8006d00b..516fbc04 100644 --- a/Procfile +++ b/Procfile @@ -1 +1 @@ -web: node --inspect=9090 --perf-basic-prof-only-functions examples/http.js +web: node --inspect=0.0.0.0:9090 --perf-basic-prof-only-functions examples/http.js From 75148ec5409ab5f19d8b5ed529421e0be8676fd4 Mon Sep 17 00:00:00 2001 From: Mark Nadal Date: Sun, 2 Feb 2020 10:33:37 -0800 Subject: [PATCH 19/76] Update Procfile --- Procfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Procfile b/Procfile index 516fbc04..f814922f 100644 --- a/Procfile +++ b/Procfile @@ -1 +1 @@ -web: node --inspect=0.0.0.0:9090 --perf-basic-prof-only-functions examples/http.js +web: node --inspect examples/http.js From a980f4efc176195ce7bdb791b0e5f68962bae87c Mon Sep 17 00:00:00 2001 From: Mark Nadal Date: Sun, 2 Feb 2020 22:30:13 -0800 Subject: [PATCH 20/76] ? trace ? tmp --- Procfile | 2 +- examples/http.js | 4 ++-- package.json | 1 + trace.js | 25 +++++++++++++++++++++++++ 4 files changed, 29 insertions(+), 3 deletions(-) create mode 100644 trace.js diff --git a/Procfile b/Procfile index f814922f..49049048 100644 --- a/Procfile +++ b/Procfile @@ -1 +1 @@ -web: node --inspect examples/http.js +web: node --inspect trace.js diff --git a/examples/http.js b/examples/http.js index 3ae161e1..dba51779 100644 --- a/examples/http.js +++ b/examples/http.js @@ -1,8 +1,8 @@ ;(function(){ - var cluster = require('cluster'); + /*var cluster = require('cluster'); if(cluster.isMaster){ return cluster.fork() && cluster.on('exit', function(){ cluster.fork() }); - } + }*/ var fs = require('fs'); var config = { port: process.env.OPENSHIFT_NODEJS_PORT || process.env.VCAP_APP_PORT || process.env.PORT || process.argv[2] || 8765 }; diff --git a/package.json b/package.json index e31efe13..1f7cdc09 100644 --- a/package.json +++ b/package.json @@ -68,6 +68,7 @@ "mocha": "^6.2.0", "panic-manager": "^1.2.0", "panic-server": "^1.1.1", + "0x": "^4.9.1", "uglify-js": "^3.6.0" } } diff --git a/trace.js b/trace.js new file mode 100644 index 00000000..8c07a085 --- /dev/null +++ b/trace.js @@ -0,0 +1,25 @@ +;(async function start(){ + var z; + try{ z = require('0x'); + } catch(e){ + return require('./examples/http.js'); + } + function go(){ + start(); + setTimeout(function(){try{ + var zip = require("child_process"); + zip.execSync('zip -r flame *', {cwd: './flame'}); + require('./lib/fsrm')('./flame'); + require('./lib/email').send({ + text: "zip attached", + from: "mark@gun.eco", + to: "mark@gun.eco", + subject: "TRACE GUN", + attachment:[{path:"./flame.zip", type:"application/zip", name:"flame.zip"}] + }, function(err){ + err && console.log("@@@@@@@@@@ EMAIL ERROR @@@@@@@@@@", err); + }) + }catch(err){ console.log("@@@@@@@@@@ TRACE ERROR @@@@@@@@@", err) }},5000); + } + require('0x')({argv: ['./examples/http.js'], outputDir: 'flame', workingDir: __dirname, onProcessExit: go}); +}()); \ No newline at end of file From 782750d7287831377536031df0cf083f88c82291 Mon Sep 17 00:00:00 2001 From: Mark Nadal Date: Sun, 2 Feb 2020 22:57:06 -0800 Subject: [PATCH 21/76] Update trace.js --- trace.js | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/trace.js b/trace.js index 8c07a085..fe75e166 100644 --- a/trace.js +++ b/trace.js @@ -8,18 +8,18 @@ start(); setTimeout(function(){try{ var zip = require("child_process"); - zip.execSync('zip -r flame *', {cwd: './flame'}); - require('./lib/fsrm')('./flame'); + zip.execSync('zip -r flametracedata.zip flamedata/'); + require('./lib/fsrm')('./flamedata'); require('./lib/email').send({ text: "zip attached", from: "mark@gun.eco", to: "mark@gun.eco", subject: "TRACE GUN", - attachment:[{path:"./flame.zip", type:"application/zip", name:"flame.zip"}] + attachment:[{path: __dirname+"/flametracedata.zip", type:"application/zip", name:"flametracedata.zip"}] }, function(err){ err && console.log("@@@@@@@@@@ EMAIL ERROR @@@@@@@@@@", err); }) }catch(err){ console.log("@@@@@@@@@@ TRACE ERROR @@@@@@@@@", err) }},5000); } - require('0x')({argv: ['./examples/http.js'], outputDir: 'flame', workingDir: __dirname, onProcessExit: go}); + require('0x')({argv: ['./examples/http.js'], outputDir: 'flamedata', workingDir: __dirname, onProcessExit: go}); }()); \ No newline at end of file From a5a0eab24ccad99fa4c172cd4c66201c2ce4d471 Mon Sep 17 00:00:00 2001 From: Mark Nadal Date: Sun, 2 Feb 2020 22:59:48 -0800 Subject: [PATCH 22/76] Update trace.js --- trace.js | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/trace.js b/trace.js index fe75e166..e7d57879 100644 --- a/trace.js +++ b/trace.js @@ -17,7 +17,9 @@ subject: "TRACE GUN", attachment:[{path: __dirname+"/flametracedata.zip", type:"application/zip", name:"flametracedata.zip"}] }, function(err){ - err && console.log("@@@@@@@@@@ EMAIL ERROR @@@@@@@@@@", err); + if(!err){ return } + console.log("@@@@@@@@@@ EMAIL ERROR @@@@@@@@@@", err); + require('./lib/email').send({text: "check https://gunjs.herokuapp.com/gun/flametracedata.zip", from: "mark@gun.eco", to: "mark@gun.eco", subject: "TRACE GUN CHECK"}, function(err){}); }) }catch(err){ console.log("@@@@@@@@@@ TRACE ERROR @@@@@@@@@", err) }},5000); } From 827197dbb7be483852a4bbf70c41ab49a73f0c65 Mon Sep 17 00:00:00 2001 From: Mark Nadal Date: Mon, 3 Feb 2020 10:27:49 -0800 Subject: [PATCH 23/76] only flamegraph emails --- gun.js | 2 +- lib/radisk.js | 2 -- 2 files changed, 1 insertion(+), 3 deletions(-) diff --git a/gun.js b/gun.js index 9a165f46..3b9860a2 100644 --- a/gun.js +++ b/gun.js @@ -2011,7 +2011,7 @@ } var S, ST; LOG && (S = +new Date); console.STAT = {}; root.on('in', msg); - if(LOG && !msg.nts && (ST = +new Date - S) > 9){ opt.log(S, ST, 'msg', msg['#'], JSON.stringify(console.STAT)); if(ST > 500){ try{ require('./lib/email').send({text: ""+ST+"ms "+JSON.stringify(msg)+" | "+JSON.stringify(console.STAT), from: "mark@gun.eco", to: "mark@gun.eco", subject: "GUN MSG"}, noop); }catch(e){} } } // this is ONLY turned on if ENV CONFIGS have email/password to send out from. + if(LOG && !msg.nts && (ST = +new Date - S) > 9){ opt.log(S, ST, 'msg', msg['#'], JSON.stringify(console.STAT)) } return; } } diff --git a/lib/radisk.js b/lib/radisk.js index b08a56e8..1efe678f 100644 --- a/lib/radisk.js +++ b/lib/radisk.js @@ -228,7 +228,6 @@ //opt.store.put(tmp, f.text, cb); // revert to this after stats done below: opt.store.put(tmp, f.text, function(err,ok){ LOG && opt.log(S, ST = +new Date - S, "wrote disk", tmp); - LOG && (ST > 500) && require('./email').send({text: ""+ST+"ms "+tmp, from: "mark@gun.eco", to: "mark@gun.eco", subject: "RAD DISK WROTE"}, noop); // this is ONLY turned on if ENV CONFIGS have email/password to send out from. cb(err,ok); }); }); @@ -438,7 +437,6 @@ p.disk.$ = json; LOG && (ST = +new Date - S) > 9 && opt.log(S, ST, 'rad parsed JSON'); map(q, p.ack); - LOG && (ST > 500) && require('./email').send({text: ""+ST+"ms "+ename(file), from: "mark@gun.eco", to: "mark@gun.eco", subject: "RAD JSON parse"}, noop); // this is ONLY turned on if ENV CONFIGS have email/password to send out from. return; }catch(e){ tmp = e } if('{' === data[0]){ From efd055c2b07f9bceb5ed4e2f2477df0b693a2ac7 Mon Sep 17 00:00:00 2001 From: Mark Nadal Date: Mon, 3 Feb 2020 14:25:41 -0800 Subject: [PATCH 24/76] notes & tests --- lib/radisk.js | 2 +- lib/radix.js | 13 +++--- lib/store.js | 2 +- test/mocha.html | 2 +- test/ptsd/radix.js | 102 +++++++++++++++++++++++++++++++++++++++++++-- test/rad/rad.js | 13 +++++- test/radix.js | 2 +- 7 files changed, 121 insertions(+), 15 deletions(-) diff --git a/lib/radisk.js b/lib/radisk.js index 3e7ac2f0..3e4fd152 100644 --- a/lib/radisk.js +++ b/lib/radisk.js @@ -274,7 +274,7 @@ if(u === o.start && u === o.end){ return tree } if(atomic(tree)){ return tree } var sub = Radix(); - Radix.map(tree, function(v,k){ + Radix.map(tree, function(v,k){ // ONLY PLACE THAT TAKES TREE, maybe reduce API for better perf? sub(k,v); }, o); return sub(''); diff --git a/lib/radix.js b/lib/radix.js index a3523f3e..e338e3fb 100644 --- a/lib/radix.js +++ b/lib/radix.js @@ -51,23 +51,24 @@ return radix; }; - Radix.map = function map(radix, cb, opt, pre){ pre = pre || []; // TODO: BUG: most out-of-memory crashes come from here. + Radix.map = function rap(radix, cb, opt, pre){ pre = pre || []; // TODO: BUG: most out-of-memory crashes come from here. var t = ('function' == typeof radix)? radix.$ || {} : radix; + //!opt && console.log("WHAT IS T?", JSON.stringify(t).length); if(!t){ return } - var keys = (t[_]||no).sort || (t[_] = function $(){ $.sort = Object.keys(t).sort(); return $ }()).sort, rev; + var keys = (t[_]||no).sort || (t[_] = function $(){ $.sort = Object.keys(t).sort(); return $ }()).sort, rev; // ONLY 17% of ops are pre-sorted! //var keys = Object.keys(t).sort(); opt = (true === opt)? {branch: true} : (opt || {}); - if(rev = opt.reverse){ keys = keys.slice().reverse() } + if(rev = opt.reverse){ keys = keys.slice(0).reverse() } var start = opt.start, end = opt.end, END = '\uffff'; var i = 0, l = keys.length; for(;i < l; i++){ var key = keys[i], tree = t[key], tmp, p, pt; if(!tree || '' === key || _ === key){ continue } - p = pre.slice(); p.push(key); + p = pre.slice(0); p.push(key); pt = p.join(''); if(u !== start && pt < (start||'').slice(0,pt.length)){ continue } if(u !== end && (end || END) < pt){ continue } if(rev){ // children must be checked first when going in reverse. - tmp = map(tree, cb, opt, p); + tmp = rap(tree, cb, opt, p); if(u !== tmp){ return tmp } } if(u !== (tmp = tree[''])){ @@ -85,7 +86,7 @@ } pre = p; if(!rev){ - tmp = map(tree, cb, opt, pre); + tmp = rap(tree, cb, opt, pre); if(u !== tmp){ return tmp } } pre.pop(); diff --git a/lib/store.js b/lib/store.js index be5c5b0a..80c26c36 100644 --- a/lib/store.js +++ b/lib/store.js @@ -102,7 +102,7 @@ Gun.on('create', function(root){ if(o.atom){ data = u; } else { - Radix.map(data, each) + Radix.map(data, each); // IS A RADIX TREE, NOT FUNCTION! } } if(!graph && data){ each(data, '') } diff --git a/test/mocha.html b/test/mocha.html index 229040e6..13bcf7a4 100644 --- a/test/mocha.html +++ b/test/mocha.html @@ -38,7 +38,7 @@ console.log('async?', Gun.debug); } var run = mocha.run(function(a,b,c){ - //document.body.prepend("MARK! REMEMBER TO REMOVE RETURN!");return; + document.body.prepend("MARK! REMEMBER TO REMOVE RETURN!");return; var yes = confirm("REFRESH BROWSER FOR ASYNC TESTS?"); if(yes){ if(location.search){ diff --git a/test/ptsd/radix.js b/test/ptsd/radix.js index 81a8227d..4e4d5f79 100644 --- a/test/ptsd/radix.js +++ b/test/ptsd/radix.js @@ -16,11 +16,105 @@ stool.run(); },1); stool.setup(window.setup = function(){ - window.BigText = Gun.text.random(1024, 'abcdef'); - window.MedText = Gun.text.random(200, 'abcdef'); - window.jsonText = JSON.stringify(window.BigText); - window.radText = Radisk.encode(window.BigText); + //window.BigText = Gun.text.random(1024, 'abcdef'); + //window.MedText = Gun.text.random(200, 'abcdef'); + //window.jsonText = JSON.stringify(window.BigText); + //window.radText = Radisk.encode(window.BigText); + + window.namez = ["Adalard","Adora","Aia","Albertina","Alfie","Allyn","Amabil","Ammamaria","Andy","Anselme","Ardeen","Armand","Ashelman","Aube","Averyl","Baker","Barger","Baten","Bee","Benia","Bernat","Bevers","Bittner","Bobbe","Bonny","Boyce","Breech","Brittaney","Bryn","Burkitt","Cadmann","Campagna","Carlee","Carver","Cavallaro","Chainey","Chaunce","Ching","Cianca","Claudina","Clyve","Colon","Cooke","Corrina","Crawley","Cullie","Dacy","Daniela","Daryn","Deedee","Denie","Devland","Dimitri","Dolphin","Dorinda","Dream","Dunham","Eachelle","Edina","Eisenstark","Elish","Elvis","Eng","Erland","Ethan","Evelyn","Fairman","Faus","Fenner","Fillander","Flip","Foskett","Fredette","Fullerton","Gamali","Gaspar","Gemina","Germana","Gilberto","Giuditta","Goer","Gotcher","Greenstein","Grosvenor","Guthrey","Haldane","Hankins","Harriette","Hayman","Heise","Hepsiba","Hewie","Hiroshi","Holtorf","Howlond","Hurless","Ieso","Ingold","Isidora","Jacoba","Janelle","Jaye","Jennee","Jillana","Johnson","Josy","Justinian","Kannan","Kast","Keeley","Kennett","Kho","Kiran","Knowles","Koser","Kroll","LaMori","Lanctot","Lasky","Laverna","Leff","Leonanie","Lewert","Lilybel","Lissak","Longerich","Lou","Ludeman","Lyman","Madai","Maia","Malvina","Marcy","Maris","Martens","Mathilda","Maye","McLain","Melamie","Meras","Micco","Millburn","Mittel","Montfort","Moth","Mutz","Nananne","Nazler","Nesta","Nicolina","Noellyn","Nuli","Ody","Olympie","Orlena","Other","Pain","Parry","Paynter","Pentheas","Pettifer","Phyllida","Plath","Posehn","Proulx","Quinlan","Raimes","Ras","Redmer","Renelle","Ricard","Rior","Rocky","Ron","Rosetta","Rubia","Ruttger","Salbu","Sandy","Saw","Scholz","Secor","September","Shanleigh","Shenan","Sholes","Sig","Sisely","Soble","Spanos","Stanwinn","Stevie","Stu","Suzanne","Tacy","Tanney","Tekla","Thackeray","Thomasin","Tilla","Tomas","Tracay","Tristis","Ty","Urana","Valdis","Vasta","Vezza","Vitoria","Wait","Warring","Weissmann","Whetstone","Williamson","Wittenburg","Wymore","Yoho","Zamir","Zimmermann"]; + window.radiz = window.radiz || Radix(); + window.arr = []; var i = 1000; while(--i){ arr.push(Math.random()) } + window.arrs = arr.slice(0).sort(); }); + /* TEMPORARY COPY OF RADIX UNIT TESTS TO BOOST SPEED */ + /* THESE ARE PROBABLY STALE AND NEED TO BE COPIED FROM UNIT TESTS AGAIN */ + stool.add('1', function(){ + var rad = Radix(); + rad('asdf.pub', 'yum'); + rad('ablah', 'cool'); + rad('node/circle.bob', 'awesome'); + + (rad('asdf.').pub[''] !== 'yum') && bad1; + (rad('nv/foo.bar') !== undefined) && bad2; + }); + stool.add('2', function(){ + var all = {}; + namez.forEach(function(v,i){ + v = v.toLowerCase(); + all[v] = v; + radiz(v, i) + }); + (Gun.obj.empty(all) === true) && bad3; + Radix.map(radiz, function(v,k){ + delete all[k]; + }); + (Gun.obj.empty(all) !== true) && bad4; + }); + stool.add('3', function(){ + var all = {}; + namez.forEach(function(v,i){ + v = v.toLowerCase(); + all[v] = v; + //rad(v, i) + }); + (Gun.obj.empty(all) === true) && bad5; + Radix.map(radiz, function(v,k){ + delete all[k]; + }); + (Gun.obj.empty(all) !== true) && bad6; + }); + stool.add('4', function(){ + var all = {}, start = 'Warring'.toLowerCase(), end = 'Zamir'.toLowerCase(); + namez.forEach(function(v,i){ + v = v.toLowerCase(); + if(v < start){ return } + if(end < v){ return } + all[v] = v; + //rad(v, i) + }); + (Gun.obj.empty(all) === true) && bad7; + Radix.map(radiz, function(v,k, a,b){ + //if(!all[k]){ throw "out of range!" } + delete all[k]; + }, {start: start, end: end}); + (Gun.obj.empty(all) !== true) && bad8; + }); + stool.add('5', function(){ + var all = {}, start = 'Warrinf'.toLowerCase(), end = 'Zamis'.toLowerCase(); + namez.forEach(function(v,i){ + v = v.toLowerCase(); + if(v < start){ return } + if(end < v){ return } + all[v] = v; + //rad(v, i) + }); + (Gun.obj.empty(all) === true) && bad9; + Radix.map(radiz, function(v,k, a,b){ + //if(!all[k]){ throw "out of range!" } + delete all[k]; + }, {start: start, end: end}); + (Gun.obj.empty(all) !== true) && bad10; + }); + stool.add('6', function(){ + var r = Radix(), tmp; + r('alice', 1);r('bob', 2);r('carl', 3);r('carlo',4); + r('dave', 5);r('zach',6);r('zachary',7); + var by = ['alice','bob','carl','carlo','dave','zach','zachary']; + Gun.obj.map(by, function(k,i){ + r(k,i); + }); + Radix.map(r, function(v,k, a,b){ + (by.pop() !== k) && bad11; + tmp = v; + }, {reverse: 1}); + (tmp !== 1) && bad12; + (by.length !== 0) && bad13; + Radix.map(r, function(v,k, a,b){ + tmp = v; + }); + (tmp !== 7) && bad14; + }); + return; stool.add('JSON encode string', function(){ JSON.stringify(window.BigText); }); diff --git a/test/rad/rad.js b/test/rad/rad.js index d1a638d2..28ccda24 100644 --- a/test/rad/rad.js +++ b/test/rad/rad.js @@ -43,8 +43,19 @@ var names = ["Adalard","Adora","Aia","Albertina","Alfie","Allyn","Amabil","Ammam //console.log("HYPER TEST");var z = 10000; while(--z){ names.push(Gun.text.random(7)) }this.timeout(9000); - describe('Radix', function(){ + describe.only('Radix', function(){ var radix = Radix(); + + it('unit', function(){ + var rad = Radix(); + rad('asdf.pub', 'yum'); + rad('ablah', 'cool'); + rad('node/circle.bob', 'awesome'); + + expect(rad('asdf.')).to.be.eql({pub: {'': 'yum'}}); + expect(rad('nv/foo.bar')).to.be(undefined); + }); + it('radix write read', function(done){ var all = {}; names.forEach(function(v,i){ diff --git a/test/radix.js b/test/radix.js index 38450c1e..7ec5f37e 100644 --- a/test/radix.js +++ b/test/radix.js @@ -3,7 +3,7 @@ var expect = global.expect = require("./expect"); var Radix = require('../lib/radix'); var _ = String.fromCharCode(29); -describe('Radix', function(){ +describe('Radix', function(){ // moved to ./rad/rad.js it('read', function(){ var rad = Radix(); rad('asdf.pub', 'yum'); From fa8a63606c14bdd993cc2dca811c7cfcb99c72c2 Mon Sep 17 00:00:00 2001 From: Mark Nadal Date: Mon, 3 Feb 2020 14:26:01 -0800 Subject: [PATCH 25/76] TODO SPAM, REMEMBER TO UNCOMMIT --- examples/todo/index.html | 24 ++++++++++++++++++++++++ 1 file changed, 24 insertions(+) diff --git a/examples/todo/index.html b/examples/todo/index.html index 1bba2922..e1061c57 100644 --- a/examples/todo/index.html +++ b/examples/todo/index.html @@ -114,6 +114,30 @@ +

    - + + @@ -16,87 +17,6 @@
    - - - - + \ No newline at end of file From 9b188dea726d37d706cd555a2e7e768774666211 Mon Sep 17 00:00:00 2001 From: Mark Nadal Date: Thu, 6 Feb 2020 14:11:47 -0800 Subject: [PATCH 28/76] Update .gitignore --- .gitignore | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/.gitignore b/.gitignore index 74fb92eb..393802ed 100644 --- a/.gitignore +++ b/.gitignore @@ -7,7 +7,10 @@ yarn.lock .idea/ *.bak *.new +*.log +v8.json *.DS_store +isolate*.log .esm-cache .sessionStorage -.localStorage \ No newline at end of file +.localStorage From 3127b41f38a29704ee929c423c195bf3523abf76 Mon Sep 17 00:00:00 2001 From: Mark Nadal Date: Sat, 8 Feb 2020 02:02:36 -0800 Subject: [PATCH 29/76] full circuit --- Aptfile | 3 - Procfile | 2 +- axe.js | 33 +++------ gun.js | 179 +++++++++++++++++++++++++---------------------- lib/multicast.js | 2 +- lib/radiskip.js | 33 ++++----- lib/radix.js | 6 +- lib/rfs.js | 8 ++- lib/rs3.js | 27 +++---- lib/store.js | 52 +++++++++----- package.json | 3 +- sea.js | 8 +-- 12 files changed, 188 insertions(+), 168 deletions(-) delete mode 100644 Aptfile diff --git a/Aptfile b/Aptfile deleted file mode 100644 index 844cac60..00000000 --- a/Aptfile +++ /dev/null @@ -1,3 +0,0 @@ -linux-tools-common -linux-tools-generic -linux-tools-aws \ No newline at end of file diff --git a/Procfile b/Procfile index 49049048..d18ab5df 100644 --- a/Procfile +++ b/Procfile @@ -1 +1 @@ -web: node --inspect trace.js +web: node --prof trace.js \ No newline at end of file diff --git a/axe.js b/axe.js index 2c5e32ba..19594d2c 100644 --- a/axe.js +++ b/axe.js @@ -102,11 +102,14 @@ if('string' != typeof (tmp = get['#'])){ return } return tmp; } + // TODO: AXE NEEDS TO BE CHECKED FOR NEW CODE SYSTEM!!!!!!!!!! var Rad = (Gun.window||{}).Radix || USE('./lib/radix', 1); at.opt.dht = Rad(); - at.on('in', function input(msg){ - var to = this.to, peer = (msg._||{}).via; + at.on('in', input); + at.on('in2', input); + function input(msg){ + var to = this.to, peer = (msg._||'').via || mesh.leap; // warning! mesh.leap could be buggy! var dht = opt.dht; var routes = axe.routes || (axe.routes = {}); // USE RAD INSTEAD! TMP TESTING! var get = msg.get, hash, tmp; @@ -131,8 +134,8 @@ } }*/ } - if((tmp = msg['@']) && (tmp = at.dup.s[tmp]) && (tmp = tmp.it)){ - (tmp = (tmp._||{})).ack = (tmp.ack || 0) + 1; // count remote ACKs to GET. + if((tmp = msg['@']) && (tmp = at.dup.s[tmp])){ + tmp.ack = (tmp.ack || 0) + 1; // count remote ACKs to GET. } to.next(msg); @@ -147,7 +150,7 @@ }); }); } - }); + }; //try{console.log(req.connection.remoteAddress)}catch(e){}; mesh.hear['opt'] = function(msg, peer){ @@ -209,7 +212,7 @@ if(peers === opt.peers){ return } // prevent infinite lack loop. return meta.turn = 0, chat(opt.peers, peers) } - var hash = msg['##'], ack = meta.ack; + var hash = msg['##'], ack = meta.ack || at.dup.s[msg['#']]; meta.lack = setTimeout(function(){ if(ack && hash && hash === msg['##']){ return } if(meta.turn >= (axe.turns || 3)){ return } // variable for later! Also consider ACK based turn limit. @@ -254,24 +257,6 @@ } }); }; - /*var connections = 0; // THIS HAS BEEN MOVED TO CORE NOW! - at.on('hi', function(opt) { - this.to.next(opt); - //console.log('AXE PEER [HI]', new Date(), opt); - connections++; - /// The first connection don't need to resubscribe the nodes. - if (connections === 1) { return; } - /// Resubscribe all nodes. - setTimeout(function() { - var souls = Object.keys(at.graph); - for (var i=0; i < souls.length; ++i) { - //at.gun.get(souls[i]).off(); - at.next[souls[i]].ack = 0; - at.gun.get(souls[i]).once(function(){}); - } - //location.reload(); - }, 500); - }, at);*/ } axe.up = {}; at.on('hi', function(peer){ diff --git a/gun.js b/gun.js index 8d2388ed..ece88877 100644 --- a/gun.js +++ b/gun.js @@ -596,9 +596,9 @@ USE('./onto'); // depends upon onto! module.exports = function ask(cb, as){ if(!this.on){ return } - if(!(cb instanceof Function)){ + if(!('function' == typeof cb)){ if(!cb || !as){ return } - var id = cb['#'] || cb, tmp = (this.tag||empty)[id]; + var id = cb['#'] || cb, tmp = (this.tag||'')[id]; if(!tmp){ return } tmp = this.on(id, as); clearTimeout(tmp.err); @@ -653,7 +653,7 @@ Gun.is = function($){ return ($ instanceof Gun) || ($ && $._ && ($ === $._.$)) || false } - Gun.version = 0.9; + Gun.version = 0.2020; Gun.chain = Gun.prototype; Gun.chain.toJSON = function(){}; @@ -680,8 +680,8 @@ if(!at.once){ at.on('in', root, at); at.on('in2', root2, at); + at.on('put2', map, at); at.on('out', root, {at: at, out: root}); - at.on('put2', cache, at); Gun.on('create', at); at.on('create', at); } @@ -689,6 +689,7 @@ return gun; } function root(msg){ + if(msg.out === root2){ this.to.next(msg); return } //add to.next(at); // TODO: MISSING FEATURE!!! var ev = this, as = ev.as, at = as.at || as, gun = at.$, dup, tmp; if(!(tmp = msg['#'])){ tmp = msg['#'] = text_rand(9) } @@ -722,81 +723,92 @@ if(dup.check(tmp)){ return } dup.track(tmp); tmp = msg._; msg._ = ('function' == typeof tmp)? tmp : function(){}; if(msg.get){ Gun.on.get(msg, gun) } - if(msg.put){ PUT(msg, gun) } + if(msg.put){ put(msg, gun) } eve.to.next(msg); + msg.out = root2; at.on('out', msg); // EXPERIMENTAL! } - }()); - - ;(function(){ - Gun.on.put2 = function(msg, gun){ - var ctx = msg._, root = ctx.root = gun._, id = msg['#']; - var ack = ctx.ack = function(soul, key, val, state){ - if(ack.err){ - if(ack.s){ root.on('in', {'@': id, err: Gun.log(ack.err)}) } - return ack.s = u; + function put(msg, gun){ + var ctx = msg._, root = ctx.root = gun._, put = msg.put, id = msg['#'], mid = 1; + if(put['#']){ root.on('put2', msg); return } + var all = ctx.all = function(err, ok){ + if(!id){ return } + if(all.err = all.err || err){ + root.on('in', {'@': id, err: Gun.log(all.err)}); + return id = u; } - var acks, tmp; - if((acks = ack.s||'')[tmp = soul+key]){ - delete acks[tmp]; - root.on('put2', {put: {'#': soul, '.': key, ':': val, '>': state}}); - } - if(!obj_empty(acks)){ return } // keep waiting - console.log("I'm all done!!!"); - } - ack.err = obj_map(msg.put, nodes, msg); - msg = ctx = u; - if(ack.err){ ack(); return } + if(mid){ return } + if(!obj_empty(all.s)){ return } // keep waiting + root.on('in', {'@': id, ok: ok || 1}); + id = u; + }; + all.err = obj_map(put, valid, msg); + ctx.node = ctx.state = u; + mid = msg = ctx = u; + all(); } - function nodes(node, soul){ + function valid(node, soul){ if(!node){ return ERR+cut(soul)+"no node." } var ctx = this._, tmp; + (ctx.root.opt||'').super && ctx.root.$.get(soul); // I think we need super for now, but since we are rewriting, should consider getting rid of it. if(!(tmp = node._)){ return ERR+cut(soul)+"no meta." } ctx.node = node; if(soul !== tmp[_soul]){ return ERR+cut(soul)+"soul not same." } ctx.soul = soul; if(!(ctx.states = tmp[state_])){ return ERR+cut(soul)+"no state." } - return obj_map(node, souls, this); + return obj_map(node, mix, this); } - function souls(val, key){ + function mix(val, key){ if(node_ === key){ return } var ctx = this._, soul = ctx.soul, state = ctx.states[key], tmp; if(u === state){ return ERR+cut(key)+"on"+cut(soul)+"no state." } if(!val_is(val)){ return ERR+cut(key)+"on"+cut(soul)+"bad "+(typeof val)+cut(val) } - ((tmp = ctx.ack).s || (tmp.s = {}))[soul+key] = 1; - ham(ctx.root.graph, soul, key, val, state, tmp); // TODO: HANDLE CALLBACK WHERE ALL DAY IS HISTORIC? + ham(val, key, soul, state, ctx); // TODO: HANDLE CALLBACK WHERE ALL DAY IS HISTORIC? } - function ham(graph, soul, key, val, state, ack){ + function ham(val, key, soul, state, ctx){ + var root = ctx.root, graph = root.graph, id = soul+key, all = ctx.all, alls; var vertex = graph[soul] || empty, was = state_is(vertex, key, 1), known = vertex[key]; var machine = State(), is = HAM(machine, state, was, val, known), u; + (alls = all.s || (all.s = {}))[id] = 1; if(!is.incoming){ if(is.defer){ var to = is.defer - machine; setTimeout(function(){ - ham(graph, soul, key, val, state, cb); + ham(val, key, soul, state, ctx); }, to > MD? MD : to); // setTimeout Max Defer 32bit :( return; } - if(ack){ ack(soul, key) } + delete alls[id]; + all(u, -1); return; } - if(ack){ ack(soul, key, val, state) } + root.on('put2', {put: {'#': soul, '.': key, ':': val, '>': state}, ack: function(err, ok){ + delete alls[id]; + all(err, ok); + }}); + } + function map(msg){ + var eve = this, root = eve.as, graph = root.graph, put = msg.put, soul = put['#'], key = put['.'], val = put[':'], state = put['>'], id = msg['#'], tmp; + graph[soul] = state_ify(graph[soul], key, state, val, soul); + debounce(root, soul, key,val, state); // TODO: This should NOT be how the API works, this should be done at an extension layer, but hacky solution to migrate with old code for now. + eve.to.next(msg); + } + function debounce(root, soul, key,val, state){ var tmp; + if(!(tmp = root.next[soul]) || !tmp.$){ return } + tmp.change = state_ify(tmp.change, key, state, val, soul); + tmp.put = state_ify(tmp.put, key, state, val, soul); + if(tmp.wait){ return } + tmp.wait = setTimeout(function(){ + var change = tmp.change; tmp.change = tmp.wait = null; + root.stop = this.stop; // temporary fix till a better solution? + tmp.on('in', {$: tmp.$, get: soul, put: change}); + root.stop = null; // temporary fix till a better solution? + },0); + } - var cut = function(s){ return " '"+(''+s).slice(0,9)+"...' " } var ERR = "Error: Invalid graph!"; + var cut = function(s){ return " '"+(''+s).slice(0,9)+"...' " } + var HAM = Gun.HAM, MD = 2147483647, State = Gun.state; }()); - var PUT = Gun.on.put2; - var HAM = Gun.HAM, MD = 2147483647, State = Gun.state; - function cache(msg){ - var eve = this, root = eve.as, graph = root.graph; - var put = msg.put, soul = put['#'], key = put['.'], val = put[':'], state = put['>']; - graph[soul] = state_ify(graph[soul], key, state, val, soul); - // trigger chains // trigger after all individual values cached? - if(eve.to.last === eve.to.next){ - console.log("THE END, reply back"); - return; - } - eve.to.next(msg); - } ;(function(){ Gun.on.put = function(msg, gun){ @@ -901,8 +913,8 @@ var faith = function(){}; faith.ram = faith.faith = true; // HNPERF: We're testing performance improvement by skipping going through security again, but this should be audited. root.on('in', { '@': msg['#'], - how: 'mem', put: node, + ram: 1, $: gun, _: faith }); @@ -1926,7 +1938,7 @@ } root.on('out', function(msg){ - if(msg.lS){ return } // TODO: for IndexedDB and others, shouldn't send to peers ACKs to our own GETs. + if(msg.lS){ return } // TODO: for IndexedDB and others, shouldn't send to peers ACKs to our own GETs. // THIS IS BLOCKING BROWSERS REPLYING TO REQUESTS, NO??? CHANGE THIS SOON!! UNDER CONTROLLED CIRCUMSTANCES!! Or maybe in-memory already doe sit? if(Gun.is(msg.$) && msg.put && !msg['@']){ id = msg['#']; Gun.graph.is(msg.put, null, map); @@ -2000,7 +2012,7 @@ data = Gun.state.to(data, has); } //if(!data && !Gun.obj.empty(opt.peers)){ return } // if data not found, don't ack if there are peers. // Hmm, what if we have peers but we are disconnected? - root.on('in', {'@': msg['#'], put: Gun.graph.node(data), how: 'lS', lS: msg.$});// || root.$}); + root.on('in', {'@': msg['#'], put: Gun.graph.node(data), lS:1});// || root.$}); }; Gun.debug? setTimeout(to,1) : to(); }); @@ -2062,7 +2074,7 @@ var m, i = 0, c = 100; // hardcoded for now? while(c-- && (m = msg[i++])){ mesh.hear(m, peer) } msg = msg.slice(i); // slicing after is faster than shifting during. - // TODO: consider triggering peer flush to response times speedier! + flush(peer); // TODO: consider triggering peer flush to response times speedier! if(!msg.length){ return } puff(go, 1); }()); @@ -2074,7 +2086,7 @@ if(!msg){ return } if(msg.DBG_s){ opt.log(+new Date - msg.DBG_s, 'to hear', msg['#']) } if(!(id = msg['#'])){ id = msg['#'] = Type.text.random(9) } - if(dup_check(id)){ return } + if(tmp = dup_check(id)){ return } /*if(!(hash = msg['##']) && u !== msg.put){ hash = msg['##'] = Type.obj.hash(msg.put) } if(hash && (tmp = msg['@'] || (msg.get && id))){ // Reduces backward daisy in case varying hashes at different daisy depths are the same. if(dup.check(tmp+hash)){ return } @@ -2082,7 +2094,7 @@ } if(tmp = msg['><']){ (msg._).to = Type.obj.map(tmp.split(','), tomap) } */ // TOOD: COME BACK TO THIS LATER!!! IMPORTANT MESH STUFF!! - (msg._ = function(){}).via = peer; + (msg._ = function(){}).via = mesh.leap = peer; if(tmp = msg.dam){ if(tmp = mesh.hear[tmp]){ tmp(msg, peer, root); @@ -2091,9 +2103,9 @@ return; } var S, ST; LOG && (S = +new Date); console.STAT = {}; - //root.on('in', msg); - root.on('in2', msg); - dup_track(id); + msg.put? root.on('in2', msg) : root.on('in', msg); // come on, pretty epic live upgrade path for tens of millions of users, no? Oh boy, this is a TEMPORARY hack, make sure future versions are stable. // REMEMBER!!! When you "finish" this, check all adapters, especially RAD, SEA, AXE, UDP & others!!! + dup_track(id).via = peer; + mesh.leap = null; // warning! mesh.leap could be buggy. if(LOG && !msg.nts && (ST = +new Date - S) > 9){ opt.log(S, ST, 'msg', msg['#'], JSON.stringify(console.STAT)) } return; } @@ -2113,20 +2125,20 @@ var S; LOG && (S = +new Date); //msg.DBG_s = msg.DBG_s || +new Date; var meta = msg._||(msg._=function(){}); if(!(id = msg['#'])){ id = msg['#'] = Type.text.random(9) } - if(!(hash = msg['##']) && u !== msg.put){ hash = msg['##'] = Type.obj.hash(msg.put) } + //if(!(hash = msg['##']) && u !== msg.put){ hash = msg['##'] = Type.obj.hash(msg.put) } if(!(raw = meta.raw)){ raw = mesh.raw(msg); - if(hash && (tmp = msg['@'])){ + /*if(hash && (tmp = msg['@'])){ dup.track(tmp+hash).it = it(msg); if(tmp = (dup.s[tmp]||ok).it){ if(hash === tmp['##']){ return false } tmp['##'] = hash; } - } + }*/ } //LOG && opt.log(S, +new Date - S, 'say prep'); - dup.track(id).it = it(msg); // track for 9 seconds, default. Earth<->Mars would need more! - if(!peer){ peer = (tmp = dup.s[msg['@']]) && (tmp = tmp.it) && (tmp = tmp._) && (tmp = tmp.via) } + dup_track(id);//.it = it(msg); // track for 9 seconds, default. Earth<->Mars would need more! + if(!peer){ peer = ((tmp = dup.s[msg['@']]) && (tmp.via || ((tmp = tmp.it) && (tmp = tmp._) && tmp.via))) || mesh.leap } // warning! mesh.leap could be buggy! if(!peer && msg['@']){ LOG && opt.log(+new Date, ++SMIA, 'total no peer to ack to'); return false; @@ -2141,39 +2153,42 @@ } if(!peer.wire && mesh.wire){ mesh.wire(peer) } if(id === peer.last){ return } peer.last = id; // was it just sent? - if(peer === meta.via){ return false } + if(peer === meta.via){ return false } // don't send back to self. if((tmp = meta.to) && (tmp[peer.url] || tmp[peer.pid] || tmp[peer.id]) /*&& !o*/){ return false } if(peer.batch){ peer.tail = (tmp = peer.tail || 0) + raw.length; if(peer.tail <= opt.pack){ - peer.batch.push(raw); // peer.batch += (tmp?'':',')+raw; // TODO: Prevent double JSON! // FOR v1.0 !? + //peer.batch.push(raw); + peer.batch += (tmp?',':'')+raw; // TODO: Prevent double JSON! // FOR v1.0 !? return; } flush(peer); } - peer.batch = []; // peer.batch = '['; // TODO: Prevent double JSON! + //peer.batch = []; + peer.batch = '['; // TODO: Prevent double JSON! var S, ST; LOG && (S = +new Date); setTimeout(function(){ LOG && (ST = +new Date - S) > 9 && opt.log(S, ST, '1ms TO'); - flush(peer) + flush(peer); }, opt.gap); send(raw, peer); } - function flush(peer){ - var tmp = peer.batch; // var tmp = peer.batch + ']'; // TODO: Prevent double JSON! - peer.batch = peer.tail = null; - if(!tmp){ return } - if(!tmp.length){ return } // if(3 > tmp.length){ return } // TODO: ^ - var S, ST; LOG && (S = +new Date); - try{tmp = (1 === tmp.length? tmp[0] : JSON.stringify(tmp)); - }catch(e){return opt.log('DAM JSON stringify error', e)} - LOG && (ST = +new Date - S) > 9 && opt.log(S, ST, 'say stringify', tmp.length); - if(!tmp){ return } - send(tmp, peer); - } mesh.say.c = mesh.say.d = 0; }()); - + + function flush(peer){ + var tmp = peer.batch, t = 'string' == typeof tmp, l; + if(t){ tmp += ']' }// TODO: Prevent double JSON! + peer.batch = peer.tail = null; + if(!tmp){ return } + if(t? 3 > tmp.length : !tmp.length){ return } // TODO: ^ + var S, ST; LOG && (S = +new Date); + if(!t){try{tmp = (1 === tmp.length? tmp[0] : JSON.stringify(tmp)); + }catch(e){return opt.log('DAM JSON stringify error', e)}} + LOG && (ST = +new Date - S) > 9 && opt.log(S, ST, 'say stringify', tmp.length); + if(!tmp){ return } + send(tmp, peer); + } // for now - find better place later. function send(raw, peer){ try{ var wire = peer.wire; @@ -2196,12 +2211,12 @@ if(!msg){ return '' } var meta = (msg._) || {}, put, hash, tmp; if(tmp = meta.raw){ return tmp } - if(typeof msg === 'string'){ return msg } - if(!msg.dam){ + if('string' == typeof msg){ return msg } + /*if(!msg.dam){ // TOOD: COME BACK TO THIS LATER!!! IMPORTANT MESH STUFF!! var i = 0, to = []; Type.obj.map(opt.peers, function(p){ to.push(p.url || p.pid || p.id); if(++i > 3){ return true } // limit server, fast fix, improve later! // For "tower" peer, MUST include 6 surrounding ids. // REDUCED THIS TO 3 for temporary relay peer performance, towers still should list neighbors. }); if(i > 1){ msg['><'] = to.join() } - } + }*/ // TOOD: COME BACK TO THIS LATER!!! IMPORTANT MESH STUFF!! var raw = $(msg); // optimize by reusing put = the JSON.stringify from .hash? /*if(u !== put){ tmp = raw.indexOf(_, raw.indexOf('put')); diff --git a/lib/multicast.js b/lib/multicast.js index d4023b22..5fcb1dd3 100644 --- a/lib/multicast.js +++ b/lib/multicast.js @@ -83,7 +83,7 @@ Gun.on('create', function(root){ } if((tmp = root.stats) && (tmp = tmp.gap) && info){ (tmp.near || (tmp.near = {}))[info.address] = info.port || 1 } // STATS! if(check.on || id === pid){ return } - root.on('out', check.on = say); + root.on('out', check.on = say); // TODO: MULTICAST NEEDS TO BE CHECKED FOR NEW CODE SYSTEM!!!!!!!!!! } setInterval(check, 1000 * 1); diff --git a/lib/radiskip.js b/lib/radiskip.js index 90ec0a47..325bbf3c 100644 --- a/lib/radiskip.js +++ b/lib/radiskip.js @@ -43,21 +43,19 @@ var r = function(key, data, cb){ if('function' === typeof data){ var o = cb || {}; - cb = val; + cb = data; r.read(key, cb); return; } //var tmp = (tmp = r.batch = r.batch || {})[key] = tmp[key] || {}; //var tmp = (tmp = r.batch = r.batch || {})[key] = data; - //console.only(1, key, data); r.save(key, data, cb); } r.save = function(key, data, cb){ var s = {key: key}; s.find = function(file){ var tmp; - //console.only(4, 'find', file); s.file = file || (file = opt.code.from); - if(tmp = r.disk[file]){ s.mix(null, tmp); return } + if(tmp = r.disk[file]){ s.mix(u, tmp); return } r.parse(file, s.mix); } s.mix = function(err, disk){ @@ -74,9 +72,7 @@ if(u === data){ cb(err, -1); return } } (s.disk = disk)(key, data); - //console.only(7, key, data, disk.$, disk.Q); if(disk.Q){ disk.Q.push(cb); return } disk.Q = [cb]; - //console.only(6, key, data, disk.$); disk.to = setTimeout(s.write, opt.until); } s.write = function(){ @@ -84,7 +80,6 @@ s.q = disk.Q; delete disk.Q; delete r.disk[file]; - //console.only(8, s.file, q.length); r.write(file, disk, s.ack); } s.ack = function(err, ok){ @@ -112,14 +107,14 @@ f.file = file = rad.file || (rad.file = file); if(!file){ cb('What file?'); return } f.write = function(){ - var S; LOG && (S = +new Date); - var text = f.text; + var text = rad.raw = f.text; r.disk[file = rad.file || f.file || file] = rad; + var S; LOG && (S = +new Date); r.find.add(file, function add(err){ if(err){ cb(err); return } opt.store.put(ename(file), text, function safe(err, ok){ LOG && opt.log(S, ST = +new Date - S, "wrote disk", JSON.stringify(file), ++RWC, 'total all writes.'); - cb(err, ok); + cb(err, ok || 1); if(!rad.Q){ delete r.disk[file] } // VERY IMPORTANT! Clean up memory, but not if there is already queued writes on it! }); }); @@ -215,7 +210,7 @@ } Q[g.file] = [{key: key, ack: cb, file: g.file, opt: o}]; if(!g.file){ - g.it(null, u, {}); + g.it(u, u, {}); return true; } r.parse(g.file, g.check); @@ -310,7 +305,7 @@ var g = {key: key}; g.find = function(file){ var tmp; g.file = file || (file = opt.code.from); // this may not be true for reads? Hit "end of dir"? - if(tmp = r.disk[file]){ g.check(null, tmp); return } + if(tmp = r.disk[file]){ g.check(u, tmp); return } r.parse(file, g.check); } g.get = function(err, disk, info){ @@ -322,11 +317,13 @@ return; } disk = r.disk[file] || (r.disk[file] = disk); - if(!disk){ cb(null); return } + if(!disk){ cb(); return } disk.file || (disk.file = file); // ---------------------------- + info = info || {}; var data = disk(key); - if(u !== data){ cb(null, data); return } + info.atom = disk.atom; + if(u !== data){ cb(u, data, info); return } // ---------------------------- return; var S; LOG && (S = +new Date); @@ -350,7 +347,6 @@ o.next = as.file; r.read(key, as.ack, o); } - r.find(key, g.find); g.check = function(err, disk, info){ g.get(err, disk, info); (info || (info = {})).file || (info.file = g.file); @@ -367,6 +363,7 @@ }) }); } + r.find(key, g.find); } function rev(a,b){ return b } var revo = {reverse: true}; @@ -389,7 +386,6 @@ p.read = function(err, data){ var tmp; LOG && opt.log(S, +new Date - S, 'read disk', JSON.stringify(file), ++RPC, 'total all parses.'); delete Q[file]; - //console.only(5, 'parse', file, data, q.length); if((p.err = err) || (p.not = !data)){ map(q, p.ack); return } if('string' !== typeof data){ try{ @@ -467,7 +463,8 @@ return l; } var S; LOG && (S = +new Date); - if(raw){ return p.read(null, raw) } + if(r.disk){ raw || (raw = (r.disk[file]||'').raw) } + if(raw){ return p.read(u, raw) } opt.store.get(ename(file), p.read); } }()); @@ -477,7 +474,6 @@ r.find = function(key, cb){ if(!dir){ if(Q){ Q.push([key, cb]); return } Q = [[key, cb]]; - //console.only(2, 'find', key); r.parse(f, init); return; } @@ -509,7 +505,6 @@ return; } if(disk){ drain(disk); return } - //console.only(3, 'init', err, disk); dir = dir || disk || Radix(); if(!opt.store.list){ drain(dir); return } // import directory. diff --git a/lib/radix.js b/lib/radix.js index ffd13c72..f33bc4b3 100644 --- a/lib/radix.js +++ b/lib/radix.js @@ -2,12 +2,13 @@ function Radix(){ var radix = function(key, val, t){ + radix.atom = 0; if(!t && u !== val){ radix.last = (''+key < radix.last)? radix.last : ''+key; delete (radix.$||{})[_]; } t = t || radix.$ || (radix.$ = {}); - if(!key && Object.keys(t).length){ return t } + if(!key && Object.keys(t).length){ console.log('wat?', t, key); return t } key = ''+key; var i = 0, l = key.length-1, k = key[i], at, tmp; while(!(at = t[k]) && i < l){ @@ -46,7 +47,8 @@ } } else if(i == l){ - if(u === val){ return (u === (tmp = at['']))? at : tmp } + //if(u === val){ return (u === (tmp = at['']))? at : tmp } // THIS CODE IS CORRECT, below is + if(u === val){ return (u === (tmp = at['']))? at : ((radix.atom = 1) && tmp) } // temporary help?? at[''] = val; //(at[_] = function $(){ $.sort = Object.keys(at).sort(); return $ }()); } else { diff --git a/lib/rfs.js b/lib/rfs.js index 8a2e805b..705ae88a 100644 --- a/lib/rfs.js +++ b/lib/rfs.js @@ -10,20 +10,24 @@ function Store(opt){ return Store[opt.file]; } Store[opt.file] = store; + var puts = {}; store.put = function(file, data, cb){ + puts[file] = data; var random = Math.random().toString(36).slice(-3); var tmp = opt.file+'-'+file+'-'+random+'.tmp'; fs.writeFile(tmp, data, function(err, ok){ + delete puts[file]; if(err){ return cb(err) } move(tmp, opt.file+'/'+file, cb); }); }; - store.get = function(file, cb){ + store.get = function(file, cb){ var tmp; + if(tmp = puts[file]){ cb(u, tmp); return } fs.readFile(opt.file+'/'+file, function(err, data){ if(err){ if('ENOENT' === (err.code||'').toUpperCase()){ - return cb(null); + return cb(); } opt.log("ERROR:", err) } diff --git a/lib/rs3.js b/lib/rs3.js index 2ad03e8b..25a5ad19 100644 --- a/lib/rs3.js +++ b/lib/rs3.js @@ -52,24 +52,25 @@ function Store(opt){ store.put = function(file, data, cb){ var params = {Bucket: opts.bucket, Key: file, Body: data}; //console.log("RS3 PUT ---->", (data||"").slice(0,20)); - Gun.obj.del(c.g, file); - Gun.obj.del(c.l, 1); - s3.putObject(params, cb); + c.p[file] = data; + delete c.g[file];//Gun.obj.del(c.g, file); + delete c.l[1];//Gun.obj.del(c.l, 1); + s3.putObject(params, function(err, ok){ + delete c.p[file]; + cb(err, 's3'); + }); }; - store.get = function(file, cb){ - if(c.g[file]){ return c.g[file].push(cb) } + store.get = function(file, cb){ var tmp; + if(tmp = c.p[file]){ cb(u, tmp); return } + if(tmp = c.g[file]){ tmp.push(cb); return } var cbs = c.g[file] = [cb]; var params = {Bucket: opts.bucket, Key: file||''}; //console.log("RS3 GET ---->", file); - s3.getObject(params, function(err, ack){ + s3.getObject(params, function got(err, ack){ //console.log("RS3 GOT <----", err, file, cbs.length, ((ack||{}).Body||'').toString().slice(0,20)); - Gun.obj.del(c.g, file); - var data, cbe = function(cb){ - if(!ack){ cb(null); return; } - cb(err, data); - }; - data = (ack||{}).Body; //if(data = (ack||{}).Body){ data = data.toString() } - Gun.obj.map(cbs, cbe); + delete c.g[file];//Gun.obj.del(c.g, file); + var data, data = (ack||'').Body; + var i = 0, cba; while(cba = cbs[i++]){ cba && cba(err, data) }//Gun.obj.map(cbs, cbe); }); }; store.list = function(cb, match, params, cbs){ diff --git a/lib/store.js b/lib/store.js index 77d19468..1ca48d89 100644 --- a/lib/store.js +++ b/lib/store.js @@ -17,23 +17,29 @@ Gun.on('create', function(root){ root.on('put2', function(msg){ this.to.next(msg); var id = msg['#'], put = msg.put, soul = put['#'], key = put['.'], val = put[':'], state = put['>'], tmp; - dare(soul+key, {':': val, '>': state}, function(err, ok){ - console.log("STORE PAT:", soul, key, val, id, err, ok); + tmp = soul+esc+key; // soul+key; // be nice to move away from escaping + dare(tmp, {':': val, '>': state}, function(err, ok){ + if(msg.ack){ msg.ack(err, ok || 1) } return; // REVISE THIS IN FUTURE!!! + if(err){ root.on('in', {'@': id, err: err}); return } + root.on('in', {'@': id, ok: ok}); }); }); root.on('put', function(msg){ this.to.next(msg); var id = msg['#'] || Gun.text.random(3), track = !msg['@'], acks = track? 0 : u; // only ack non-acks. - var _ = (msg._||empty), got = _.rad; + var _ = (msg._||''), got = _.rad; + if(got){ return } // RAD's own ACKs to GETs do not need to be written to disk again. if(_.ram){ return } // in-memory ACKs to GETs do not need to be written to disk again. + root.on('in', {'@': msg['#'], err: 'Migration not done, please report this to & complain at Mark in http://chat.gun.eco !'}); + return; var S = (+new Date), C = 0; // STATS! var now = Gun.state(); Gun.graph.is(msg.put, null, function(val, key, node, soul){ if(!track && got){ - var at = (root.next||empty)[soul]; + var at = (root.next||'')[soul]; if(!at){ return } - if(u !== got['.']){ at = (at.next||empty)[key] } + if(u !== got['.']){ at = (at.next||'')[key] } if(!at){ return } at.rad = now; return; @@ -91,26 +97,32 @@ Gun.on('create', function(root){ o.limit = (tmp <= (o.pack || (1000 * 100)))? tmp : 1; } if(has['-'] || (soul||{})['-']){ o.reverse = true } - if((tmp = (root.next||empty)[soul]) && tmp.put){ - var SPUT = tmp.put; + if((tmp = (root.next||'')[soul]) && tmp.put){ if(o.atom){ - tmp = (tmp.next||empty)[o.atom] ; + tmp = (tmp.next||'')[o.atom] ; if(tmp && tmp.rad){ return } } else if(tmp && tmp.rad){ return } } + var now = Gun.state(); var S = (+new Date), C = 0; // STATS! - dare(key, function(err, data, o){ - console.log("STORE GOT:", err, data, o); - }, o); - rad(key||'', function(err, data, o){ + //rad(key||'', function(err, data, o){ + dare(key||'', function(err, data, info){ try{opt.store.stats.get.time[statg % 50] = (+new Date) - S; ++statg; opt.store.stats.get.count++; if(err){ opt.store.stats.get.err = err } }catch(e){} // STATS! - //if(u === data && o.chunks > 1){ return } // if we already sent a chunk, ignore ending empty responses. // this causes tests to fail. + //if(u === data && info.chunks > 1){ return } // if we already sent a chunk, ignore ending empty responses. // this causes tests to fail. LOG && Gun.log(S, +new Date - S, 'got', JSON.stringify(key)); S = +new Date; - if(data){ + info = info || {}; + var va, ve; + if(info.atom && data && u !== (va = data[':']) && u !== (ve = data['>'])){ // new format + var tmp = key.split(esc), so = tmp[0], ha = tmp[1]; + (graph = graph || {})[so] = Gun.state.ify(graph[so], ha, ve, va, so); + root.$.get(so).get(ha)._.rad = now; + // REMEMBER TO ADD _rad TO NODE/SOUL QUERY! + } else + if(data){ // old code path if(typeof data !== 'string'){ if(o.atom){ data = u; @@ -122,8 +134,9 @@ Gun.on('create', function(root){ } console.STAT && (console.STAT.radgetcount = C); if(LOG && (ST = +new Date - S) > 9){ Gun.log(S, ST, 'got prep time'); Gun.log(S, C, 'got prep #') } C = 0; S = +new Date; + //console.log("STORE GOT:", graph); var faith = function(){}; faith.faith = true; faith.rad = get; // HNPERF: We're testing performance improvement by skipping going through security again, but this should be audited. - root.on('in', {'@': id, put: graph, '%': o.more? 1 : u, err: err? err : u, _: faith}); + root.on('in', {'@': id, put: graph, '%': info.more? 1 : u, err: err? err : u, _: faith}); LOG && (ST = +new Date - S) > 9 && Gun.log(S, ST, 'got emit', Object.keys(graph||{}).length); graph = u; // each is outside our scope, we have to reset graph to nothing! }, o); @@ -135,15 +148,22 @@ Gun.on('create', function(root){ has = (key+has).split(esc); var soul = has.slice(0,1)[0]; has = has.slice(-1)[0]; + if(o.limit && o.limit <= o.count){ return true } + var va, ve, so = soul, ha = has; + if((va = val[':']) && (ve = val['>'])){ // THIS HANDLES NEW CODE! + (graph = graph || {})[so] = Gun.state.ify(graph[so], ha, ve, va, so); + o.count = (o.count || 0) + ((va||'').length || 9); + return; + } o.count = (o.count || 0) + val.length; var tmp = val.lastIndexOf('>'); var state = Radisk.decode(val.slice(tmp+1), null, esc); val = Radisk.decode(val.slice(0,tmp), null, esc); (graph = graph || {})[soul] = Gun.state.ify(graph[soul], has, state, val, soul); - if(o.limit && o.limit <= o.count){ return true } } LOG = console.LOG; }); + var val_is = Gun.val.is opt.store.stats = {get:{time:{}, count:0}, put: {time:{}, count:0}}; // STATS! var statg = 0, statp = 0; // STATS! }); \ No newline at end of file diff --git a/package.json b/package.json index 1f7cdc09..e10efe53 100644 --- a/package.json +++ b/package.json @@ -7,7 +7,8 @@ "ios": "browser.ios.js", "android": "browser.android.js", "scripts": { - "start": "node examples/http.js", + "start": "node --prof examples/http.js", + "debug": "node --prof-process --preprocess -j isolate*.log > v8.json && rm isolate*.log && echo 'drag & drop ./v8.json into https://mapbox.github.io/flamebearer/'", "https": "HTTPS_KEY=test/https/server.key HTTPS_CERT=test/https/server.crt npm start", "prepublishOnly": "npm run unbuild", "test": "mocha", diff --git a/sea.js b/sea.js index bd8dbd9c..3730fc21 100644 --- a/sea.js +++ b/sea.js @@ -1151,10 +1151,11 @@ } var u; - function check(msg){ + function check(msg){ // REVISE / IMPROVE, NO NEED TO PASS MSG/EVE EACH SUB? var eve = this, at = eve.as, put = msg.put, soul = put['#'], key = put['.'], val = put[':'], state = put['>'], id = msg['#'], tmp; //console.log("security check", msg); - var no = function(why){ at.on('in', {'@': id, err: why}) } + //var no = function(why){ at.on('in', {'@': id, err: why}) } + var no = function(why){ msg.ack(why) } if('#' === soul){ // special case for content addressing immutable hashed data. check.hash(eve, msg, val, key, soul, at, no); return; } @@ -1167,7 +1168,7 @@ if('~' === soul.slice(0,1) && 2 === (tmp = soul.slice(1)).split('.').length){ // special case, account data for a public key. check.pub(eve, msg, val, key, soul, at, no, (msg._||'').user, tmp); return; } - check.any(eve, msg, val, key, soul, at, no, (msg._||'').user); return; + check.any(eve, msg, val, key, soul, at, no, (msg._||'noop').user); return; eve.to.next(msg); // not handled } check.hash = function(eve, msg, val, key, soul, at, no){ @@ -1191,7 +1192,6 @@ if(val === pub){ return eve.to.next(msg) } // the account MUST match `pub` property that equals the ID of the public key. return no("Account not same!"); } - check['user'+soul+key] = 1; if(Gun.is(msg.$) && user && user.is && pub === user.is.pub){ SEA.sign(msg.put, (user._).sea, function(data){ var rel; if(u === data){ return no(SEA.err || 'Signature fail.') } From 0936c326f8031c5fcd9426b80a7c63ac3ffc6eac Mon Sep 17 00:00:00 2001 From: Mark Nadal Date: Mon, 10 Feb 2020 15:13:37 -0800 Subject: [PATCH 30/76] unit tests passing --- gun.js | 86 ++++-- lib/radisk.js | 559 ++++++++++++++------------------------- lib/radiskip.js | 600 ------------------------------------------ lib/radix.js | 6 +- lib/rfs.js | 3 +- lib/store.js | 25 +- sea.js | 87 +++--- test/common.js | 193 ++++++-------- test/mocha.html | 7 +- test/rad/bench.js | 2 +- test/rad/browser.html | 3 +- test/rad/rad.js | 14 +- test/sea/sea.js | 38 ++- 13 files changed, 443 insertions(+), 1180 deletions(-) delete mode 100644 lib/radiskip.js diff --git a/gun.js b/gun.js index ece88877..fdced091 100644 --- a/gun.js +++ b/gun.js @@ -97,9 +97,9 @@ return !o? o : JSON.parse(JSON.stringify(o)); // is shockingly faster than anything else, and our data has to be a subset of JSON anyways! } ;(function(){ - function empty(v,i){ var n = this.n; + function empty(v,i){ var n = this.n, u; if(n && (i === n || (obj_is(n) && obj_has(n, i)))){ return } - if(i){ return true } + if(u !== i){ return true } } Type.obj.empty = function(o, n){ if(!o){ return true } @@ -115,20 +115,21 @@ } t.r = t.r || []; t.r.push(k); }; - var keys = Object.keys, map; + var keys = Object.keys, map, u; Object.keys = Object.keys || function(o){ return map(o, function(v,k,t){t(k)}) } Type.obj.map = map = function(l, c, _){ - var u, i = 0, x, r, ll, lle, f = fn_is(c); - t.r = null; + var u, i = 0, x, r, ll, lle, f = 'function' == typeof c; + t.r = u; if(keys && obj_is(l)){ ll = keys(l); lle = true; } + _ = _ || {}; if(list_is(l) || ll){ x = (ll || l).length; for(;i < x; i++){ var ii = (i + Type.list.index); if(f){ - r = lle? c.call(_ || this, l[ll[i]], ll[i], t) : c.call(_ || this, l[i], ii, t); + r = lle? c.call(_, l[ll[i]], ll[i], t) : c.call(_, l[i], ii, t); if(r !== u){ return r } } else { //if(Type.test.is(c,l[i])){ return ii } // should implement deep equality testing! @@ -171,7 +172,7 @@ tmp.next(arg); }} }}); - if(arg instanceof Function){ + if('function' == typeof arg){ var be = { off: onto.off || (onto.off = function(){ @@ -323,7 +324,7 @@ Node.ify = function(obj, o, as){ // returns a node from a shallow object. if(!o){ o = {} } else if(typeof o === 'string'){ o = {soul: o} } - else if(o instanceof Function){ o = {map: o} } + else if('function' == typeof o){ o = {map: o} } if(o.map){ o.node = o.map.call(as, obj, u, o.node || {}) } if(o.node = Node.soul.ify(o.node || {}, o)){ obj_map(obj, map, {o:o,as:as}); @@ -464,9 +465,13 @@ if(typeof env === 'string'){ env = {soul: env}; } else - if(env instanceof Function){ + if('function' == typeof env){ env.map = env; } + if(typeof as === 'string'){ + env.soul = env.soul || as; + as = u; + } if(env.soul){ at.link = Val.link.ify(env.soul); } @@ -681,7 +686,8 @@ at.on('in', root, at); at.on('in2', root2, at); at.on('put2', map, at); - at.on('out', root, {at: at, out: root}); + //at.on('out', root, {at: at, out: root}); + at.on('out', root2, at); Gun.on('create', at); at.on('create', at); } @@ -704,10 +710,10 @@ if(tmp = msg['@']){ dup.track(tmp) } // HNPERF: Bump original request's liveliness. if(!at.ask(tmp, msg)){ if(msg.get){ - Gun.on.get(msg, gun); //at.on('get', get(msg)); + Gun.on._get(msg, gun); //at.on('get', get(msg)); } if(msg.put){ - Gun.on.put(msg, gun); //at.on('put', put(msg)); + //Gun.on._put(msg, gun); //at.on('put', put(msg)); } } ev.to.next(msg); @@ -718,6 +724,7 @@ } function root2(msg){ if(!msg){ return } + if(msg.out === root2 || msg.out === root){ this.to.next(msg); return } var eve = this, as = eve.as, at = as.at || as, gun = at.$, dup = at.dup, tmp; if(!(tmp = msg['#'])){ tmp = msg['#'] = text_rand(9) } if(dup.check(tmp)){ return } dup.track(tmp); @@ -741,15 +748,16 @@ root.on('in', {'@': id, ok: ok || 1}); id = u; }; + var set = root.set || (root.set = {'':1}); all.err = obj_map(put, valid, msg); ctx.node = ctx.state = u; mid = msg = ctx = u; - all(); - } + all(); // if synchronous + fire(root, ''); // if synchronous + } Gun.on.put = put; function valid(node, soul){ if(!node){ return ERR+cut(soul)+"no node." } var ctx = this._, tmp; - (ctx.root.opt||'').super && ctx.root.$.get(soul); // I think we need super for now, but since we are rewriting, should consider getting rid of it. if(!(tmp = node._)){ return ERR+cut(soul)+"no meta." } ctx.node = node; if(soul !== tmp[_soul]){ return ERR+cut(soul)+"soul not same." } @@ -769,6 +777,7 @@ var vertex = graph[soul] || empty, was = state_is(vertex, key, 1), known = vertex[key]; var machine = State(), is = HAM(machine, state, was, val, known), u; (alls = all.s || (all.s = {}))[id] = 1; + (root.set || (root.set = {}))[id] = 1; // tmp code; if(!is.incoming){ if(is.defer){ var to = is.defer - machine; @@ -789,29 +798,46 @@ function map(msg){ var eve = this, root = eve.as, graph = root.graph, put = msg.put, soul = put['#'], key = put['.'], val = put[':'], state = put['>'], id = msg['#'], tmp; graph[soul] = state_ify(graph[soul], key, state, val, soul); - debounce(root, soul, key,val, state); // TODO: This should NOT be how the API works, this should be done at an extension layer, but hacky solution to migrate with old code for now. + chain(root, soul, key, (u !== (tmp = put['=']))? tmp : val, state); // TODO: This should NOT be how the API works, this should be done at an extension layer, but hacky solution to migrate with old code for now. + fire(root, soul+key); // ^ eve.to.next(msg); } - function debounce(root, soul, key,val, state){ var tmp; - if(!(tmp = root.next[soul]) || !tmp.$){ return } - tmp.change = state_ify(tmp.change, key, state, val, soul); + function chain(root, soul, key,val, state){ var tmp, put; + (root.opt||'').super && root.$.get(soul); // I think we need super for now, but since we are rewriting, should consider getting rid of it. + if(!root || !(tmp = root.next) || !(tmp = tmp[soul]) || !tmp.$){ return } + (put = root.put || (root.put = {}))[soul] = state_ify(put[soul], key, state, val, soul); tmp.put = state_ify(tmp.put, key, state, val, soul); + return; + tmp.change = state_ify(tmp.change, key, state, val, soul); if(tmp.wait){ return } + var stop = {}; tmp.wait = setTimeout(function(){ var change = tmp.change; tmp.change = tmp.wait = null; - root.stop = this.stop; // temporary fix till a better solution? + root.stop = stop; // temporary fix till a better solution? tmp.on('in', {$: tmp.$, get: soul, put: change}); root.stop = null; // temporary fix till a better solution? },0); } + function fire(root, id){ var set; + if(set = root.set){ delete set[id] } + if(!obj_empty(set)){ return } + var stop = {}; + var next = root.next||'', put = root.put; root.put = root.set = null; + Gun.graph.is(put, function(node,soul){ var tmp; + if(!(tmp = next[soul]) || !tmp.$){ return } + root.stop = stop; // temporary fix till a better solution? + tmp.on('in', {$: tmp.$, get: soul, put: node}); + root.stop = null; // temporary fix till a better solution? + }) + } var ERR = "Error: Invalid graph!"; var cut = function(s){ return " '"+(''+s).slice(0,9)+"...' " } var HAM = Gun.HAM, MD = 2147483647, State = Gun.state; }()); ;(function(){ - Gun.on.put = function(msg, gun){ + Gun.on._put = function(msg, gun){ var at = gun._, ctx = {$: gun, graph: at.graph, put: {}, map: {}, souls: {}, machine: Gun.state(), ack: msg['@'], cat: at, stop: {}}; if(!Gun.obj.map(msg.put, perf, ctx)){ return } // HNPERF: performance test, not core code, do not port. if(!Gun.graph.is(msg.put, null, verify, ctx)){ ctx.err = "Error: Invalid graph!" } @@ -821,7 +847,7 @@ if(u !== ctx.defer){ var to = ctx.defer - ctx.machine; setTimeout(function(){ - Gun.on.put(msg, gun); + Gun.on._put(msg, gun); }, to > MD? MD : to ); // setTimeout Max Defer 32bit :( } if(!ctx.diff){ return } @@ -894,7 +920,7 @@ } function perf(node, soul){ if(node !== this.graph[soul]){ return true } } // HNPERF: do not port! - Gun.on.get = function(msg, gun){ + Gun.on._get = function(msg, gun){ var root = gun._, get = msg.get, soul = get[_soul], node = root.graph[soul], has = get[_has], tmp; var next = root.next || (root.next = {}), at = next[soul]; // queue concurrent GETs? @@ -1008,7 +1034,7 @@ } return; } - if(n instanceof Function){ + if('function' == typeof n){ var yes, tmp = {back: at}; while((tmp = tmp.back) && u === (yes = n(tmp, opt))){} @@ -1081,6 +1107,12 @@ put._ = meta; back.on('in', {$: back.$, put: put, get: back.get}) } + if(tmp = at.lex){ + tmp = (tmp._) || (tmp._ = function(){}); + if(back.ack < tmp.ask){ tmp.ask = back.ack } + if(tmp.ask){ return } + tmp.ask = 1; + } } root.ask(ack, msg); return root.on('in', msg); @@ -1311,7 +1343,7 @@ at.on('in', {get: at.get, put: Gun.val.link.ify(get['#']), $: at.$, '@': msg['@']}); return; } - Gun.on.put(msg, at.root.$); + Gun.on._put(msg, at.root.$); } var empty = {}, u; var obj = Gun.obj, obj_has = obj.has, obj_put = obj.put, obj_del = obj.del, obj_to = obj.to, obj_map = obj.map; @@ -1331,7 +1363,7 @@ } gun = gun.$; } else - if(key instanceof Function){ + if('function' == typeof key){ if(true === cb){ return soul(this, key, cb, as), this } gun = this; var at = gun._, root = at.root, tmp = root.now, ev; @@ -1367,7 +1399,7 @@ if(tmp = this._.stun){ // TODO: Refactor? gun._.stun = gun._.stun || tmp; } - if(cb && cb instanceof Function){ + if(cb && 'function' == typeof cb){ gun.get(cb, as); } return gun; diff --git a/lib/radisk.js b/lib/radisk.js index 2a1fa2a6..712b5b7c 100644 --- a/lib/radisk.js +++ b/lib/radisk.js @@ -40,159 +40,57 @@ 1. Because writing to disk takes time, we should batch data to disk. This improves performance, and reduces potential disk corruption. 2. If a batch exceeds a certain number of writes, we should immediately write to disk when physically possible. This caps total performance, but reduces potential loss. */ - var r = function(key, val, cb){ - key = ''+key; - if(val instanceof Function){ + var r = function(key, data, cb){ + if('function' === typeof data){ var o = cb || {}; - cb = val; - var S; LOG && (S = +new Date); - val = r.batch(key); - LOG && (ST = +new Date - S) > 9 && opt.log(S, ST, 'rad mem'); - if(u !== val){ - cb(u, r.range(val, o), o); - if(atomic(val)){ return } - // if a node is requested and some of it is cached... the other parts might not be. - } - if(r.thrash.at){ - val = r.thrash.at(key); - if(u !== val){ - cb(u, r.range(val, o), o); - if(atomic(val)){ cb(u, val, o); return } - // if a node is requested and some of it is cached... the other parts might not be. - } - } - return r.read(key, cb, o); - } - r.batch(key, val); - if(cb){ r.batch.acks.push(cb) } - if(++r.batch.ed >= opt.batch){ return r.thrash() } // (2) - if(r.batch.to){ return } - //clearTimeout(r.batch.to); // (1) // THIS LINE IS EVIL! NEVER USE IT! ALSO NEVER DELETE THIS SO WE NEVER MAKE THE SAME MISTAKE AGAIN! - r.batch.to = setTimeout(r.thrash, opt.until || 1); - } - - r.batch = Radix(); - r.batch.acks = []; - r.batch.ed = 0; - - r.thrash = function(){ - var thrash = r.thrash; - if(thrash.ing){ return thrash.more = true } - LOG = console.LOG; // dirty place to cheaply update LOG settings over time. - thrash.more = false; - thrash.ing = true; - var batch = thrash.at = r.batch, i = 0; - clearTimeout(r.batch.to); - r.batch = null; - r.batch = Radix(); - r.batch.acks = []; - r.batch.ed = 0; - //console.debug(99); var ID = Gun.text.random(2), S = (+new Date); console.log("[[[[[[[[", ID, batch.acks.length); - r.save(batch, function(err, ok){ - if(++i > 1){ opt.log('RAD ERR: Radisk has callbacked multiple times, please report this as a BUG at github.com/amark/gun/issues ! ' + i); return } - if(err){ opt.log('err', err) } - //console.debug(99); var TMP; console.log("]]]]]]]]", ID, batch.acks.length, (TMP = +new Date) - S, 'more?', thrash.more); - map(batch.acks, function(cb){ cb(err, ok) }); - //console.log("][", +new Date - TMP, thrash.more); - thrash.at = null; - thrash.ing = false; - if(thrash.more){ thrash() } - }); - } - - /* - 1. Find the first radix item in memory. - 2. Use that as the starting index in the directory of files. - 3. Find the first file that is lexically larger than it, - 4. Read the previous file to that into memory - 5. Scan through the in memory radix for all values lexically less than the limit. - 6. Merge and write all of those to the in-memory file and back to disk. - 7. If file too large, split. More details needed here. - */ - /* NEW APPROACH: - 1. For each item in radix memory - 2. Add it to a radix bucket corresponding to directory of files - 3. Iterate over each bucket - 4. Resume old approach. - */ - r.save = function(rad, cb){ - if(r.save.ing){ - r.save.ing.push({rad: rad, ack: cb}); + cb = data; + r.read(key, cb, o); return; } - //console.only(99); var ID = Gun.text.random(2), S = (+new Date); console.log("[[[[[[[[", ID); - r.save.ing = []; - var ack = cb; - var s = function Span(err, ok){ - var tmp = r.save.ing; - //console.only(99); var TMP; console.log("]]]]]]]]", ID, (TMP = +new Date) - S, 'more?', !!tmp); - r.save.ing = null; - map(tmp, function(q){ // if many, not the most efficient to requeue, but works for now. - if(!q || !q.rad || !q.ack){ return } - r.save(q.rad, q.ack); - }) - ack(err, ok); - }; - cb = s; - s.files = {}; - s.i = 0; // TODO: revise? Using counter for critical path not my favorite. - s.place = function(tree, key){ - var go = function(file, last){ - file = decodeURIComponent(file || last || opt.code.from); - (s.files[file] || (s.files[file] = Radix()))(key, tree); - if(!(--s.i)){ s.go() } // TODO: See above, revise? - return true; - } - go.reverse = 1; - go.end = key; - r.list(go); - ++s.i; // TODO: See above, revise? - } - s.go = function(){ - if(s.gone){ return } s.gone = true; - s.seq = []; - map(s.files, function(mem, file){ s.seq.push({file: file, mem: mem}) }); - LOG && opt.log(+new Date, s.seq.length, "rad files #"); - s.files = null; - s.c = 0; - s.merge(s.c); - } - s.merge = function(i){ - i = i || 0; - //var at = s.seq[i]; - var at = s.seq.shift(); - if(!at){ - if(s.ok){ return cb(null, s.ok) } - return cb("No file to save data to."); - } - var file = at.file, mem = at.mem; - r.parse(file, function(err, disk){ - if(err){ return cb(err) } - if(!disk && file !== opt.code.from){ // corrupt file? - r.list.bad(file); // remove from dir list - r.save(rad, cb); // try again - return; - } - disk = disk || Radix(); - var S, C = 0; LOG && (S = +new Date); - Radix.map(mem, function(val, key){ - C++; - // PLUGIN: consider adding HAM as an extra layer of protection - disk(key, val); // merge batch[key] -> disk[key] - }); - LOG && opt.log(S, +new Date - S, "rad merge"); - LOG && opt.log(S, C, "rad merge #"); - r.write(file, disk, s.pop); - }) - } - s.pop = function(err, ok){ - if(s.err = err || s.err){ return cb(err) } - s.ok = ok || s.ok || 1; - s.merge(++s.c); - } - Radix.map(rad, s.place); - if(!s.i){ s.go() }; // TODO: See above, revise? + //var tmp = (tmp = r.batch = r.batch || {})[key] = tmp[key] || {}; + //var tmp = (tmp = r.batch = r.batch || {})[key] = data; + r.save(key, data, cb); } + r.save = function(key, data, cb){ + var s = {key: key}; + s.find = function(file){ var tmp; + s.file = file || (file = opt.code.from); + if(tmp = r.disk[file]){ s.mix(u, tmp); return } + r.parse(file, s.mix); + } + s.mix = function(err, disk){ + if(s.err = err || s.err){ cb(err); return } + var file = s.file = (disk||'').file || s.file; + if(!disk && file !== opt.code.from){ // corrupt file? + r.find.bad(file); // remove from dir list + r.save(key, data, cb); // try again + return; + } + (disk = r.disk[file] || (r.disk[file] = disk || Radix())).file || (disk.file = file); + if(opt.compare){ + data = opt.compare(disk(key), data, key, file); + if(u === data){ cb(err, -1); return } + } + (s.disk = disk)(key, data); + if(disk.Q){ disk.Q.push(cb); return } disk.Q = [cb]; + disk.to = setTimeout(s.write, opt.until); + } + s.write = function(){ + var file = s.file, disk = s.disk; + s.q = disk.Q; + delete disk.Q; + delete r.disk[file]; + r.write(file, disk, s.ack); + } + s.ack = function(err, ok){ + var q = s.q || [], i = 0, ack; + //var S = +new Date; + while(ack = q[i++]){ ack(err, ok) } + //console.log('acks:', +new Date - S, s.file, q.length); + } + r.find(key, s.find); + } + r.disk = {}; /* Any storage engine at some point will have to do a read in order to write. @@ -200,70 +98,77 @@ Therefore it is unavoidable that a read will have to happen, the question is just how long you delay it. */ + var RWC = 0; r.write = function(file, rad, cb, o){ + if(!rad){ cb('No radix!'); return } o = ('object' == typeof o)? o : {force: o}; - var f = function Fractal(){}; + var f = function Fractal(){}, a, b; f.text = ''; - f.count = 0; - f.file = file; - f.each = function(val, key, k, pre){ - //console.log("RAD:::", JSON.stringify([val, key, k, pre])); - if(u !== val){ f.count++ } - if(opt.pack <= (val||'').length){ return cb("Record too big!"), true } - var enc = Radisk.encode(pre.length) +'#'+ Radisk.encode(k) + (u === val? '' : ':'+ Radisk.encode(val)) +'\n'; - if((opt.chunk < f.text.length + enc.length) && (1 < f.count) && !o.force){ - f.text = ''; - f.limit = Math.ceil(f.count/2); - f.count = 0; - f.sub = Radix(); - // IMPORTANT: DO THIS IN REVERSE, SO LAST HALF OF DATA MOVED TO NEW FILE BEFORE DROPPING FROM CURRENT FILE. - Radix.map(rad, f.slice, {reverse: true}); - return true; - } - f.text += enc; - } + f.file = file = rad.file || (rad.file = file); + if(!file){ cb('What file?'); return } f.write = function(){ - var tmp = ename(file); + var text = rad.raw = f.text; + r.disk[file = rad.file || f.file || file] = rad; var S; LOG && (S = +new Date); - r.list.add(tmp, function(err){ - if(err){ return cb(err) } - //opt.store.put(tmp, f.text, cb); // revert to this after stats done below: - opt.store.put(tmp, f.text, function(err,ok){ - LOG && opt.log(S, ST = +new Date - S, "wrote disk", tmp); - cb(err,ok); + r.find.add(file, function add(err){ + if(err){ cb(err); return } + opt.store.put(ename(file), text, function safe(err, ok){ + LOG && opt.log(S, ST = +new Date - S, "wrote disk", JSON.stringify(file), ++RWC, 'total all writes.'); + cb(err, ok || 1); + if(!rad.Q){ delete r.disk[file] } // VERY IMPORTANT! Clean up memory, but not if there is already queued writes on it! }); }); } + f.split = function(){ + f.text = ''; + if(!f.count){ f.count = 0; + Radix.map(rad, function count(){ f.count++ }); // TODO: Perf? Any faster way to get total length? + } + f.limit = Math.ceil(f.count/2); + f.count = 0; + f.sub = Radix(); + Radix.map(rad, f.slice, {reverse: 1}); // IMPORTANT: DO THIS IN REVERSE, SO LAST HALF OF DATA MOVED TO NEW FILE BEFORE DROPPING FROM CURRENT FILE. + r.write(f.end, f.sub, f.both, o); + f.hub = Radix(); + Radix.map(rad, f.stop); + r.write(rad.file, f.hub, f.both, o); + return true; + } f.slice = function(val, key){ f.sub(f.end = key, val); - if(f.limit <= (++f.count)){ - r.write(key, f.sub, f.swap, o); - return true; - } - } - f.swap = function(err){ - if(err){ return cb(err) } - f.sub = Radix(); - Radix.map(rad, f.stop); - r.write(f.file, f.sub, cb, o); + if(f.limit <= (++f.count)){ return true } } f.stop = function(val, key){ if(key >= f.end){ return true } - f.sub(key, val); + f.hub(key, val); } - if(opt.jsonify){ return r.write.jsonify(f, file, rad, cb, o) } // temporary testing idea + f.both = function(err, ok){ + if(b){ cb(err || b); return } + if(a){ cb(err, ok); return } + a = true; + b = err; + } + f.each = function(val, key, k, pre){ + //console.log("RAD:::", JSON.stringify([val, key, k, pre])); + if(u !== val){ f.count++ } + if(opt.pack <= (val||'').length){ return cb("Data too big!"), true } + var enc = Radisk.encode(pre.length) +'#'+ Radisk.encode(k) + (u === val? '' : ':'+ Radisk.encode(val)) +'\n'; + if((opt.chunk < f.text.length + enc.length) && (1 < f.count) && !o.force){ + return f.split(); + } + f.text += enc; + } + if(opt.jsonify){ r.write.jsonify(f, rad, cb, o); return } // temporary testing idea if(!Radix.map(rad, f.each, true)){ f.write() } } - r.write.jsonify = function(f, file, rad, cb, o){ + r.write.jsonify = function(f, rad, cb, o){ var raw; var S; LOG && (S = +new Date); try{raw = JSON.stringify(rad.$); - }catch(e){ return cb("Record too big!") } + }catch(e){ cb("Cannot radisk!"); return } LOG && opt.log(S, +new Date - S, "rad stringified JSON"); - if(opt.chunk < raw.length && !o.force){ - if(Radix.map(rad, f.each, true)){ return } - } + if(opt.chunk < raw.length && !o.force){ return f.split() } f.text = raw; f.write(); } @@ -273,129 +178,68 @@ if(u === o.start && u === o.end){ return tree } if(atomic(tree)){ return tree } var sub = Radix(); - Radix.map(tree, function(v,k){ // ONLY PLACE THAT TAKES TREE, maybe reduce API for better perf? - sub(k,v); - }, o); + Radix.map(tree, function(v,k){ sub(k,v) }, o); // ONLY PLACE THAT TAKES TREE, maybe reduce API for better perf? return sub(''); } ;(function(){ - var Q = {}; r.read = function(key, cb, o){ o = o || {}; - if(RAD && !o.next){ // cache - var S; LOG && (S = +new Date); - var val = RAD(key); - LOG && (ST = +new Date - S) > 9 && opt.log(S, ST, 'rad cached'); - //if(u !== val){ - //cb(u, val, o); - if(atomic(val)){ cb(u, val, o); return } - // if a node is requested and some of it is cached... the other parts might not be. - //} + var g = {key: key}; + g.find = function(file){ var tmp; + g.file = file || (file = opt.code.from); + if(tmp = r.disk[g.file = file]){ g.check(u, tmp); return } + r.parse(file, g.check); } - o.span = (u !== o.start) || (u !== o.end); // is there a start or end? - var g = function Get(){}; - g.lex = function(file){ var tmp; // // TODO: this had a out-of-memory crash! - file = (u === file)? u : decodeURIComponent(file); - tmp = o.next || key || (o.reverse? o.end || '\uffff' : o.start || ''); - if(!file || (o.reverse? file < tmp : file > tmp)){ - LOG && opt.log(S, +new Date - S, 'rad read lex'); S = +new Date; - if(o.next || o.reverse){ g.file = file } - if(tmp = Q[g.file]){ - tmp.push({key: key, ack: cb, file: g.file, opt: o}); - return true; - } - Q[g.file] = [{key: key, ack: cb, file: g.file, opt: o}]; - if(!g.file){ - g.it(null, u, {}); - return true; - } - r.parse(g.file, g.check); - return true; - } - g.file = file; - } - g.it = function(err, disk, info){ - if(g.err = err){ opt.log('err', err) } - if(!disk && g.file){ // corrupt file? - r.list.bad(g.file); // remove from dir list - r.read(key, cb, o); // look again + g.get = function(err, disk, info){ + if(g.err = err || g.err){ cb(err); return } + var file = g.file = (disk||'').file || g.file; + if(!disk && file !== opt.code.from){ // corrupt file? + r.find.bad(file); // remove from dir list + r.read(key, cb, o); // try again return; } - g.info = info; - if(disk){ RAD = g.disk = disk } - disk = Q[g.file]; delete Q[g.file]; - map(disk, g.ack); - } - g.ack = function(as){ - if(!as.ack){ return } - var S; LOG && (S = +new Date); - var key = as.key, o = as.opt, info = g.info, rad = g.disk || noop, data = r.range(rad(key), o), last = rad.last || Radix.map(rad, rev, revo); - LOG && (ST = +new Date - S) > 9 && opt.log(S, ST, "rad range loaded"); - o.parsed = (o.parsed || 0) + (info.parsed||0); + disk = r.disk[file] || (r.disk[file] = disk); + if(!disk){ cb("No file!"); return } + disk.file || (disk.file = file); + var data = r.range(disk(key), o); + o.unit = disk.unit; o.chunks = (o.chunks || 0) + 1; - o.more = true; - if((!as.file) // if no more places to look - || (!o.span && last === key) // if our key exactly matches the very last atomic record - || (!o.span && last && last > key && 0 != last.indexOf(key)) // 'zach' may be lexically larger than 'za', but there still might be more, like 'zane' in the 'za' prefix bucket so do not end here. - ){ - o.more = u; - as.ack(g.err, data, o); - return + o.parsed = (o.parsed || 0) + ((info||'').parsed||(o.chunks*opt.chunk)); + o.more = 1; + o.next = u; + Radix.map(r.list, function next(v,f){ + if(!v || file === f){ return } + o.next = f; + return 1; + }, o.reverse? {reverse: 1, end: file} : {start: file}); + if(!o.next){ o.more = 0 } + if(o.next){ + if(!o.reverse && (key < o.next && 0 != o.next.indexOf(key)) || (u !== o.end && (o.end || '\uffff') < o.next)){ o.more = 0 } + if(o.reverse && (key > o.next && 0 != key.indexOf(o.next)) || (u !== o.start && (o.start || '') > o.next)){ o.more = 0 } } - if(u !== data){ - as.ack(g.err, data, o); // more might be coming! - if(o.parsed >= o.limit){ return } // even if more, we've hit our limit, asking peer will need to make a new ask with a new starting point. - } - o.next = as.file; - r.read(key, as.ack, o); + if(!o.more){ cb(g.err, data, o); return } + if(data){ cb(g.err, data, o) } + if(o.parsed >= o.limit){ return } + r.parse(o.next, g.check); } g.check = function(err, disk, info){ - g.it(err, disk, info); - var good = true; + g.get(err, disk, info); + (info || (info = {})).file || (info.file = g.file); Radix.map(disk, function(val, key){ - // assume in memory for now, since both write/read already call r.list which will init it. - var go = function(file){ - if(info.file !== file){ - good = false - } - return true; - } - go.reverse = 1; - go.end = key; - r.list(go); - }); - if(good){ return } - var id = Gun.text.random(3); - r.save(disk, function ack(err, ok){ - if(err){ return r.save(disk, ack) } // ad infinitum??? - console.log("MISLOCATED DATA CORRECTED", id); + // assume in memory for now, since both write/read already call r.find which will init it. + r.find(key, function(file){ + if((file || (file = opt.code.from)) === info.file){ return } + var id = Gun.text.random(3); + console.log("MISLOCATED DATA", id, key, info.file, file); + r.save(key, val, function ack(err, ok){ + if(err){ r.save(key, val, ack); return } // ad infinitum??? + console.log("MISLOCATED DATA CORRECTED", id); + }); + }) }); } - /*g.check2 = function(err, disk, info){ - if(err || !disk){ return g.it(err, disk, info) } - var good = true; - Radix.map(disk, function(val, key){ - // assume in memory for now, since both write/read already call r.list which will init it. - var go = function(file){ - if(info.file !== file){ good = false } - return true; - } - go.reverse = 1; - go.end = key; - r.list(go); - }); - if(good){ return g.it(err, disk, info) } - var id = Gun.text.random(3); console.log("MISLOCATED DATA", id); - r.save(disk, function ack(err, ok){ - if(err){ return r.save(disk, ack) } // ad infinitum??? - console.log("MISLOCATED CORRECTED", id); - r.read(key, cb, o); - }); - }*/ - if(o.reverse){ g.lex.reverse = true } - LOG && (S = +new Date); - r.list(g.lex); + r.find(key, g.find); } function rev(a,b){ return b } var revo = {reverse: true}; @@ -409,18 +253,18 @@ with how much performance and scale we can get out of only one. Then we can work on the harder problem of being multi-process. */ + var RPC = 0; var Q = {}, s = String.fromCharCode(31); r.parse = function(file, cb, raw){ var q; - if(q = Q[file]){ return q.push(cb) } q = Q[file] = [cb]; - var p = function Parse(){}, info = {file: ename(file)}; - p.disk = Radix(); + if(!file){ return cb(); } + if(q = Q[file]){ q.push(cb); return } q = Q[file] = [cb]; + var p = function Parse(){}, info = {file: file}; + (p.disk = Radix()).file = file; p.read = function(err, data){ var tmp; - LOG && opt.log(S, +new Date - S, 'read disk', ename(file)); + LOG && opt.log(S, +new Date - S, 'read disk', JSON.stringify(file), ++RPC, 'total all parses.'); delete Q[file]; - if((p.err = err) || (p.not = !data)){ - return map(q, p.ack); - } - if(typeof data !== 'string'){ + if((p.err = err) || (p.not = !data)){ map(q, p.ack); return } + if('string' !== typeof data){ try{ if(opt.pack <= data.length){ p.err = "Chunk too big!"; @@ -428,12 +272,11 @@ data = data.toString(); // If it crashes, it crashes here. How!?? We check size first! } }catch(e){ p.err = e } - if(p.err){ return map(q, p.ack) } + if(p.err){ map(q, p.ack); return } } info.parsed = data.length; - LOG && (S = +new Date); - if(opt.jsonify || '{' === data[0]){ // temporary testing idea + if(opt.jsonify || '{' === data[0]){ try{ var json = JSON.parse(data); // TODO: this caused a out-of-memory crash! p.disk.$ = json; @@ -443,14 +286,27 @@ }catch(e){ tmp = e } if('{' === data[0]){ p.err = tmp || "JSON error!"; - return map(q, p.ack); + map(q, p.ack); + return; } } + p.radec(err, data); + } + p.ack = function(cb){ + if(!cb){ return } + if(p.err || p.not){ + cb(p.err, u, info); + return; + } + cb(u, p.disk, info); + } + p.radec = function(err, data){ LOG && (S = +new Date); var tmp = p.split(data), pre = [], i, k, v; if(!tmp || 0 !== tmp[1]){ p.err = "File '"+file+"' does not have root radix! "; - return map(q, p.ack); + map(q, p.ack); + return; } while(tmp){ k = v = u; @@ -470,7 +326,6 @@ tmp = p.split(tmp[2]); } LOG && opt.log(S, +new Date - S, 'parsed RAD'); - //cb(err, p.disk); map(q, p.ack); }; p.split = function(t){ @@ -484,75 +339,63 @@ l[2] = t.slice(i + o.i); return l; } - p.ack = function(cb){ - if(!cb){ return } - if(p.err || p.not){ return cb(p.err, u, info) } - cb(u, p.disk, info); - } var S; LOG && (S = +new Date); - if(raw){ return p.read(null, raw) } + if(r.disk){ raw || (raw = (r.disk[file]||'').raw) } + if(raw){ return p.read(u, raw) } opt.store.get(ename(file), p.read); } }()); ;(function(){ - var dir, q, f = String.fromCharCode(28), ef = ename(f); - r.list = function(cb){ - if(dir){ - var last, tmp = {reverse: (cb.reverse)? 1 : 0, start: cb.start, end: cb.end}; - Radix.map(dir, function(val, key){ - if(!val){ return } - return cb(last = key); - }, tmp) || cb(u, last); + var dir, f = String.fromCharCode(28), Q; + r.find = function(key, cb){ + if(!dir){ + if(Q){ Q.push([key, cb]); return } Q = [[key, cb]]; + r.parse(f, init); return; } - if(q){ return q.push(cb) } q = [cb]; - r.parse(f, r.list.init); + Radix.map(r.list = dir, function(val, key){ + if(!val){ return } + return cb(key) || true; + }, {reverse: 1, end: key}) || cb(opt.code.from); } - r.list.add = function(file, cb){ + r.find.add = function(file, cb){ var has = dir(file); - if(has || file === ef){ - return cb(u, 1); - } + if(has || file === f){ cb(u, 1); return } dir(file, 1); - cb.listed = (cb.listed || 0) + 1; + cb.found = (cb.found || 0) + 1; r.write(f, dir, function(err, ok){ - if(err){ return cb(err) } - cb.listed = (cb.listed || 0) - 1; - if(cb.listed !== 0){ return } + if(err){ cb(err); return } + cb.found = (cb.found || 0) - 1; + if(0 !== cb.found){ return } cb(u, 1); }, true); } - r.list.bad = function(file, cb){ - dir(ename(file), 0); + r.find.bad = function(file, cb){ + dir(file, 0); r.write(f, dir, cb||noop); } - r.list.init = function(err, disk){ + function init(err, disk){ if(err){ opt.log('list', err); - setTimeout(function(){ r.parse(f, r.list.init) }, 1000); - return; - } - if(disk){ - r.list.drain(disk); - return; - } - if(!opt.store.list){ - r.list.drain(Radix()); + setTimeout(function(){ r.parse(f, init) }, 1000); return; } + if(disk){ drain(disk); return } + dir = dir || disk || Radix(); + if(!opt.store.list){ drain(dir); return } // import directory. opt.store.list(function(file){ - dir = dir || Radix(); - if(!file){ return r.list.drain(dir) } - r.list.add(file, noop); + if(!file){ drain(dir); return } + r.find.add(file, noop); }); } - r.list.drain = function(rad, tmp){ - r.list.dir = dir = rad; - tmp = q; q = null; - Gun.list.map(tmp, function(cb){ - r.list(cb); + function drain(rad, tmp){ + dir = dir || rad; + dir.file = f; + tmp = Q; Q = null; + Gun.list.map(tmp, function(arg){ + r.find(arg[0], arg[1]); }); } }()); @@ -562,8 +405,6 @@ return r; } - - ;(function(){ var _ = String.fromCharCode(31), u; Radisk.encode = function(d, o, s){ s = s || _; diff --git a/lib/radiskip.js b/lib/radiskip.js deleted file mode 100644 index 325bbf3c..00000000 --- a/lib/radiskip.js +++ /dev/null @@ -1,600 +0,0 @@ -;(function(){ - - function Radisk(opt){ - - opt = opt || {}; - opt.log = opt.log || console.log; - opt.file = String(opt.file || 'radata'); - var has = (Radisk.has || (Radisk.has = {}))[opt.file]; - if(has){ return has } - - opt.pack = opt.pack || (opt.memory? (opt.memory * 1000 * 1000) : 1399000000) * 0.3; // max_old_space_size defaults to 1400 MB. - opt.until = opt.until || opt.wait || 250; - opt.batch = opt.batch || (10 * 1000); - opt.chunk = opt.chunk || (1024 * 1024 * 1); // 1MB - opt.code = opt.code || {}; - opt.code.from = opt.code.from || '!'; - opt.jsonify = true; - - function ename(t){ return encodeURIComponent(t).replace(/\*/g, '%2A') } - function atomic(v){ return u !== v && (!v || 'object' != typeof v) } - var map = Gun.obj.map; - var LOG = console.LOG; - var ST = 0; - - if(!opt.store){ - return opt.log("ERROR: Radisk needs `opt.store` interface with `{get: fn, put: fn (, list: fn)}`!"); - } - if(!opt.store.put){ - return opt.log("ERROR: Radisk needs `store.put` interface with `(file, data, cb)`!"); - } - if(!opt.store.get){ - return opt.log("ERROR: Radisk needs `store.get` interface with `(file, cb)`!"); - } - if(!opt.store.list){ - //opt.log("WARNING: `store.list` interface might be needed!"); - } - - /* - Any and all storage adapters should... - 1. Because writing to disk takes time, we should batch data to disk. This improves performance, and reduces potential disk corruption. - 2. If a batch exceeds a certain number of writes, we should immediately write to disk when physically possible. This caps total performance, but reduces potential loss. - */ - var r = function(key, data, cb){ - if('function' === typeof data){ - var o = cb || {}; - cb = data; - r.read(key, cb); - return; - } - //var tmp = (tmp = r.batch = r.batch || {})[key] = tmp[key] || {}; - //var tmp = (tmp = r.batch = r.batch || {})[key] = data; - r.save(key, data, cb); - } - r.save = function(key, data, cb){ - var s = {key: key}; - s.find = function(file){ var tmp; - s.file = file || (file = opt.code.from); - if(tmp = r.disk[file]){ s.mix(u, tmp); return } - r.parse(file, s.mix); - } - s.mix = function(err, disk){ - if(err){ cb(err); return } - var file = s.file = (disk||'').file || s.file; - if(!disk && file !== opt.code.from){ // corrupt file? - r.find.bad(file); // remove from dir list - r.save(key, data, cb); // try again - return; - } - (disk = r.disk[file] || (r.disk[file] = disk || Radix())).file || (disk.file = file); - if(opt.compare){ - data = opt.compare(disk(key), data, key, file); - if(u === data){ cb(err, -1); return } - } - (s.disk = disk)(key, data); - if(disk.Q){ disk.Q.push(cb); return } disk.Q = [cb]; - disk.to = setTimeout(s.write, opt.until); - } - s.write = function(){ - var file = s.file, disk = s.disk; - s.q = disk.Q; - delete disk.Q; - delete r.disk[file]; - r.write(file, disk, s.ack); - } - s.ack = function(err, ok){ - var q = s.q || [], i = 0, ack; - //var S = +new Date; - while(ack = q[i++]){ ack(err, ok) } - //console.log('acks:', +new Date - S, s.file, q.length); - } - r.find(key, s.find); - } - r.disk = {}; - - /* - Any storage engine at some point will have to do a read in order to write. - This is true of even systems that use an append only log, if they support updates. - Therefore it is unavoidable that a read will have to happen, - the question is just how long you delay it. - */ - var RWC = 0; - r.write = function(file, rad, cb, o){ - if(!rad){ cb('No radix!'); return } - o = ('object' == typeof o)? o : {force: o}; - var f = function Fractal(){}, a, b; - f.text = ''; - f.file = file = rad.file || (rad.file = file); - if(!file){ cb('What file?'); return } - f.write = function(){ - var text = rad.raw = f.text; - r.disk[file = rad.file || f.file || file] = rad; - var S; LOG && (S = +new Date); - r.find.add(file, function add(err){ - if(err){ cb(err); return } - opt.store.put(ename(file), text, function safe(err, ok){ - LOG && opt.log(S, ST = +new Date - S, "wrote disk", JSON.stringify(file), ++RWC, 'total all writes.'); - cb(err, ok || 1); - if(!rad.Q){ delete r.disk[file] } // VERY IMPORTANT! Clean up memory, but not if there is already queued writes on it! - }); - }); - } - f.split = function(){ - f.text = ''; - if(!f.count){ f.count = 0; - Radix.map(rad, function count(){ f.count++ }); // TODO: Perf? Any faster way to get total length? - } - f.limit = Math.ceil(f.count/2); - f.count = 0; - f.sub = Radix(); - Radix.map(rad, f.slice, {reverse: 1}); // IMPORTANT: DO THIS IN REVERSE, SO LAST HALF OF DATA MOVED TO NEW FILE BEFORE DROPPING FROM CURRENT FILE. - r.write(f.end, f.sub, f.both, o); - f.hub = Radix(); - Radix.map(rad, f.stop); - r.write(rad.file, f.hub, f.both, o); - return true; - } - f.slice = function(val, key){ - f.sub(f.end = key, val); - if(f.limit <= (++f.count)){ return true } - } - f.stop = function(val, key){ - if(key >= f.end){ return true } - f.hub(key, val); - } - f.both = function(err, ok){ - if(b){ cb(err || b); return } - if(a){ cb(err, ok); return } - a = true; - b = err; - } - f.each = function(val, key, k, pre){ - //console.log("RAD:::", JSON.stringify([val, key, k, pre])); - if(u !== val){ f.count++ } - if(opt.pack <= (val||'').length){ return cb("Record too big!"), true } - var enc = Radisk.encode(pre.length) +'#'+ Radisk.encode(k) + (u === val? '' : ':'+ Radisk.encode(val)) +'\n'; - if((opt.chunk < f.text.length + enc.length) && (1 < f.count) && !o.force){ - return f.split(); - } - f.text += enc; - } - if(opt.jsonify){ r.write.jsonify(f, rad, cb, o); return } // temporary testing idea - if(!Radix.map(rad, f.each, true)){ f.write() } - } - - r.write.jsonify = function(f, rad, cb, o){ - var raw; - var S; LOG && (S = +new Date); - try{raw = JSON.stringify(rad.$); - }catch(e){ cb("Cannot radisk!"); return } - LOG && opt.log(S, +new Date - S, "rad stringified JSON"); - if(opt.chunk < raw.length && !o.force){ return f.split() } - f.text = raw; - f.write(); - } - - r.range = function(tree, o){ - if(!tree || !o){ return } - if(u === o.start && u === o.end){ return tree } - if(atomic(tree)){ return tree } - var sub = Radix(); - Radix.map(tree, function(v,k){ sub(k,v) }, o); // ONLY PLACE THAT TAKES TREE, maybe reduce API for better perf? - return sub(''); - } - - ;(function(){ - var Q = {}; - r.read = function(key, cb, o){ - o = o || {}; - if(RAD && !o.next){ // cache - var S; LOG && (S = +new Date); - var val = RAD(key); - LOG && (ST = +new Date - S) > 9 && opt.log(S, ST, 'rad cached'); - //if(u !== val){ - //cb(u, val, o); - if(atomic(val)){ cb(u, val, o); return } - // if a node is requested and some of it is cached... the other parts might not be. - //} - } - o.span = (u !== o.start) || (u !== o.end); // is there a start or end? - var g = function Get(){}; - g.lex = function(file){ var tmp; // // TODO: this had a out-of-memory crash! - file = (u === file)? u : decodeURIComponent(file); - tmp = o.next || key || (o.reverse? o.end || '\uffff' : o.start || ''); - if(!file || (o.reverse? file < tmp : file > tmp)){ - LOG && opt.log(S, +new Date - S, 'rad read lex'); S = +new Date; - if(o.next || o.reverse){ g.file = file } - if(tmp = Q[g.file]){ - tmp.push({key: key, ack: cb, file: g.file, opt: o}); - return true; - } - Q[g.file] = [{key: key, ack: cb, file: g.file, opt: o}]; - if(!g.file){ - g.it(u, u, {}); - return true; - } - r.parse(g.file, g.check); - return true; - } - g.file = file; - } - g.it = function(err, disk, info){ - if(g.err = err){ opt.log('err', err) } - if(!disk && g.file){ // corrupt file? - r.find.bad(g.file); // remove from dir list - r.read(key, cb, o); // look again - return; - } - g.info = info; - if(disk){ RAD = g.disk = disk } - disk = Q[g.file]; delete Q[g.file]; - map(disk, g.ack); - } - g.ack = function(as){ - if(!as.ack){ return } - var S; LOG && (S = +new Date); - var key = as.key, o = as.opt, info = g.info, rad = g.disk || noop, data = r.range(rad(key), o), last = rad.last || Radix.map(rad, rev, revo); - LOG && (ST = +new Date - S) > 9 && opt.log(S, ST, "rad range loaded"); - o.parsed = (o.parsed || 0) + (info.parsed||0); - o.chunks = (o.chunks || 0) + 1; - o.more = true; - if((!as.file) // if no more places to look - || (!o.span && last === key) // if our key exactly matches the very last atomic record - || (!o.span && last && last > key && 0 != last.indexOf(key)) // 'zach' may be lexically larger than 'za', but there still might be more, like 'zane' in the 'za' prefix bucket so do not end here. - ){ - o.more = u; - as.ack(g.err, data, o); - return; - } - if(u !== data){ - as.ack(g.err, data, o); // more might be coming! - if(o.parsed >= o.limit){ return } // even if more, we've hit our limit, asking peer will need to make a new ask with a new starting point. - } - o.next = as.file; - r.read(key, as.ack, o); - } - g.check = function(err, disk, info){ - g.it(err, disk, info); - var good = true; - Radix.map(disk, function(val, key){ - // assume in memory for now, since both write/read already call r.find which will init it. - var go = function(file){ - if(info.file !== file){ - good = false - } - return true; - } - go.reverse = 1; - go.end = key; - r.list(go); - }); - if(good){ return } - var id = Gun.text.random(3); - r.save(disk, function ack(err, ok){ - if(err){ r.save(disk, ack); return } // ad infinitum??? - console.log("MISLOCATED DATA CORRECTED", id); - }); - } - /*g.check2 = function(err, disk, info){ - if(err || !disk){ g.it(err, disk, info); return } - var good = true; - Radix.map(disk, function(val, key){ - // assume in memory for now, since both write/read already call r.find which will init it. - var go = function(file){ - if(info.file !== file){ good = false } - return true; - } - go.reverse = 1; - go.end = key; - r.list(go); - }); - if(good){ g.it(err, disk, info); return } - var id = Gun.text.random(3); console.log("MISLOCATED DATA", id); - r.save(disk, function ack(err, ok){ - if(err){ r.save(disk, ack); return } // ad infinitum??? - console.log("MISLOCATED CORRECTED", id); - r.read(key, cb, o); - }); - }*/ - if(o.reverse){ g.lex.reverse = true } - LOG && (S = +new Date); - r.find(key, g.lex); - } - - r.read = function(key, cb, o){ - var g = {key: key}; - g.find = function(file){ var tmp; - g.file = file || (file = opt.code.from); // this may not be true for reads? Hit "end of dir"? - if(tmp = r.disk[file]){ g.check(u, tmp); return } - r.parse(file, g.check); - } - g.get = function(err, disk, info){ - if(err){ cb(err); return } - var file = g.file = (disk||'').file || g.file; - if(!disk && file !== opt.code.from){ // corrupt file? - r.find.bad(file); // remove from dir list - r.save(key, cb); // try again - return; - } - disk = r.disk[file] || (r.disk[file] = disk); - if(!disk){ cb(); return } - disk.file || (disk.file = file); - // ---------------------------- - info = info || {}; - var data = disk(key); - info.atom = disk.atom; - if(u !== data){ cb(u, data, info); return } - // ---------------------------- - return; - var S; LOG && (S = +new Date); - var rad = disk || noop, data = r.range(rad(key), o), last = rad.last || Radix.map(rad, rev, revo); - LOG && (ST = +new Date - S) > 9 && opt.log(S, ST, "rad range loaded"); - o.parsed = (o.parsed || 0) + (info.parsed||0); - o.chunks = (o.chunks || 0) + 1; - o.more = true; - if((!as.file) // if no more places to look - || (!o.span && last === key) // if our key exactly matches the very last atomic record - || (!o.span && last && last > key && 0 != last.indexOf(key)) // 'zach' may be lexically larger than 'za', but there still might be more, like 'zane' in the 'za' prefix bucket so do not end here. - ){ - o.more = u; - as.ack(g.err, data, o); - return; - } - if(u !== data){ - as.ack(g.err, data, o); // more might be coming! - if(o.parsed >= o.limit){ return } // even if more, we've hit our limit, asking peer will need to make a new ask with a new starting point. - } - o.next = as.file; - r.read(key, as.ack, o); - } - g.check = function(err, disk, info){ - g.get(err, disk, info); - (info || (info = {})).file || (info.file = g.file); - Radix.map(disk, function(val, key){ - // assume in memory for now, since both write/read already call r.find which will init it. - r.find(key, function(file){ - if(file === info.file){ return } - var id = Gun.text.random(3); - console.log("MISLOCATED DATA", id, key, info.file, file); - r.save(key, val, function ack(err, ok){ - if(err){ r.save(key, val, ack); return } // ad infinitum??? - console.log("MISLOCATED DATA CORRECTED", id); - }); - }) - }); - } - r.find(key, g.find); - } - function rev(a,b){ return b } - var revo = {reverse: true}; - }()); - - ;(function(){ - /* - Let us start by assuming we are the only process that is - changing the directory or bucket. Not because we do not want - to be multi-process/machine, but because we want to experiment - with how much performance and scale we can get out of only one. - Then we can work on the harder problem of being multi-process. - */ - var RPC = 0; - var Q = {}, s = String.fromCharCode(31); - r.parse = function(file, cb, raw){ var q; - if(q = Q[file]){ q.push(cb); return } q = Q[file] = [cb]; - var p = function Parse(){}, info = {file: file}; - (p.disk = Radix()).file = file; - p.read = function(err, data){ var tmp; - LOG && opt.log(S, +new Date - S, 'read disk', JSON.stringify(file), ++RPC, 'total all parses.'); - delete Q[file]; - if((p.err = err) || (p.not = !data)){ map(q, p.ack); return } - if('string' !== typeof data){ - try{ - if(opt.pack <= data.length){ - p.err = "Chunk too big!"; - } else { - data = data.toString(); // If it crashes, it crashes here. How!?? We check size first! - } - }catch(e){ p.err = e } - if(p.err){ map(q, p.ack); return } - } - info.parsed = data.length; - LOG && (S = +new Date); - if(opt.jsonify || '{' === data[0]){ - try{ - var json = JSON.parse(data); // TODO: this caused a out-of-memory crash! - p.disk.$ = json; - LOG && (ST = +new Date - S) > 9 && opt.log(S, ST, 'rad parsed JSON'); - map(q, p.ack); - return; - }catch(e){ tmp = e } - if('{' === data[0]){ - p.err = tmp || "JSON error!"; - map(q, p.ack); - return; - } - } - p.radec(err, data); - } - p.ack = function(cb){ - if(!cb){ return } - if(p.err || p.not){ - cb(p.err, u, info); - return; - } - cb(u, p.disk, info); - } - p.radec = function(err, data){ - LOG && (S = +new Date); - var tmp = p.split(data), pre = [], i, k, v; - if(!tmp || 0 !== tmp[1]){ - p.err = "File '"+file+"' does not have root radix! "; - map(q, p.ack); - return; - } - while(tmp){ - k = v = u; - i = tmp[1]; - tmp = p.split(tmp[2])||''; - if('#' == tmp[0]){ - k = tmp[1]; - pre = pre.slice(0,i); - if(i <= pre.length){ - pre.push(k); - } - } - tmp = p.split(tmp[2])||''; - if('\n' == tmp[0]){ continue } - if('=' == tmp[0] || ':' == tmp[0]){ v = tmp[1] } - if(u !== k && u !== v){ p.disk(pre.join(''), v) } - tmp = p.split(tmp[2]); - } - LOG && opt.log(S, +new Date - S, 'parsed RAD'); - map(q, p.ack); - }; - p.split = function(t){ - if(!t){ return } - var l = [], o = {}, i = -1, a = '', b, c; - i = t.indexOf(s); - if(!t[i]){ return } - a = t.slice(0, i); - l[0] = a; - l[1] = b = Radisk.decode(t.slice(i), o); - l[2] = t.slice(i + o.i); - return l; - } - var S; LOG && (S = +new Date); - if(r.disk){ raw || (raw = (r.disk[file]||'').raw) } - if(raw){ return p.read(u, raw) } - opt.store.get(ename(file), p.read); - } - }()); - - ;(function(){ - var dir, f = String.fromCharCode(28), Q; - r.find = function(key, cb){ - if(!dir){ - if(Q){ Q.push([key, cb]); return } Q = [[key, cb]]; - r.parse(f, init); - return; - } - Radix.map(dir, function(val, key){ - if(!val){ return } - return cb(key) || true; - }, {reverse: 1, end: key}) || cb(); - } - r.find.add = function(file, cb){ - var has = dir(file); - if(has || file === f){ cb(u, 1); return } - dir(file, 1); - cb.found = (cb.found || 0) + 1; - r.write(f, dir, function(err, ok){ - if(err){ cb(err); return } - cb.found = (cb.found || 0) - 1; - if(0 !== cb.found){ return } - cb(u, 1); - }, true); - } - r.find.bad = function(file, cb){ - dir(file, 0); - r.write(f, dir, cb||noop); - } - function init(err, disk){ - if(err){ - opt.log('list', err); - setTimeout(function(){ r.parse(f, init) }, 1000); - return; - } - if(disk){ drain(disk); return } - dir = dir || disk || Radix(); - if(!opt.store.list){ drain(dir); return } - // import directory. - opt.store.list(function(file){ - if(!file){ drain(dir); return } - r.find.add(file, noop); - }); - } - function drain(rad, tmp){ - dir = dir || rad; - dir.file = f; - tmp = Q; Q = null; - Gun.list.map(tmp, function(arg){ - r.find(arg[0], arg[1]); - }); - } - }()); - - var noop = function(){}, RAD, u; - Radisk.has[opt.file] = r; - return r; - } - - ;(function(){ - var _ = String.fromCharCode(31), u; - Radisk.encode = function(d, o, s){ s = s || _; - var t = s, tmp; - if(typeof d == 'string'){ - var i = d.indexOf(s); - while(i != -1){ t += s; i = d.indexOf(s, i+1) } - return t + '"' + d + s; - } else - if(d && d['#'] && (tmp = Gun.val.link.is(d))){ - return t + '#' + tmp + t; - } else - if(Gun.num.is(d)){ - return t + '+' + (d||0) + t; - } else - if(null === d){ - return t + ' ' + t; - } else - if(true === d){ - return t + '+' + t; - } else - if(false === d){ - return t + '-' + t; - }// else - //if(binary){} - } - Radisk.decode = function(t, o, s){ s = s || _; - var d = '', i = -1, n = 0, c, p; - if(s !== t[0]){ return } - while(s === t[++i]){ ++n } - p = t[c = n] || true; - while(--n >= 0){ i = t.indexOf(s, i+1) } - if(i == -1){ i = t.length } - d = t.slice(c+1, i); - if(o){ o.i = i+1 } - if('"' === p){ - return d; - } else - if('#' === p){ - return Gun.val.link.ify(d); - } else - if('+' === p){ - if(0 === d.length){ - return true; - } - return parseFloat(d); - } else - if(' ' === p){ - return null; - } else - if('-' === p){ - return false; - } - } - }()); - - if(typeof window !== "undefined"){ - var Gun = window.Gun; - var Radix = window.Radix; - window.Radisk = Radisk; - } else { - var Gun = require('../gun'); - var Radix = require('./radix'); - //var Radix = require('./radix2'); Radisk = require('./radisk2'); - try{ module.exports = Radisk }catch(e){} - } - - Radisk.Radix = Radix; - -}()); \ No newline at end of file diff --git a/lib/radix.js b/lib/radix.js index f33bc4b3..1d28b46a 100644 --- a/lib/radix.js +++ b/lib/radix.js @@ -2,13 +2,13 @@ function Radix(){ var radix = function(key, val, t){ - radix.atom = 0; + radix.unit = 0; if(!t && u !== val){ radix.last = (''+key < radix.last)? radix.last : ''+key; delete (radix.$||{})[_]; } t = t || radix.$ || (radix.$ = {}); - if(!key && Object.keys(t).length){ console.log('wat?', t, key); return t } + if(!key && Object.keys(t).length){ return t } key = ''+key; var i = 0, l = key.length-1, k = key[i], at, tmp; while(!(at = t[k]) && i < l){ @@ -48,7 +48,7 @@ } else if(i == l){ //if(u === val){ return (u === (tmp = at['']))? at : tmp } // THIS CODE IS CORRECT, below is - if(u === val){ return (u === (tmp = at['']))? at : ((radix.atom = 1) && tmp) } // temporary help?? + if(u === val){ return (u === (tmp = at['']))? at : ((radix.unit = 1) && tmp) } // temporary help?? at[''] = val; //(at[_] = function $(){ $.sort = Object.keys(at).sort(); return $ }()); } else { diff --git a/lib/rfs.js b/lib/rfs.js index 705ae88a..f66ddb4c 100644 --- a/lib/rfs.js +++ b/lib/rfs.js @@ -12,6 +12,7 @@ function Store(opt){ Store[opt.file] = store; var puts = {}; + // TODO!!! ADD ZLIB INFLATE / DEFLATE COMPRESSION! store.put = function(file, data, cb){ puts[file] = data; var random = Math.random().toString(36).slice(-3); @@ -29,7 +30,7 @@ function Store(opt){ if('ENOENT' === (err.code||'').toUpperCase()){ return cb(); } - opt.log("ERROR:", err) + opt.log("ERROR:", err); } cb(err, data); }); diff --git a/lib/store.js b/lib/store.js index 1ca48d89..4e17c809 100644 --- a/lib/store.js +++ b/lib/store.js @@ -6,13 +6,11 @@ Gun.on('create', function(root){ var opt = root.opt, empty = {}, u; if(false === opt.radisk){ return } var Radisk = (Gun.window && Gun.window.Radisk) || require('./radisk'); - var Radiskip = (Gun.window && Gun.window.Radisk) || require('./radiskip'); var Radix = Radisk.Radix; var LOG = console.LOG, ST = 0; opt.store = opt.store || (!Gun.window && require('./rfs')(opt)); - var rad = Radisk(opt), esc = String.fromCharCode(27); - var dare = Radiskip(opt); + var dare = Radisk(opt), esc = String.fromCharCode(27); root.on('put2', function(msg){ this.to.next(msg); @@ -31,8 +29,7 @@ Gun.on('create', function(root){ var _ = (msg._||''), got = _.rad; if(got){ return } // RAD's own ACKs to GETs do not need to be written to disk again. if(_.ram){ return } // in-memory ACKs to GETs do not need to be written to disk again. - root.on('in', {'@': msg['#'], err: 'Migration not done, please report this to & complain at Mark in http://chat.gun.eco !'}); - return; + if(true || !Gun.TESTING){ root.on('in', {'@': msg['#'], err: console.log('Migration not done, please report this to & complain at Mark in http://chat.gun.eco !')}); return } var S = (+new Date), C = 0; // STATS! var now = Gun.state(); Gun.graph.is(msg.put, null, function(val, key, node, soul){ @@ -45,9 +42,9 @@ Gun.on('create', function(root){ return; } if(track){ ++acks } - //console.log('put:', soul, key, val); - val = Radisk.encode(val, null, esc)+'>'+Radisk.encode(Gun.state.is(node, key), null, esc); - rad(soul+esc+key, val, (track? ack : u)); + dare(soul+esc+key, {':': val, '>': Gun.state.is(node, key)}, (track? ack : u)); + //val = Radisk.encode(val, null, esc)+'>'+Radisk.encode(Gun.state.is(node, key), null, esc); + //rad(soul+esc+key, val, (track? ack : u)); C++; }); if(LOG && (ST = +new Date - S) > 9){ Gun.log(S, ST, 'put loop'); Gun.log(S, C, 'put loop #') } @@ -107,16 +104,18 @@ Gun.on('create', function(root){ var now = Gun.state(); var S = (+new Date), C = 0; // STATS! //rad(key||'', function(err, data, o){ + //console.log("STORE GET:", JSON.stringify(key||''), o); dare(key||'', function(err, data, info){ + //console.log("STORE GOT:", data); try{opt.store.stats.get.time[statg % 50] = (+new Date) - S; ++statg; opt.store.stats.get.count++; if(err){ opt.store.stats.get.err = err } }catch(e){} // STATS! //if(u === data && info.chunks > 1){ return } // if we already sent a chunk, ignore ending empty responses. // this causes tests to fail. LOG && Gun.log(S, +new Date - S, 'got', JSON.stringify(key)); S = +new Date; - info = info || {}; + info = info || ''; var va, ve; - if(info.atom && data && u !== (va = data[':']) && u !== (ve = data['>'])){ // new format + if(info.unit && data && u !== (va = data[':']) && u !== (ve = data['>'])){ // new format var tmp = key.split(esc), so = tmp[0], ha = tmp[1]; (graph = graph || {})[so] = Gun.state.ify(graph[so], ha, ve, va, so); root.$.get(so).get(ha)._.rad = now; @@ -134,7 +133,6 @@ Gun.on('create', function(root){ } console.STAT && (console.STAT.radgetcount = C); if(LOG && (ST = +new Date - S) > 9){ Gun.log(S, ST, 'got prep time'); Gun.log(S, C, 'got prep #') } C = 0; S = +new Date; - //console.log("STORE GOT:", graph); var faith = function(){}; faith.faith = true; faith.rad = get; // HNPERF: We're testing performance improvement by skipping going through security again, but this should be audited. root.on('in', {'@': id, put: graph, '%': info.more? 1 : u, err: err? err : u, _: faith}); LOG && (ST = +new Date - S) > 9 && Gun.log(S, ST, 'got emit', Object.keys(graph||{}).length); @@ -150,8 +148,11 @@ Gun.on('create', function(root){ has = has.slice(-1)[0]; if(o.limit && o.limit <= o.count){ return true } var va, ve, so = soul, ha = has; - if((va = val[':']) && (ve = val['>'])){ // THIS HANDLES NEW CODE! + //if(u !== (va = val[':']) && u !== (ve = val['>'])){ // THIS HANDLES NEW CODE! + if('string' != typeof val){ // THIS HANDLES NEW CODE! + va = val[':']; ve = val['>']; (graph = graph || {})[so] = Gun.state.ify(graph[so], ha, ve, va, so); + //root.$.get(so).get(ha)._.rad = now; o.count = (o.count || 0) + ((va||'').length || 9); return; } diff --git a/sea.js b/sea.js index 3730fc21..7c062d12 100644 --- a/sea.js +++ b/sea.js @@ -439,7 +439,7 @@ opt = opt || {}; // SEA.I // verify is free! Requires no user permission. var pub = pair.pub || pair; - var key = SEA.opt.slow_leak? await SEA.opt.slow_leak(pub) : await (shim.ossl || shim.subtle).importKey('jwk', jwk, {name: 'ECDSA', namedCurve: 'P-256'}, false, ['verify']); + var key = SEA.opt.slow_leak? await SEA.opt.slow_leak(pub) : await (shim.ossl || shim.subtle).importKey('jwk', S.jwk(pub), {name: 'ECDSA', namedCurve: 'P-256'}, false, ['verify']); var hash = await sha(json.m); var buf, sig, check, tmp; try{ buf = shim.Buffer.from(json.s, opt.encode || 'base64'); // NEW DEFAULT! @@ -474,9 +474,11 @@ return knownKeys[pair]; }; - + var O = SEA.opt; SEA.opt.fall_verify = async function(data, pair, cb, opt, f){ if(f === SEA.opt.fallback){ throw "Signature did not match" } f = f || 1; + var tmp = data||''; + data = SEA.opt.unpack(data) || data; var json = S.parse(data), pub = pair.pub || pair, key = await SEA.opt.slow_leak(pub); var hash = (f <= SEA.opt.fallback)? shim.Buffer.from(await shim.subtle.digest({name: 'SHA-256'}, new shim.TextEncoder().encode(S.parse(json.m)))) : await sha(json.m); // this line is old bad buggy code but necessary for old compatibility. var buf; var sig; var check; try{ @@ -491,6 +493,7 @@ if(!check){ throw "Signature did not match." } } var r = check? S.parse(json.m) : u; + O.fall_soul = tmp['#']; O.fall_key = tmp['.']; O.fall_val = data; O.fall_state = tmp['>']; if(cb){ try{ cb(r) }catch(e){console.log(e)} } return r; } @@ -1099,7 +1102,7 @@ if(!at.sea){ // only add SEA once per instance, on the "at" context. at.sea = {own: {}}; at.on('in', security, at); // now listen to all input data, acting as a firewall. - at.on('out', signature, at); // and output listeners, to encrypt outgoing data. + //at.on('out', signature, at); // and output listeners, to encrypt outgoing data. at.on('node', each, at); at.on('put2', check, at); } @@ -1165,10 +1168,11 @@ if('~@' === soul.slice(0,2)){ // special case for shared system data, the list of public keys for an alias. check.pubs(eve, msg, val, key, soul, at, no); return; } - if('~' === soul.slice(0,1) && 2 === (tmp = soul.slice(1)).split('.').length){ // special case, account data for a public key. - check.pub(eve, msg, val, key, soul, at, no, (msg._||'').user, tmp); return; + //if('~' === soul.slice(0,1) && 2 === (tmp = soul.slice(1)).split('.').length){ // special case, account data for a public key. + if(tmp = SEA.opt.pub(soul)){ // special case, account data for a public key. + check.pub(eve, msg, val, key, soul, at, no, at.user||'', tmp); return; } - check.any(eve, msg, val, key, soul, at, no, (msg._||'noop').user); return; + check.any(eve, msg, val, key, soul, at, no, at.user||''); return; eve.to.next(msg); // not handled } check.hash = function(eve, msg, val, key, soul, at, no){ @@ -1192,59 +1196,34 @@ if(val === pub){ return eve.to.next(msg) } // the account MUST match `pub` property that equals the ID of the public key. return no("Account not same!"); } - if(Gun.is(msg.$) && user && user.is && pub === user.is.pub){ - SEA.sign(msg.put, (user._).sea, function(data){ var rel; + if((tmp = user.is) && pub === tmp.pub){ + SEA.sign(SEA.opt.pack(msg.put), (user._).sea, function(data){ if(u === data){ return no(SEA.err || 'Signature fail.') } - if(rel = link_is(val)){ (at.sea.own[rel] = at.sea.own[rel] || {})[pub] = 1 } - console.log("WHAT HAPPENS HERE?", data.m, SEA.opt.unpack(data.m), key, soul); - msg.put[':'] = JSON.stringify({':': SEA.opt.unpack(data.m), '~': data.s}); - //node[key] = JSON.stringify({':': SEA.opt.unpack(data.m), '~': data.s}); + if(tmp = link_is(val)){ (at.sea.own[tmp] = at.sea.own[tmp] || {})[pub] = 1 } + msg.put[':'] = JSON.stringify({':': tmp = SEA.opt.unpack(data.m), '~': data.s}); + msg.put['='] = tmp; eve.to.next(msg); - }, {check: msg.put, raw: 1}); + }, {raw: 1}); return; } - SEA.verify(msg.put, pub, function(data){ var rel, tmp; - console.log("WHAT VERIFIES HERE?", data, SEA.opt.unpack(data, key), key, soul); - data = SEA.opt.unpack(data, key); + SEA.verify(SEA.opt.pack(msg.put), pub, function(data){ var tmp; + data = SEA.opt.unpack(data); if(u === data){ return no("Unverified data.") } // make sure the signature matches the account it claims to be on. // reject any updates that are signed with a mismatched account. - if((rel = link_is(data)) && pub === SEA.opt.pub(rel)){ - (at.sea.own[rel] = at.sea.own[rel] || {})[pub] = 1; - } + if((tmp = link_is(data)) && pub === SEA.opt.pub(tmp)){ (at.sea.own[tmp] = at.sea.own[tmp] || {})[pub] = 1 } + msg.put['='] = data; eve.to.next(msg); }); }; check.any = function(eve, msg, val, key, soul, at, no, user){ var tmp, pub; - if(!(pub = SEA.opt.pub(soul))){ - if(at.opt.secure){ return no("Soul missing public key at '" + key + "'.") } - // TODO: Ask community if should auto-sign non user-graph data. - at.on('secure', function(msg){ this.off(); - if(!at.opt.secure){ return eve.to.next(msg) } - no("Data cannot be changed."); - }).on.on('secure', msg); - return; - } - // TODO: DEDUP WITH check.pub ??? - if(Gun.is(msg.$) && user && user.is && pub === user.is.pub){ - SEA.sign(mgs.put, (user._).sea, function(data){ - if(u === data){ return no('User signature fail.') } - console.log("WHAT HAPPENS HERE??", data.m, SEA.opt.unpack(data.m), key, soul); - msg.put[':'] = JSON.stringify({':': SEA.opt.unpack(data.m), '~': data.s}); - //node[key] = JSON.stringify({':': SEA.opt.unpack(data.m), '~': data.s}); - eve.to.next(msg); - }, {check: msg.put, raw: 1}); - return; - } - SEA.verify(msg.put, pub, function(data){ var rel; - console.log("WHAT VERIFIES HERE?", data, SEA.opt.unpack(data, key), key, soul); - data = SEA.opt.unpack(data, key); - if(u === data){ return no("Not owner on '" + key + "'.") } // thanks @rogowski ! - if((rel = link_is(data)) && pub === SEA.opt.pub(rel)){ - (at.sea.own[rel] = at.sea.own[rel] || {})[pub] = 1; - } - eve.to.next(msg); - }); + if(at.opt.secure){ return no("Soul missing public key at '" + key + "'.") } + // TODO: Ask community if should auto-sign non user-graph data. + at.on('secure', function(msg){ this.off(); + if(!at.opt.secure){ return eve.to.next(msg) } + no("Data cannot be changed."); + }).on.on('secure', msg); + return; } - var link_is = Gun.val.link.is; + var link_is = Gun.val.link.is, state_ify = Gun.state.ify; // okay! The security function handles all the heavy lifting. // It needs to deal read and write of input and output of system data, account/public key data, and regular data. @@ -1408,6 +1387,7 @@ if(!s || !(s = s[1])){ return } s = s.split('.'); if(!s || 2 > s.length){ return } + if('@' === (s[0]||'')[0]){ return } // TODO: Should check ~X.Y. are alphanumeric, not just not @. s = s.slice(0,2).join('.'); return s; } @@ -1416,16 +1396,18 @@ } SEA.opt.pack = function(d,k, n,s){ // pack for verifying if(SEA.opt.check(d)){ return d } - var meta = (Gun.obj.ify(d)||noop), sig = meta['~']; - return sig? {m: {'#':s,'.':k,':':meta[':'],'>':Gun.state.is(n, k)}, s: sig} : d; + var meta = (Gun.obj.ify(d)||''), sig = meta['~']; + return sig? {m: {'#':s||d['#'],'.':k||d['.'],':':meta[':'],'>':d['>']||Gun.state.is(n, k)}, s: sig} : d; } + var O = SEA.opt; SEA.opt.unpack = function(d, k, n){ var tmp; if(u === d){ return } if(d && (u !== (tmp = d[':']))){ return tmp } + k = k || O.fall_key; if(!n && O.fall_val){ n = {}; n[k] = O.fall_val } if(!k || !n){ return } if(d === n[k]){ return d } if(!SEA.opt.check(n[k])){ return d } - var soul = Gun.node.soul(n), s = Gun.state.is(n, k); + var soul = Gun.node.soul(n) || O.fall_soul, s = Gun.state.is(n, k) || O.fall_state; if(d && 4 === d.length && soul === d[0] && k === d[1] && fl(s) === fl(d[3])){ return d[2]; } @@ -1437,6 +1419,7 @@ var noop = function(){}, u; var fl = Math.floor; // TODO: Still need to fix inconsistent state issue. var rel_is = Gun.val.rel.is; + var obj_ify = Gun.obj.ify; // TODO: Potential bug? If pub/priv key starts with `-`? IDK how possible. })(USE, './index'); diff --git a/test/common.js b/test/common.js index 91f6615b..1cf70726 100644 --- a/test/common.js +++ b/test/common.js @@ -1279,6 +1279,14 @@ describe('Gun', function(){ this.to.next(root); });*/ } + Gun.on('create', function(root){ + root.on('test', function(msg){ + var put = msg.put; + this.to.next(msg); + root.on('out', msg); + Gun.graph.is(put, function(n,s){ root.$.get(s).off() }); + }) + }) var gun = Gun(); it.skip('gun chain separation', function(done){ // TODO: UNDO! @@ -1397,8 +1405,7 @@ describe('Gun', function(){ - Performant read lock on write contexts. - Proxying event across maps. */ - var s = Gun.state.map();s.soul = 'u/m'; - gun.on('put', {$: gun, put: Gun.graph.ify({ + gun.on('test', {$: gun, put: Gun.graph.ify({ alice: { age: 26, name: "Alice", @@ -1409,7 +1416,7 @@ describe('Gun', function(){ name: "Bob!", pet: {b:2, name: "Frisky"} } - }, s)}); + }, Gun.state.map(), 'u/m')}); var check = {}, count = {}; gun.get('u/m').map().on(function(v,f){ check[f] = v; @@ -1433,8 +1440,7 @@ describe('Gun', function(){ }); it('uncached synchronous map get on', function(done){ - var s = Gun.state.map();s.soul = 'u/m/p'; - gun.on('put', {$: gun, put: Gun.graph.ify({ + gun.on('test', {$: gun, put: Gun.graph.ify({ alice: { age: 26, name: "alice", @@ -1445,7 +1451,7 @@ describe('Gun', function(){ name: "bob", pet: {b:2, name: "Frisky"} } - }, s)}); + }, Gun.state.map(), 'u/m/p')}); var check = {}, count = {}; gun.get('u/m/p').map().get('name').on(function(v,f){ //console.log("*****************", f, v); @@ -1465,8 +1471,7 @@ describe('Gun', function(){ }); it('uncached synchronous map get on node', function(done){ - var s = Gun.state.map();s.soul = 'u/m/p/n'; - gun.on('put', {$: gun, put: Gun.graph.ify({ + gun.on('test', {$: gun, put: Gun.graph.ify({ alice: { age: 26, name: "alice", @@ -1477,7 +1482,7 @@ describe('Gun', function(){ name: "bob", pet: {b:2, name: "Frisky"} } - }, s)}); + }, Gun.state.map(), 'u/m/p/n')}); var check = {}, count = {}; gun.get('u/m/p/n').map().get('pet').on(function(v,f){ //console.log("********************", f,v); @@ -1500,8 +1505,7 @@ describe('Gun', function(){ it('uncached synchronous map get on node get', function(done){ var gun = Gun(); - var s = Gun.state.map();s.soul = 'u/m/p/n/p'; - gun.on('put', {$: gun, put: Gun.graph.ify({ + gun.on('test', {$: gun, put: Gun.graph.ify({ alice: { age: 26, name: "alice", @@ -1512,7 +1516,7 @@ describe('Gun', function(){ name: "bob", pet: {b:2, name: "Frisky"} } - }, s)}); + }, Gun.state.map(), 'u/m/p/n/p')}); var check = {}, count = {}; //console.debug.i=1;console.log('-------------------'); gun.get('u/m/p/n/p').map().get('pet').get('name').on(function(v,f){ @@ -1539,8 +1543,7 @@ describe('Gun', function(){ }); it('uncached synchronous map on mutate', function(done){ - var s = Gun.state.map();s.soul = 'u/m/mutate'; - gun.on('put', {$: gun, put: Gun.graph.ify({ + gun.on('test', {$: gun, put: Gun.graph.ify({ alice: { age: 26, name: "Alice", @@ -1551,7 +1554,7 @@ describe('Gun', function(){ name: "Bob", pet: {b:2, name: "Frisky"} } - }, s)}); + }, Gun.state.map(), 'u/m/mutate')}); var check = {}, count = {}; gun.get('u/m/mutate').map().get('name').get(function(at,ev){ var e = at.err, v = at.put, f = at.get; @@ -1575,8 +1578,7 @@ describe('Gun', function(){ }); it('uncached synchronous map on mutate node', function(done){ - var s = Gun.state.map();s.soul = 'u/m/mutate/n'; - gun.on('put', {$: gun, put: Gun.graph.ify({ + gun.on('test', {$: gun, put: Gun.graph.ify({ alice: {_:{'#':'umaliceo'}, age: 26, name: "Alice", @@ -1587,7 +1589,7 @@ describe('Gun', function(){ name: "Bob", pet: {b:2, name: "Frisky"} } - }, s)}); + }, Gun.state.map(), 'u/m/mutate/n')}); var check = {}, count = {}; gun.get('u/m/mutate/n').map().get('name').get(function(at,ev){ var e = at.err, v = at.put, f = at.get; @@ -1622,8 +1624,7 @@ describe('Gun', function(){ }); it('uncached synchronous map on mutate node uncached', function(done){ - var s = Gun.state.map();s.soul = 'u/m/mutate/n/u'; - gun.on('put', {$: gun, put: Gun.graph.ify({ + gun.on('test', {$: gun, put: Gun.graph.ify({ alice: {_:{'#':'umaliceo1'}, age: 26, name: "Alice", @@ -1634,7 +1635,7 @@ describe('Gun', function(){ name: "Bob", pet: {b:2, name: "Frisky"} } - }, s)}); + }, Gun.state.map(), 'u/m/mutate/n/u')}); var check = {}, count = {}; gun.get('u/m/mutate/n/u').map().on(function(v,f){ check[v.name] = f; @@ -1654,10 +1655,9 @@ describe('Gun', function(){ } }); setTimeout(function(){ - var s = Gun.state.map();s.soul = 'u/m/m/n/u/soul'; - gun.on('put', {$: gun, put: Gun.graph.ify({ + gun.on('test', {$: gun, put: Gun.graph.ify({ name: 'Alice Zzxyz' - }, s)}); + }, Gun.state.map(), 'u/m/m/n/u/soul')}); //console.debug.i=1;console.log("---------------"); gun.get('u/m/mutate/n/u').put({ alice: {'#':'u/m/m/n/u/soul'}, @@ -1679,8 +1679,7 @@ describe('Gun', function(){ }); it('uncached synchronous map on get mutate node uncached', function(done){ - var s = Gun.state.map();s.soul = 'u/m/p/mutate/n/u'; - gun.on('put', {$: gun, put: Gun.graph.ify({ + gun.on('test', {$: gun, put: Gun.graph.ify({ alice: {_:{'#':'umaliceo2'}, age: 26, name: "Alice", @@ -1691,7 +1690,7 @@ describe('Gun', function(){ name: "Bob", pet: {b:2, name: "Frisky"} } - }, s)}); + }, Gun.state.map(), 'u/m/p/mutate/n/u')}); var check = {}, count = {}; gun.get('u/m/p/mutate/n/u').map().get('name').on(function(v,f){ check[v] = f; @@ -1712,10 +1711,9 @@ describe('Gun', function(){ } }); setTimeout(function(){ - var s = Gun.state.map();s.soul = 'u/m/p/m/n/u/soul'; - gun.on('put', {$: gun, put: Gun.graph.ify({ + gun.on('test', {$: gun, put: Gun.graph.ify({ name: 'Alice Zzxyz', age: 34 - }, s)}); + }, Gun.state.map(), 'u/m/p/m/n/u/soul')}); gun.get('u/m/p/mutate/n/u').put({ alice: {'#':'u/m/p/m/n/u/soul'}, }); @@ -1729,8 +1727,7 @@ describe('Gun', function(){ }); it('uncached synchronous map on get node mutate node uncached', function(done){ - var s = Gun.state.map();s.soul = 'u/m/p/n/mutate/n/u'; - gun.on('put', {$: gun, put: Gun.graph.ify({ + gun.on('test', {$: gun, put: Gun.graph.ify({ alice: {_:{'#':'umaliceo3'}, age: 26, name: "Alice", @@ -1741,7 +1738,7 @@ describe('Gun', function(){ name: "Bob", pet: {b:2, name: "Frisky"} } - }, s)}); + }, Gun.state.map(), 'u/m/p/n/mutate/n/u')}); var check = {}, count = {}; gun.get('u/m/p/n/mutate/n/u').map().get('pet').on(function(v,f){ check[v.name] = f; @@ -1760,11 +1757,10 @@ describe('Gun', function(){ } }); setTimeout(function(){ - var s = Gun.state.map();s.soul = 'alice/fuzz/soul'; - gun.on('put', {$: gun, put: Gun.graph.ify({ + gun.on('test', {$: gun, put: Gun.graph.ify({ name: 'Alice Zzxyz', age: 34, pet: {c:3, name: "Fuzzball"} - }, s)}); + }, Gun.state.map(), 'alice/fuzz/soul')}); gun.get('u/m/p/n/mutate/n/u').put({ alice: {'#':'alice/fuzz/soul'}, }); @@ -2844,15 +2840,13 @@ describe('Gun', function(){ }); it('get node after recursive field', function(done){ - var s = Gun.state.map();s.soul = 'node/circle'; var bob = {age: 29, name: "Bob!"}; var cat = {name: "Fluffy", species: "kitty"}; var user = {bob: bob}; bob.pet = cat; cat.slave = bob; - gun.on('put', {$: gun, put: Gun.graph.ify(user, s)}); - //console.debug.i=1;console.log("-------------"); - gun.get(s.soul).get('bob').get('pet').get('slave').once(function(data){ + gun.on('test', {$: gun, put: Gun.graph.ify(user, Gun.state.map(), 'node/circle')}); + gun.get('node/circle').get('bob').get('pet').get('slave').once(function(data){ //clearTimeout(done.to); //setTimeout(function(){ //console.log("*****************", data);return; @@ -2945,7 +2939,9 @@ describe('Gun', function(){ list.set(gun.get('dave').put({name: "Dave", group: "awesome", married: true})); var check = {}, count = {}; - list.map().once(function(data, id){ + //console.log("==============================="); + //console.only.i=1; + list.map().on(function(data, id){ //console.log("***************", id, data); check[id] = data; count[id] = (count[id] || 0) + 1; @@ -3090,13 +3086,12 @@ describe('Gun', function(){ }); it('get get get any parallel', function(done){ - var s = Gun.state.map();s.soul = 'parallel'; - gun.on('put', {$: gun, put: Gun.graph.ify({ + gun.on('test', {$: gun, put: Gun.graph.ify({ bob: { age: 29, name: "Bob!" } - }, s)}); + }, Gun.state.map(), 'parallel')}); gun.get('parallel').get('bob').get('age').get(function(at, ev){ var err = at.err, data = at.put, field = at.get; //console.log("***** age", data, at.$._.ack);//return; @@ -3117,13 +3112,12 @@ describe('Gun', function(){ }); it('get get get any later', function(done){ - var s = Gun.state.map();s.soul = 'parallel/later'; - gun.on('put', {$: gun, put: Gun.graph.ify({ + gun.on('test', {$: gun, put: Gun.graph.ify({ bob: {_:{'#':'ddfsa'}, age: 29, name: "Bob!" } - }, s)}); + }, Gun.state.map(), 'parallel/later')}); gun.get('parallel/later').get('bob').get('age').get(function(at, ev){ var err = at.err, data = at.put, field = at.get; //console.log("***** age", data); @@ -3189,11 +3183,10 @@ describe('Gun', function(){ }); it('get any any', function(done){ - var s = Gun.state.map();s.soul = 'full'; - gun.on('put', {$: gun, put: Gun.graph.ify({ + gun.on('test', {$: gun, put: Gun.graph.ify({ hello: 'world', goodbye: 'mars' - }, s)}); + }, Gun.state.map(), 'full')}); gun.get('full').get(function(at, ev){ var err = at.err, data = at.$._.put || at.put, field = at.get; //console.log("*****1", data); @@ -3211,11 +3204,10 @@ describe('Gun', function(){ }); it('get any any later', function(done){ - var s = Gun.state.map();s.soul = 'full/later'; - gun.on('put', {$: gun, put: Gun.graph.ify({ + gun.on('test', {$: gun, put: Gun.graph.ify({ hello: 'world', goodbye: 'mars' - }, s)}); + }, Gun.state.map(), 'full/later')}); gun.get('full/later').get(function(at, ev){ var err = at.err, data = at.$._.put || at.put, field = at.get; //console.log("*****", data); @@ -3303,8 +3295,7 @@ describe('Gun', function(){ it('multiple times partial', function(done){ var gun = Gun(); - var s = Gun.state.map();s.soul = 'mult/times/part'; - gun.on('put', {$: gun, put: Gun.graph.ify({ + gun.on('test', {$: gun, put: Gun.graph.ify({ alias: { mark: { pub: {_:{'#':'PUB'}, @@ -3314,9 +3305,9 @@ describe('Gun', function(){ } } } - }, s)}); + }, Gun.state.map(), 'mult/times/part')}); - var app = gun.get(s.soul); + var app = gun.get('mult/times/part'); //console.debug.i=1;console.log("==================="); app.get('alias').get('mark').map().once(function(alias){ @@ -3457,6 +3448,7 @@ describe('Gun', function(){ list.get('message').put('hello world'); // outputs "message: hello world" list.get('message').put(null); // throws Uncaught TypeError: Cannot read property '#' of null }); + return; it('Check multi instance message passing', function(done){ try{ require('fs').unlinkSync('bdata') }catch(e){} @@ -3541,10 +3533,10 @@ describe('Gun', function(){ it('If chain cannot be called, ack', function(done){ var gun = Gun(), u; - gun.on('put', {$: gun, put: Gun.graph.ify({ + gun.on('test', {$: gun, put: Gun.graph.ify({ wat: 1, a: true - }, 'nl/app')}); + }, Gun.state.map(), 'nl/app')}); // prev had no state_map? var app = gun.get('nl/app'); @@ -3565,11 +3557,11 @@ describe('Gun', function(){ it('Chain on known nested object should ack', function(done){ var gun = Gun(), u; - gun.on('put', {$: gun, put: Gun.graph.ify({ + gun.on('test', {$: gun, put: Gun.graph.ify({ bar: { wat: 1 } - }, 'nl/app')}); + }, Gun.state.map(), 'nl/app')}); var app = gun.get('nl/app').get('bar'); @@ -3953,13 +3945,12 @@ describe('Gun', function(){ });return; it('get get any parallel', function(done){ - var s = Gun.state.map();s.soul = 'parallel/get/get'; - Gun.on('put', {$: gun, put: Gun.graph.ify({ + Gun.on('test', {$: gun, put: Gun.graph.ify({ bob: { age: 29, name: "Bob!" } - }, s)}); + }, Gun.state.map(), 'parallel/get/get')}); gun.get('parallel/get/get').path('bob').any(function(err, data, field, at, ev){ //console.log("***** 1", data); expect(data.age).to.be(29); @@ -3974,13 +3965,12 @@ describe('Gun', function(){ }); it('get get any parallel later', function(done){ - var s = Gun.state.map();s.soul = 'parallel/get/get/later'; - Gun.on('put', {$: gun, put: Gun.graph.ify({ + Gun.on('test', {$: gun, put: Gun.graph.ify({ bob: { age: 29, name: "Bob!" } - }, s)}); + }, Gun.state.map(), 'parallel/get/get/later')}); gun.get('parallel/get/get/later').path('bob').any(function(err, data, field, at, ev){ //console.log("***** 1", data); expect(data.age).to.be(29); @@ -3997,13 +3987,12 @@ describe('Gun', function(){ }); it('get get any none', function(done){ - var s = Gun.state.map();s.soul = 'get/get/none'; - Gun.on('put', {$: gun, put: Gun.graph.ify({ + Gun.on('test', {$: gun, put: Gun.graph.ify({ alice: { age: 31, name: "alice" } - }, s)}); + }, Gun.state.map(), 'get/get/none')}); var c = 0, s = 0; gun.get('get/get/none').path('bob').any(function(err, data, field, at, ev){ //console.log("***** 1", data); @@ -4025,13 +4014,12 @@ describe('Gun', function(){ }); it('get get any none later', function(done){ - var s = Gun.state.map();s.soul = 'get/get/none/later'; - Gun.on('put', {$: gun, put: Gun.graph.ify({ + Gun.on('test', {$: gun, put: Gun.graph.ify({ alice: { age: 31, name: "alice" } - }, s)}); + }, Gun.state.map(), 'get/get/none/later')}); var c = 0; gun.get('get/get/none/later').path('bob').any(function(err, data, field, at, ev){ //console.log("***** 1", data); @@ -4051,10 +4039,9 @@ describe('Gun', function(){ }); it('get get primitive get any', function(done){ - var s = Gun.state.map();s.soul = 'get/get/prim'; - Gun.on('put', {$: gun, put: Gun.graph.ify({ + Gun.on('test', {$: gun, put: Gun.graph.ify({ bob: "is awesome" - }, s)}); + }, Gun.state.map(), 'get/get/prim')}); gun.get('get/get/prim').path('bob').path('age').any(function(err, data, field, at, ev){ //console.log("***** 1", data); expect(data).to.be(undefined); @@ -4067,10 +4054,9 @@ describe('Gun', function(){ }); it('get put any', function(done){ - var s = Gun.state.map();s.soul = 'get/put/any'; - Gun.on('put', {$: gun, put: Gun.graph.ify({ + Gun.on('test', {$: gun, put: Gun.graph.ify({ here: "we go" - }, s)}); + }, Gun.state.map(), 'get/put/any')}); //console.debug.i=1;console.log("---------------"); gun.get('get/put/any') .put({}) @@ -4081,10 +4067,9 @@ describe('Gun', function(){ }); return; it('get any, get put any', function(done){ - var s = Gun.state.map();s.soul = 'get/any/get/put/any'; - Gun.on('put', {$: gun, put: Gun.graph.ify({ + Gun.on('test', {$: gun, put: Gun.graph.ify({ here: "we go" - }, s)}); + }, Gun.state.map(), 'get/any/get/put/any')}); gun.get('get/any/get/put/any') .any(function(err, data, field, at, ev){ if(done.first){ return } // it is okay for `any` to get called multiple times. @@ -4110,8 +4095,7 @@ describe('Gun', function(){ }); it('mutate pointer to primitive deep on', function(done){ - var s = Gun.state.map();s.soul = 'change/pointer'; - Gun.on('put', {$: gun, put: Gun.graph.ify({ + Gun.on('test', {$: gun, put: Gun.graph.ify({ bob: { age: 29, name: "Bob!", @@ -4120,7 +4104,7 @@ describe('Gun', function(){ species: "kitty" } } - }, s)}); + }, Gun.state.map(), 'change/pointer')}); gun.get('change/pointer').path('bob').path('pet').any(function(err, data, f, at, ev){ //console.log("***", data);return setTimeout(function(){asdf},500); if(done.c){ @@ -4159,8 +4143,7 @@ describe('Gun', function(){ }); it('get only soul', function(done){ - var s = Gun.state.map();s.soul = 'only/soul'; - Gun.on('put', {$: gun, put: Gun.graph.ify({ + Gun.on('test', {$: gun, put: Gun.graph.ify({ bob: { age: 29, name: "Bob!", @@ -4169,7 +4152,7 @@ describe('Gun', function(){ species: "kitty" } } - }, s)}); + }, Gun.state.map(), 'only/soul')}); gun.get('only/soul')/*.path('bob')*/.any(function(err, data){ expect(Gun.obj.empty(data, '_')).to.be.ok(); done(); @@ -4177,8 +4160,7 @@ describe('Gun', function(){ }); it('get path only soul', function(done){ - var s = Gun.state.map();s.soul = 'only/p/soul'; - Gun.on('put', {$: gun, put: Gun.graph.ify({ + Gun.on('test', {$: gun, put: Gun.graph.ify({ bob: { age: 29, name: "Bob!", @@ -4187,7 +4169,7 @@ describe('Gun', function(){ species: "kitty" } } - }, s)}); + }, Gun.state.map(), 'only/p/soul')}); gun.get('only/p/soul').path('bob').any(function(err, data){ //console.log("*********", err, data); expect(Gun.val.link.is(data)).to.be.ok(); @@ -4197,8 +4179,7 @@ describe('Gun', function(){ }); it('mutate pointer to self', function(done){ - var s = Gun.state.map();s.soul = 'change/pointer/point'; - Gun.on('put', {$: gun, put: Gun.graph.ify({ + Gun.on('test', {$: gun, put: Gun.graph.ify({ bob: { age: 29, name: "Bob!", @@ -4207,7 +4188,7 @@ describe('Gun', function(){ species: "kitty" } } - }, s)}); + }, Gun.state.map(), 'change/pointer/point')}); gun.get('change/pointer/point').path('bob').any(function(err, data){ if(done.c){ expect(data.age).to.be(30); @@ -4228,8 +4209,7 @@ describe('Gun', function(){ },400); }); it('mutate pointer to self deep', function(done){ - var s = Gun.state.map();s.soul = 'change/pointer/point/deep'; - Gun.on('put', {$: gun, put: Gun.graph.ify({ + Gun.on('test', {$: gun, put: Gun.graph.ify({ bob: { age: 29, name: "Bob!", @@ -4238,7 +4218,7 @@ describe('Gun', function(){ species: "kitty" } } - }, s)}); + }, Gun.state.map(), 'change/pointer/point/deep')}); gun.get('change/pointer/point/deep').path('bob').any(function(err, data){ //console.log("***", data); if(done.c){ @@ -4259,8 +4239,7 @@ describe('Gun', function(){ }); it('mutate pointer to primitive after any', function(done){ - var s = Gun.state.map();s.soul = 'change/pointer/to/prime'; - Gun.on('put', {$: gun, put: Gun.graph.ify({ + Gun.on('test', {$: gun, put: Gun.graph.ify({ bob: {_: {'#': 'asdffdsa'}, age: 29, name: "Bob!", @@ -4269,7 +4248,7 @@ describe('Gun', function(){ species: "kitty" } } - }, s)}); + }, Gun.state.map(), 'change/pointer/to/prime')}); var bob = gun.get('asdffdsa').any(function(err, data){ //console.log("***", data); }); @@ -4294,8 +4273,7 @@ describe('Gun', function(){ }); it('mutate pointer to primitive after any deep', function(done){ - var s = Gun.state.map();s.soul = 'change/pointer/to/prime/deep'; - Gun.on('put', {$: gun, put: Gun.graph.ify({ + Gun.on('test', {$: gun, put: Gun.graph.ify({ bob: { age: 29, name: "Bob!", @@ -4304,7 +4282,7 @@ describe('Gun', function(){ species: "kitty" } } - }, s)}); + }, Gun.state.map(), 'change/pointer/to/prime/deep')}); var cat = gun.get('sadffads').any(function(err, data){ //console.log("***", data); }); @@ -4328,8 +4306,7 @@ describe('Gun', function(){ }); return; it.only('mutate pointer to another pointer after any', function(done){ - var s = Gun.state.map();s.soul = 'change/pointer/to/pointer'; - Gun.on('put', {$: gun, put: Gun.graph.ify({ + Gun.on('test', {$: gun, put: Gun.graph.ify({ bob: {_: {'#': 'dafssfad'}, age: 29, name: "Bob!", @@ -4338,7 +4315,7 @@ describe('Gun', function(){ species: "kitty" } } - }, s)}); + }, Gun.state.map(), 'change/pointer/to/pointer')}); var bob = gun.get('dafssfad').any(function(err, data){ console.log("***", data); }); @@ -4929,9 +4906,7 @@ describe('Gun', function(){ }); it('get get not', function(done){ - var s = Gun.state.map(); - s.soul = 'a'; - Gun.on('put', {$: gun, put: Gun.graph.ify({b: 1, c: 2}, s)}); + Gun.on('test', {$: gun, put: Gun.graph.ify({b: 1, c: 2}, Gun.state.map(), 'a')}); function cb(e,d,f,a){ if('b' === f && 1 === d){ done.b = true; diff --git a/test/mocha.html b/test/mocha.html index 13bcf7a4..e5af40c4 100644 --- a/test/mocha.html +++ b/test/mocha.html @@ -17,12 +17,7 @@ - - + diff --git a/test/rad/bench.js b/test/rad/bench.js index d9c00b6e..c03a6a80 100644 --- a/test/rad/bench.js +++ b/test/rad/bench.js @@ -1,6 +1,6 @@ var Gun = (typeof window !== "undefined")? window.Gun : require('../../../gun/gun'); -var Radisk = (Gun.window && window.Radisk) || require('../../../gun/lib/radiskip'); +var Radisk = (Gun.window && window.Radisk) || require('../../../gun/lib/radisk'); Gun.TESTING = true; try{localStorage.clear()}catch(e){} try{indexedDB.deleteDatabase('radatatest');}catch(e){} diff --git a/test/rad/browser.html b/test/rad/browser.html index 087c3cfd..3821b567 100644 --- a/test/rad/browser.html +++ b/test/rad/browser.html @@ -4,8 +4,7 @@ - - + diff --git a/test/rad/rad.js b/test/rad/rad.js index eef442aa..28c91dba 100644 --- a/test/rad/rad.js +++ b/test/rad/rad.js @@ -43,7 +43,7 @@ var names = ["Adalard","Adora","Aia","Albertina","Alfie","Allyn","Amabil","Ammam //console.log("HYPER TEST");var z = 10000; while(--z){ names.push(Gun.text.random(7)) }this.timeout(9000); - describe.only('Radix', function(){ + describe('Radix', function(){ var radix = Radix(); it('unit', function(){ @@ -264,6 +264,7 @@ var names = ["Adalard","Adora","Aia","Albertina","Alfie","Allyn","Amabil","Ammam if(v.indexOf(find) == 0){ all[v] = true } }); rad(find, function(err, data, info){ + expect(data).to.be.ok(); Radix.map(data, function(v,k){ delete all[find+k]; }); @@ -312,12 +313,22 @@ var names = ["Adalard","Adora","Aia","Albertina","Alfie","Allyn","Amabil","Ammam }) }); + it('read one', function(done){ + //gun.get('names').get({'.': {'*': find}, '%': 1000 * 100}).once().map().once(function(data, key){ + gun.get('names').get('stu').once(function(data, key){ + expect(data.name).to.be.ok(); + expect(data.age).to.be.ok(); + done(); + }); + }); + it('read contacts', function(done){ var all = {}, find = 'm', to; names.forEach(function(v){ v = v.toLowerCase(); if(v.indexOf(find) == 0){ all[v] = true } }); + //console.log("<<<<<<<<<"); gun.get('names').get({'.': {'*': find}, '%': 1000 * 100}).once().map().once(function(data, key){ expect(data.name).to.be.ok(); expect(data.age).to.be.ok(); @@ -328,6 +339,7 @@ var names = ["Adalard","Adora","Aia","Albertina","Alfie","Allyn","Amabil","Ammam done(); },100); }); + //console.log(">>>>>>>>>"); }); it('read contacts again', function(done){ diff --git a/test/sea/sea.js b/test/sea/sea.js index 1a166804..dca8f6fd 100644 --- a/test/sea/sea.js +++ b/test/sea/sea.js @@ -197,20 +197,20 @@ describe('SEA', function(){ expect(dec.priv).to.be(okey.priv); expect(dec.epriv).to.be(okey.epriv); - var gun = Gun(), tmp = Gun.node.soul(old); + var gun = Gun({super: true}), tmp = Gun.node.soul(old); var graph = {}; graph[tmp] = old; var alias = await SEA.verify(old.alias, false); expect(alias).to.be('bob'); alias = Gun.state.ify({}, tmp, 1, Gun.val.rel.ify(tmp), tmp = '~@'+alias); graph[tmp] = alias; - gun.on('put', {$: gun, put: graph}); + gun.on('test', {$: gun, put: graph}); var use = gun.user(); use.auth('bob', 'test123', function(ack){ expect(ack.err).to.not.be.ok(); done(); }); - }())}) + }())}); it('legacy []', function(done){ (async function(){ var pw = 'test123'; @@ -218,14 +218,14 @@ describe('SEA', function(){ var old = JSON.parse(atob("eyJfIjp7IiMiOiJ+VThkS0dySFJhX01sMFZ1YlR5OUZBYTlQS1ZlYlh0eTFjS05zWWxnYjduNC5QeVd5cUVVb0ZpYVduUElOV0Nad0xBbzFobjN1MldPWTU3SzZHZnpsNjhVIiwiPiI6eyJwdWIiOjE1NDY5MDI1MDQ5NzksImFsaWFzIjoxNTQ2OTAyNTA0OTc5LCJlcHViIjoxNTQ2OTAyNTA0OTc5LCJhdXRoIjoxNTQ2OTAyNTA0OTc5fX0sInB1YiI6IlU4ZEtHckhSYV9NbDBWdWJUeTlGQWE5UEtWZWJYdHkxY0tOc1lsZ2I3bjQuUHlXeXFFVW9GaWFXblBJTldDWndMQW8xaG4zdTJXT1k1N0s2R2Z6bDY4VSIsImFsaWFzIjoiU0VBe1wibVwiOltcIn5VOGRLR3JIUmFfTWwwVnViVHk5RkFhOVBLVmViWHR5MWNLTnNZbGdiN240LlB5V3lxRVVvRmlhV25QSU5XQ1p3TEFvMWhuM3UyV09ZNTdLNkdmemw2OFVcIixcImFsaWFzXCIsXCJhbGljZVwiLDE1NDY5MDI1MDQ5NzldLFwic1wiOlwienpuaGtIZjhZdFpZM2lGd3FVd0lJUldMTjhZMmlHbmNkcnVTaStGNDNmU1BLYWpSZlI0VzhXVHM4bElSMDBndGJmTWJxS0NjQkpGN3VNSkdGRC9WV2c9PVwifSIsImVwdWIiOiJTRUF7XCJtXCI6W1wiflU4ZEtHckhSYV9NbDBWdWJUeTlGQWE5UEtWZWJYdHkxY0tOc1lsZ2I3bjQuUHlXeXFFVW9GaWFXblBJTldDWndMQW8xaG4zdTJXT1k1N0s2R2Z6bDY4VVwiLFwiZXB1YlwiLFwiRkRzM1VvNTNFZEp6eFNocEpDaVctRGZPQ3lUS0M2U3cxeS1PZVJxam5ZRS5xVGdyYTlFQk1maEpNdVlMVmNaejRZYklLRm85enNBMHpMcV82dEVPMHI0XCIsMTU0NjkwMjUwNDk3OV0sXCJzXCI6XCJPZzRVVjY4OTluSjE4dC9ybWVnV0lkdnNqN01KaEpFc29ranZYQmdteVVRUXVNVjFTdnh4cXJqOFoyV1o2Q25XSkZnTlVDbEVYYWxuMURjUFE3M1R6UT09XCJ9IiwiYXV0aCI6IlNFQXtcIm1cIjpbXCJ+VThkS0dySFJhX01sMFZ1YlR5OUZBYTlQS1ZlYlh0eTFjS05zWWxnYjduNC5QeVd5cUVVb0ZpYVduUElOV0Nad0xBbzFobjN1MldPWTU3SzZHZnpsNjhVXCIsXCJhdXRoXCIsXCJ7XFxcImVrXFxcIjpcXFwiU0VBe1xcXFxcXFwiY3RcXFxcXFxcIjpcXFxcXFxcIi94ZnNPdVNkQUtrNkJiR00zbUV6MnVlSjI3Y0tJNThYMEtUL1FsaExSZXpWcjRkNzVZb2M5QlZNRjkzejl4QXI4N080S2FDNjJUWGVoeERQN0FFa2V4N1paaEpYL2hsVm9kK1FIcVFaaUZMK2lVQzFvL2hpUEJGWElBZmtINGRrcklGOFdqcEVaU3NIVmRSOVRhY2ZzbTB3aHN5NGJXN1ZLSEUySGc9PVxcXFxcXFwiLFxcXFxcXFwiaXZcXFxcXFxcIjpcXFxcXFxcIjhWekduTStEc1lTUktIU3Z4cSszTGc9PVxcXFxcXFwiLFxcXFxcXFwic1xcXFxcXFwiOlxcXFxcXFwibVVSSlJ4TzUvdXM9XFxcXFxcXCJ9XFxcIixcXFwic1xcXCI6XFxcImE1SlA3VFpuVE9jYjEwMGJOejlscEU4dnpqcUE3TWl0NHcwN3pjQTdIOFV0bml1WnVHSmdpZnNNQlFNSGdRdE5cXFwifVwiLDE1NDY5MDI1MDQ5NzldLFwic1wiOlwiSGFzMytJaHFEZTYyN016cElXZVE1cVFrZ2NOMlk3WHRpNGw0TFU3T2JyaktxSlBnSllrVWE2bk9YdlRmQkFzV1BPVzVnemh4Q2RPVGNFQm5icWlpWXc9PVwifSJ9")); var okey = {"pub":"U8dKGrHRa_Ml0VubTy9FAa9PKVebXty1cKNsYlgb7n4.PyWyqEUoFiaWnPINWCZwLAo1hn3u2WOY57K6Gfzl68U","epub":"FDs3Uo53EdJzxShpJCiW-DfOCyTKC6Sw1y-OeRqjnYE.qTgra9EBMfhJMuYLVcZz4YbIKFo9zsA0zLq_6tEO0r4","priv":"jMy7WfcldJ4esZEijAj4LTb99smtY_H0yKJLemJl2HI","epriv":"1DszMh-85pGTPLYtRunG-Q-xB78AE4k07PPkbedYYwk"} - var gun = Gun(), tmp = Gun.node.soul(old); + var gun = Gun({super: true}), tmp = Gun.node.soul(old); var graph = {}; graph[tmp] = old; var alias = SEA.opt.unpack(await SEA.verify(old.alias, false), 'alias', old); expect(alias).to.be('alice'); alias = Gun.state.ify({}, tmp, 1, Gun.val.rel.ify(tmp), tmp = '~@'+alias); graph[tmp] = alias; - gun.on('put', {$: gun, put: graph}); + gun.on('test', {$: gun, put: graph}); var use = gun.user(); use.auth('alice', 'test123', function(ack){ expect(ack.err).to.not.be.ok(); @@ -277,8 +277,30 @@ describe('SEA', function(){ }); describe('User', function(){ + var gun = Gun(), gtmp; + + it('test', function(done){ + var g = Gun(); + user = g.user(); + var gid; + SEA.pair(function(p){ + user.is = user._.sea = p; + gtmp = gid = 'test~'+p.pub; + g.get(gid).put({yo: 'hi'}, function(ack){ + var data = SEA.opt.parse(g._.graph[gid].yo); + expect(data[':']).to.be('hi'); + expect(data['~']).to.be.ok(); + g.get(gid).get('yo').once(function(r){ + expect(r).to.be('hi'); + user.leave(); + done(); + }) + }) + }) + }); + it('is instantiable', function(done){ - gun = Gun(); + user.leave(); user = gun.user(); done(); }) @@ -289,7 +311,7 @@ describe('SEA', function(){ expect(ack.err).to.not.be.ok(); done(); }) - }) + }); it('login users', function(done){ user.auth('carl', 'test123', function(ack){ @@ -347,6 +369,7 @@ describe('SEA', function(){ var ref = user.get('who').get('all').set(msg); user.get('who').get('said').set(ref); user.get('who').get('said').map().once(function(data){ + //console.log("*****", data); expect(data.what).to.be.ok(); done(); }) @@ -356,6 +379,7 @@ describe('SEA', function(){ it('set user ref null override', function(done){ this.timeout(9000); var gun = Gun(); + //user.leave(); var user = gun.user(); var msg = {what: 'hello world'}; user.create('xavier', 'password'); From d6dae4871e0166555b7885dca10c32e8b7c420fb Mon Sep 17 00:00:00 2001 From: Mark Nadal Date: Mon, 10 Feb 2020 15:22:07 -0800 Subject: [PATCH 31/76] p v b --- package-lock.json | 10 +++++++--- package.json | 3 +-- 2 files changed, 8 insertions(+), 5 deletions(-) diff --git a/package-lock.json b/package-lock.json index aff7ee5f..1d114ffc 100644 --- a/package-lock.json +++ b/package-lock.json @@ -1,6 +1,6 @@ { "name": "gun", - "version": "0.2019.1211", + "version": "0.2020.116", "lockfileVersion": 1, "requires": true, "dependencies": { @@ -165,6 +165,7 @@ "version": "1.3.1", "resolved": "https://registry.npmjs.org/b64-lite/-/b64-lite-1.3.1.tgz", "integrity": "sha512-A3n/YWszeQeUeKKy+XRfsDW/s/+1cvD110HYA4uXtkNswNLd1MxrNblYt2lgKisUS3UfPPTaXhvFegD+l4YuOA==", + "optional": true, "requires": { "base-64": "^0.1.0" } @@ -193,7 +194,8 @@ "base-64": { "version": "0.1.0", "resolved": "https://registry.npmjs.org/base-64/-/base-64-0.1.0.tgz", - "integrity": "sha1-eAqZyE59YAJgNhURxId2E78k9rs=" + "integrity": "sha1-eAqZyE59YAJgNhURxId2E78k9rs=", + "optional": true }, "base64-arraybuffer": { "version": "0.1.5", @@ -897,7 +899,8 @@ "js-tokens": { "version": "4.0.0", "resolved": "https://registry.npmjs.org/js-tokens/-/js-tokens-4.0.0.tgz", - "integrity": "sha512-RdJUflcE3cUzKiMqQgsCu06FPu9UdIJO0beYbPhHN4k6apgJtifcoCtT9bcxOpYBtpD2kCM6Sbzg4CausW/PKQ==" + "integrity": "sha512-RdJUflcE3cUzKiMqQgsCu06FPu9UdIJO0beYbPhHN4k6apgJtifcoCtT9bcxOpYBtpD2kCM6Sbzg4CausW/PKQ==", + "optional": true }, "js-yaml": { "version": "3.13.1", @@ -953,6 +956,7 @@ "version": "1.4.0", "resolved": "https://registry.npmjs.org/loose-envify/-/loose-envify-1.4.0.tgz", "integrity": "sha512-lyuxPGr/Wfhrlem2CL/UcnUc1zcqKAImBDzukY7Y5F/yQiNdko6+fRLevlw1HgMySw7f611UIY408EtxRSoK3Q==", + "optional": true, "requires": { "js-tokens": "^3.0.0 || ^4.0.0" } diff --git a/package.json b/package.json index e10efe53..4154ae90 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "gun", - "version": "0.2020.116", + "version": "0.2020.201", "description": "A realtime, decentralized, offline-first, graph data synchronization engine.", "main": "index.js", "browser": "browser.js", @@ -69,7 +69,6 @@ "mocha": "^6.2.0", "panic-manager": "^1.2.0", "panic-server": "^1.1.1", - "0x": "^4.9.1", "uglify-js": "^3.6.0" } } From bc40109e614955a78c1e1be21cf9793e54bdc8bb Mon Sep 17 00:00:00 2001 From: Mark Nadal Date: Mon, 10 Feb 2020 17:48:39 -0800 Subject: [PATCH 32/76] fixup --- axe.js | 2 +- examples/http.js | 6 +++--- examples/todo/index.html | 25 ------------------------- gun.js | 2 +- lib/crashed.js | 32 ++++++++++++++++++++++++++++++++ trace.js | 27 --------------------------- 6 files changed, 37 insertions(+), 57 deletions(-) create mode 100644 lib/crashed.js delete mode 100644 trace.js diff --git a/axe.js b/axe.js index 19594d2c..16566f0f 100644 --- a/axe.js +++ b/axe.js @@ -109,7 +109,7 @@ at.on('in', input); at.on('in2', input); function input(msg){ - var to = this.to, peer = (msg._||'').via || mesh.leap; // warning! mesh.leap could be buggy! + var to = this.to, peer = (msg._||'').via; // warning! mesh.leap could be buggy! var dht = opt.dht; var routes = axe.routes || (axe.routes = {}); // USE RAD INSTEAD! TMP TESTING! var get = msg.get, hash, tmp; diff --git a/examples/http.js b/examples/http.js index dba51779..9c011f49 100644 --- a/examples/http.js +++ b/examples/http.js @@ -1,8 +1,8 @@ ;(function(){ - /*var cluster = require('cluster'); + var cluster = require('cluster'); if(cluster.isMaster){ - return cluster.fork() && cluster.on('exit', function(){ cluster.fork() }); - }*/ + return cluster.fork() && cluster.on('exit', function(){ cluster.fork(); require('../lib/crashed'); }); + } var fs = require('fs'); var config = { port: process.env.OPENSHIFT_NODEJS_PORT || process.env.VCAP_APP_PORT || process.env.PORT || process.argv[2] || 8765 }; diff --git a/examples/todo/index.html b/examples/todo/index.html index c8a5d305..3764edc4 100644 --- a/examples/todo/index.html +++ b/examples/todo/index.html @@ -113,31 +113,6 @@ - -