From 3c9727ba2f4d94bc3f96c0fddf08cfd0d13c2073 Mon Sep 17 00:00:00 2001 From: Mark Nadal Date: Fri, 2 Jul 2021 06:24:03 -0700 Subject: [PATCH] rad lex once map once --- gun.js | 30 ++++++++++++++++++++---------- lib/radisk.js | 16 ++++++++++++---- lib/rfs.js | 15 ++++++++++----- lib/rindexed.js | 16 +++++++++++++--- lib/store.js | 20 +++++++++++++------- sea.js | 4 ++-- test/rad/rad.js | 45 +++++++++++++++++++++++++++++++++++++++++++-- test/sea/sea.js | 15 +++++++++++---- 8 files changed, 124 insertions(+), 37 deletions(-) diff --git a/gun.js b/gun.js index 691e40ab..3b974f2f 100644 --- a/gun.js +++ b/gun.js @@ -624,7 +624,7 @@ (back.ask || (back.ask = {}))[at.get] = msg.$._; // TODO: PERFORMANCE? More elegant way? return back.on('out', msg); } - msg = {get: {}, $: at.$}; + msg = {get: at.lex? msg.get : {}, $: at.$}; return back.on('out', msg); } (at.ask || (at.ask = {}))[''] = at; //at.ack = at.ack || -1; @@ -639,11 +639,17 @@ function input(msg, cat){ cat = cat || this.as; // TODO: V8 may not be able to optimize functions with different parameter calls, so try to do benchmark to see if there is any actual difference. var root = cat.root, gun = msg.$ || (msg.$ = cat.$), at = (gun||'')._ || empty, tmp = msg.put||'', soul = tmp['#'], key = tmp['.'], change = (u !== tmp['='])? tmp['='] : tmp[':'], state = tmp['>'] || -Infinity, sat; // eve = event, at = data at, cat = chain at, sat = sub at (children chains). - if(tmp && tmp._ && tmp._['#']){ // convert from old format - return setTimeout.each(Object.keys(tmp).sort(), function(k){ // TODO: .keys( is slow // BUG? ?Some re-in logic may depend on this being sync? - if('_' == k || !(state = state_is(tmp, k))){ return } - cat.on('in', {$: gun, put: {'#': tmp._['#'], '.': k, ':': tmp[k], '>': state}}); - }); + if(u !== msg.put && (u === tmp['#'] || u === tmp['.'] || (u === tmp[':'] && u === tmp['=']) || u === tmp['>'])){ // convert from old format + if(!valid(tmp)){ + if(!(soul = ((tmp||'')._||'')['#'])){ console.log("chain not yet supported for", tmp, '...', msg, cat); return; } + gun = cat.root.$.get(soul); + return setTimeout.each(Object.keys(tmp).sort(), function(k){ // TODO: .keys( is slow // BUG? ?Some re-in logic may depend on this being sync? + if('_' == k || u === (state = state_is(tmp, k))){ return } + cat.on('in', {$: gun, put: {'#': soul, '.': k, '=': tmp[k], '>': state}, VIA: msg}); + }); + } + cat.on('in', {$: at.back.$, put: {'#': soul = at.back.soul, '.': key = at.has || at.get, '=': tmp, '>': state_is(at.back.put, key)}, via: msg}); // TODO: This could be buggy! It assumes/approxes data, other stuff could have corrupted it. + return; } if((msg.seen||'')[cat.id]){ return } (msg.seen || (msg.seen = function(){}))[cat.id] = cat; // help stop some infinite loops @@ -856,8 +862,9 @@ if(Object.plain(key)){ gun = this; if(tmp = ((tmp = key['#'])||'')['='] || tmp){ return gun.get(tmp) } - gun._.lex = key; - return gun; + (tmp = gun.chain()._).lex = key; // LEX: // TODO! Consider making this only a `.map(` thing? + gun.on('in', function(eve){ this.to.next(eve); tmp.on('in', eve) }); // should filter here but ^ + return tmp.$; } else { (as = this.chain())._.err = {err: Gun.log('Invalid get request!', key)}; // CLEAN UP if(cb){ cb.call(as, as._.err) } @@ -1019,7 +1026,7 @@ } setTimeout.each(Object.keys(stun), function(cb){ if(cb = stun[cb]){cb()} }); // resume the stunned reads // Any perf reasons to CPU schedule this .keys( ? }).hatch = tmp; // this is not official yet ^ - //console.log("PUT!", as.run, as.graph); + //console.only(1, "PUT", as.run, as.graph); (as.via._).on('out', {put: as.out = as.graph, opt: as.opt, '#': ask, _: tmp}); } @@ -1200,7 +1207,8 @@ if(u === next){ return } if(data === next){ return chain._.on('in', msg) } if(Gun.is(next)){ return chain._.on('in', next._) } - chain._.on('in', {get: key, put: {'=':next}}); + var tmp = {}; Object.keys(msg.put).forEach(function(k){ tmp[k] = msg.put[k] }, tmp); tmp['='] = next; + chain._.on('in', {get: key, put: tmp}); }); return chain; } @@ -1637,7 +1645,9 @@ data = Gun.state.ify({}, tmp, Gun.state.is(data, tmp), data[tmp], soul); } if(data){ (tmp = {})[soul] = data } // back into a graph. + //setTimeout(function(){ root.on('in', {'@': msg['#'], put: tmp, lS:1});// || root.$}); + //}, Math.random() * 10); // FOR TESTING PURPOSES! }); root.on('put', function(msg){ diff --git a/lib/radisk.js b/lib/radisk.js index 2691db08..db3db69c 100644 --- a/lib/radisk.js +++ b/lib/radisk.js @@ -62,6 +62,7 @@ s.file = file || (file = opt.code.from); DBG && (DBG = DBG[file] = DBG[file] || {}); DBG && (DBG.sf = DBG.sf || +new Date); + //console.only.i && console.log('found', file); if(tmp = r.disk[file]){ s.mix(u, tmp); return } r.parse(file, s.mix, u, DBG); } @@ -86,6 +87,7 @@ cb = null; } DBG && (DBG.st = DBG.st || +new Date); + //console.only.i && console.log('mix', disk.Q); if(disk.Q){ cb && disk.Q.push(cb); return } disk.Q = (cb? [cb] : []); disk.to = setTimeout(s.write, opt.until); } @@ -97,6 +99,7 @@ delete disk.Q; delete r.disk[file]; delete disk.tags; + //console.only.i && console.log('write', file, disk, 'was saving:', key, data); r.write(file, disk, s.ack, u, DBG); } s.ack = function(err, ok){ @@ -131,6 +134,7 @@ cb || (cb = function(err, ok){ // test delete! if(!err){ return } }); + //console.only.i && console.log('save', key); r.find(key, s.find); } r.disk = {}; @@ -156,12 +160,15 @@ r.disk[file = rad.file || f.file || file] = rad; var S = +new Date; DBG && (DBG.wd = S); + //console.only.i && console.log('add', file); r.find.add(file, function add(err){ DBG && (DBG.wa = +new Date); if(err){ cb(err); return } + //console.only.i && console.log('disk', file, text); opt.store.put(ename(file), text, function safe(err, ok){ DBG && (DBG.wp = +new Date); console.STAT && console.STAT(S, ST = +new Date - S, "wrote disk", JSON.stringify(file), ++RWC, 'total all writes.'); + //console.only.i && console.log('done', err, ok || 1, cb); cb(err, ok || 1); if(!rad.Q){ delete r.disk[file] } // VERY IMPORTANT! Clean up memory, but not if there is already queued writes on it! }); @@ -216,6 +223,7 @@ } f.text += enc; } + //console.only.i && console.log('writing'); if(opt.jsonify){ r.write.jsonify(f, rad, cb, o, DBG); return } // temporary testing idea if(!Radix.map(rad, f.each, true)){ f.write() } } @@ -312,7 +320,7 @@ // assume in memory for now, since both write/read already call r.find which will init it. r.find(key, function(file){ if((file || (file = opt.code.from)) === info.file){ return } - var id = Gun.text.random(3); + var id = (''+Math.random()).slice(-3); puff(function(){ r.save(key, val, function ack(err, ok){ if(err){ r.save(key, val, ack); return } // ad infinitum??? @@ -531,10 +539,10 @@ while(i != -1){ t += s; i = d.indexOf(s, i+1) } return t + '"' + d + s; } else - if(d && d['#'] && (tmp = Gun.val.link.is(d))){ + if(d && d['#'] && 1 == Object.keys(d).length){ return t + '#' + tmp + t; } else - if(Gun.num.is(d)){ + if('number' == typeof d){ return t + '+' + (d||0) + t; } else if(null === d){ @@ -561,7 +569,7 @@ return d; } else if('#' === p){ - return Gun.val.link.ify(d); + return {'#':d}; } else if('+' === p){ if(0 === d.length){ diff --git a/lib/rfs.js b/lib/rfs.js index e61d2f58..7ab32697 100644 --- a/lib/rfs.js +++ b/lib/rfs.js @@ -14,17 +14,22 @@ function Store(opt){ // TODO!!! ADD ZLIB INFLATE / DEFLATE COMPRESSION! store.put = function(file, data, cb){ - puts[file] = data; var random = Math.random().toString(36).slice(-3); + puts[file] = {id: random, data: data}; var tmp = opt.file+'-'+file+'-'+random+'.tmp'; fs.writeFile(tmp, data, function(err, ok){ - delete puts[file]; - if(err){ return cb(err) } - move(tmp, opt.file+'/'+file, cb); + if(err){ + if(random === (puts[file]||'').id){ delete puts[file] } + return cb(err); + } + move(tmp, opt.file+'/'+file, function(err, ok){ + if(random === (puts[file]||'').id){ delete puts[file] } + cb(err, ok || !err); + }); }); }; store.get = function(file, cb){ var tmp; // this took 3s+? - if(tmp = puts[file]){ cb(u, tmp); return } + if(tmp = puts[file]){ cb(u, tmp.data); return } fs.readFile(opt.file+'/'+file, function(err, data){ if(err){ if('ENOENT' === (err.code||'').toUpperCase()){ diff --git a/lib/rindexed.js b/lib/rindexed.js index d3adda27..d3f3af7d 100644 --- a/lib/rindexed.js +++ b/lib/rindexed.js @@ -1,11 +1,20 @@ ;(function(){ - +/* // from @jabis +if (navigator.storage && navigator.storage.estimate) { + const quota = await navigator.storage.estimate(); + // quota.usage -> Number of bytes used. + // quota.quota -> Maximum number of bytes available. + const percentageUsed = (quota.usage / quota.quota) * 100; + console.log(`You've used ${percentageUsed}% of the available storage.`); + const remaining = quota.quota - quota.usage; + console.log(`You can write up to ${remaining} more bytes.`); +} +*/ function Store(opt){ opt = opt || {}; opt.file = String(opt.file || 'radata'); var db = null, u; - - try{opt.indexedDB = opt.indexedDB || indexedDB}catch(e){} + try{opt.indexedDB = opt.indexedDB || Store.indexedDB || indexedDB}catch(e){} try{if(!opt.indexedDB || 'file:' == location.protocol){ var store = {}, s = {}; store.put = function(f, d, cb){ s[f] = d; cb(null, 1) }; @@ -53,6 +62,7 @@ if(typeof window !== "undefined"){ (Store.window = window).RindexedDB = Store; + Store.indexedDB = window.indexedDB; // safari bug } else { try{ module.exports = Store }catch(e){} } diff --git a/lib/store.js b/lib/store.js index 0e990209..d28c2a93 100644 --- a/lib/store.js +++ b/lib/store.js @@ -7,7 +7,7 @@ Gun.on('create', function(root){ if(false === opt.rad || false === opt.radisk){ return } var Radisk = (Gun.window && Gun.window.Radisk) || require('./radisk'); var Radix = Radisk.Radix; - var dare = Radisk(opt), esc = String.fromCharCode(27); + var dare = Radisk(opt), esc = String.fromCharCode(26); var ST = 0; root.on('put', function(msg){ @@ -15,15 +15,19 @@ Gun.on('create', function(root){ if((msg._||'').rad){ return } // don't save what just came from a read. var id = msg['#'], put = msg.put, soul = put['#'], key = put['.'], val = put[':'], state = put['>'], tmp; var DBG = (msg._||'').DBG; DBG && (DBG.sp = DBG.sp || +new Date); - var lot = (msg._||'').lot||''; count[id] = (count[id] || 0) + 1; + //var lot = (msg._||'').lot||''; count[id] = (count[id] || 0) + 1; var S = (msg._||'').RPS || ((msg._||'').RPS = +new Date); - dare(soul+esc+key, {':': val, '>': state}, dare.one[id] || function(err, ok){ + //console.only.i && console.log("PUT ------->>>", soul,key, val, state); + //dare(soul+esc+key, {':': val, '>': state}, dare.one[id] || function(err, ok){ + dare(soul+esc+key, {':': val, '>': state}, function(err, ok){ + //console.only.i && console.log("<<<------- PAT", soul,key, val, state, 'in', +new Date - S); DBG && (DBG.spd = DBG.spd || +new Date); console.STAT && console.STAT(S, +new Date - S, 'put'); - if(!err && count[id] !== lot.s){ console.log(err = "Disk count not same as ram count."); console.STAT && console.STAT(+new Date, lot.s - count[id], 'put ack != count') } delete count[id]; + //if(!err && count[id] !== lot.s){ console.log(err = "Disk count not same as ram count."); console.STAT && console.STAT(+new Date, lot.s - count[id], 'put ack != count') } delete count[id]; if(err){ root.on('in', {'@': id, err: err, DBG: DBG}); return } root.on('in', {'@': id, ok: ok, DBG: DBG}); - }, id, DBG && (DBG.r = DBG.r || {})); + //}, id, DBG && (DBG.r = DBG.r || {})); + }, false && id, DBG && (DBG.r = DBG.r || {})); DBG && (DBG.sps = DBG.sps || +new Date); }); var count = {}, obj_empty = Object.empty; @@ -56,7 +60,7 @@ Gun.on('create', function(root){ if((tmp = get['%']) || o.limit){ o.limit = (tmp <= (o.pack || (1000 * 100)))? tmp : 1; } - if(has['-'] || (soul||{})['-']){ o.reverse = true } + if(has['-'] || (soul||{})['-'] || get['-']){ o.reverse = true } if((tmp = (root.next||'')[soul]) && tmp.put){ if(o.atom){ tmp = (tmp.next||'')[o.atom] ; @@ -67,7 +71,9 @@ Gun.on('create', function(root){ var now = Gun.state(); var S = (+new Date), C = 0, SPT = 0; // STATS! DBG && (DBG.sgm = S); + //var GID = String.random(3); console.only.i && console.log("GET ------->>>", GID, key, o, '?', get); dare(key||'', function(err, data, info){ + //console.only.i && console.log("<<<------- GOT", GID, err, data); DBG && (DBG.sgr = +new Date); DBG && (DBG.sgi = info); try{opt.store.stats.get.time[statg % 50] = (+new Date) - S; ++statg; @@ -137,6 +143,6 @@ Gun.on('create', function(root){ } }); var val_is = Gun.valid; - opt.store.stats = {get:{time:{}, count:0}, put: {time:{}, count:0}}; // STATS! + (opt.store||{}).stats = {get:{time:{}, count:0}, put: {time:{}, count:0}}; // STATS! var statg = 0, statp = 0; // STATS! }); \ No newline at end of file diff --git a/sea.js b/sea.js index 58ab989c..2880b89f 100644 --- a/sea.js +++ b/sea.js @@ -766,7 +766,7 @@ })(USE, './user'); ;USE(function(module){ - var u, Gun = (''+u != typeof window)? (window.Gun||{chain:{}}) : require('../gun'); + var u, Gun = (''+u != typeof window)? (window.Gun||{chain:{}}) : USE((''+u === typeof MODULE?'.':'')+'./gun', 1); Gun.chain.then = function(cb, opt){ var gun = this, p = (new Promise(function(res, rej){ gun.once(res, opt); @@ -1174,7 +1174,7 @@ ;USE(function(module){ var SEA = USE('./sea'), noop = function(){}, u; - var Gun = (''+u != typeof window)? (window.Gun||{on:noop}) : require('../gun'); + var Gun = (''+u != typeof window)? (window.Gun||{on:noop}) : USE((''+u === typeof MODULE?'.':'')+'./gun', 1); // After we have a GUN extension to make user registration/login easy, we then need to handle everything else. // We do this with a GUN adapter, we first listen to when a gun instance is created (and when its options change) diff --git a/test/rad/rad.js b/test/rad/rad.js index 4516fce4..c5febc2d 100644 --- a/test/rad/rad.js +++ b/test/rad/rad.js @@ -6,7 +6,7 @@ var Gun; if(typeof window !== 'undefined'){ env = window } root = env.window? env.window : global; try{ env.window && root.localStorage && root.localStorage.clear() }catch(e){} - try{ indexedDB.deleteDatabase('radatatest') }catch(e){} + //try{ indexedDB.deleteDatabase('radatatest') }catch(e){} if(root.Gun){ root.Gun = root.Gun; root.Gun.TESTING = true; @@ -15,7 +15,6 @@ var Gun; try{ require('../../lib/fsrm')('radatatest') }catch(e){} root.Gun = require('../../gun'); root.Gun.TESTING = true; - //require('../lib/file'); require('../../lib/store'); require('../../lib/rfs'); } @@ -170,6 +169,14 @@ var names = ["Adalard","Adora","Aia","Albertina","Alfie","Allyn","Amabil","Ammam return; });*/ + it('deleting old RAD tests (may take long time)', function(done){ + done(); // Mocha doesn't print test until after its done, so show this first. + }); + it('deleted', function(done){ + this.timeout(60 * 1000); + if(!Gun.window){ return done() } + indexedDB.deleteDatabase('radatatest').onsuccess = function(e){ done() } + }); it('write contacts', function(done){ var all = {}, to, start; @@ -315,6 +322,14 @@ var names = ["Adalard","Adora","Aia","Albertina","Alfie","Allyn","Amabil","Ammam var ochunk = 1000; var gun = Gun({chunk: ochunk}); + /*it('deleting old tests (may take long time)', function(done){ + done(); // Mocha doesn't print test until after its done, so show this first. + }); it('deleted', function(done){ + this.timeout(60 * 1000); + if(!Gun.window){ return done() } + indexedDB.deleteDatabase('radatatest').onsuccess = function(e){ done() } + });*/ + /*it('write same', function(done){ var all = {}, to, start, tmp; var names = [], c = 285; @@ -363,7 +378,9 @@ var names = ["Adalard","Adora","Aia","Albertina","Alfie","Allyn","Amabil","Ammam gun.get('users').get('alice').put({cool: 'beans'}); gun.get('users').get('alexander').put({nice: 'beans'}); gun.get('users').get('bob').put({lol: 'beans'}); + //console.log("=================");console.only.i=1; gun.get('users').get({'.': {'*': 'a'}, '%': 1000 * 100}).map().on(function(d,k){ + //console.log("small range:", k, d); expect('a' === k[0]).to.be.ok(); check[k] = d; if(check.alice && check.alexander){ @@ -373,6 +390,29 @@ var names = ["Adalard","Adora","Aia","Albertina","Alfie","Allyn","Amabil","Ammam }); }); + /*it.only('small range once TEST', function(done){ + var gun = Gun({file: 'yuio'}); + var check = {}; + gun.get('people').get('alice').put({cool: 'beans'}); + gun.get('people').get('alexander').put({nice: 'beans'}); + gun.get('people').get('bob').put({lol: 'beans'}); + //setTimeout(function(){ + console.only.i=1; + console.log("=================="); + console.log("=================="); + console.log("=================="); + gun.get('people').get({'.': {'*': 'a'}, '%': 1000 * 100}).once().map().once(function(d,k){ + console.log("***********", k,d); + expect('a' === k[0]).to.be.ok(); + check[k] = d; + if(check.alice && check.alexander){ + if(done.c){ return } done.c = 1; + done(); + } + }); + //},500); + });*/ + it('small range once', function(done){ var check = {}; gun.get('people').get('alice').put({cool: 'beans'}); @@ -443,6 +483,7 @@ var names = ["Adalard","Adora","Aia","Albertina","Alfie","Allyn","Amabil","Ammam if(v.indexOf(find) == 0){ all[v] = true } }); gun.get('names').get({'.': {'*': find}, '%': 1000 * 100}).once().map().once(function(data, key){ + //console.log("*******", key, data, this._.back.get); expect(data.name).to.be.ok(); expect(data.age).to.be.ok(); delete all[key]; diff --git a/test/sea/sea.js b/test/sea/sea.js index 6ea63795..0d5308f8 100755 --- a/test/sea/sea.js +++ b/test/sea/sea.js @@ -6,7 +6,7 @@ var Gun; if(typeof window !== 'undefined'){ env = window } root = env.window? env.window : global; try{ env.window && root.localStorage && root.localStorage.clear() }catch(e){} - try{ indexedDB.deleteDatabase('radatatest') }catch(e){} + //try{ indexedDB.deleteDatabase('radatatest') }catch(e){} if(root.Gun){ root.Gun = root.Gun; root.Gun.TESTING = true; @@ -15,7 +15,6 @@ var Gun; try{ require('../../lib/fsrm')('radatatest') }catch(e){} root.Gun = require('../../gun'); root.Gun.TESTING = true; - //require('../lib/file'); require('../../lib/store'); require('../../lib/rfs'); } @@ -41,6 +40,14 @@ describe('SEA', function(){ var prep = async function(d,k, n,s){ return {'#':s,'.':k,':': await SEA.opt.parse(d),'>':Gun.state.is(n, k)} }; // shim for old - prep for signing. var pack = function(d,cb,k, n,s){ return new Promise(function(res, rej){ SEA.opt.pack(d, function(r){ res(r) }, k,n,s) }) }; // make easier to upgrade test, cb to await describe('Utility', function(){ + it('deleting old SEA tests (may take long time)', function(done){ + done(); // Mocha doesn't print test until after its done, so show this first. + }); + it('deleted', function(done){ + this.timeout(60 * 1000); + if(!Gun.window){ return done() } + indexedDB.deleteDatabase('radatatest').onsuccess = function(e){ done() } + }); /*it('generates aeskey from jwk', function(done) { // DEPRECATED!!! console.log("WARNING: THIS DOES NOT WORK IN BROWSER!!!! NEEDS FIX"); SEA.opt.aeskey('x','x').then(k => { @@ -236,7 +243,7 @@ describe('SEA', function(){ }); }())}) - it('JSON escape', async function(done){ + it('JSON escape', function(done){ (async function(){ var plain = "hello world"; var json = JSON.stringify({hello:'world'}); @@ -249,7 +256,7 @@ describe('SEA', function(){ tmp = SEA.opt.unpack(tmp); expect(tmp.hello).to.be("world"); done(); - }); + }())}); it('double sign', function(done){ (async function(){ var pair = await SEA.pair();