diff --git a/gun.js b/gun.js index 2014ec62..3dad5647 100644 --- a/gun.js +++ b/gun.js @@ -1360,7 +1360,22 @@ // #soul.has=value>state // ~who#where.where=what>when@was // TODO: BUG! Put probably cannot handle plural chains! - var gun = this, at = (gun._), root = at.root.$, tmp; + var gun = this, at = (gun._), root = at.root.$, ctx = root._, tmp; + if(tmp = ctx.puts){ + if(tmp > 100){ // without this, when synchronous, writes to a 'not found' pile up, when 'not found' resolves it recursively calls `put` which incrementally resolves each write. Stack overflow limits can be as low as 10K, so this limit is hardcoded to 1% of 10K. + (ctx.stack || (ctx.stack = [])).push([gun, data, cb, as]); + clearTimeout(ctx.puto); + ctx.puto = setTimeout(function(){ + var stack = ctx.stack, i = 0, tmp; + ctx.stack = ctx.puts = ctx.puto = null; + while(tmp = stack[i++]){ + tmp[0].put(tmp[1], tmp[2], tmp[3]); + } + }, ctx.opt.wait || 1); + return gun; + } + ++ctx.puts; + } else { ctx.puts = 1 } as = as || {}; as.data = data; as.via = as.$ = as.via || as.$ || gun; diff --git a/lib/radisk.js b/lib/radisk.js index 64fca6cf..249da91d 100644 --- a/lib/radisk.js +++ b/lib/radisk.js @@ -14,6 +14,7 @@ opt.chunk = opt.chunk || (1024 * 1024 * 10); // 10MB opt.code = opt.code || {}; opt.code.from = opt.code.from || '!'; + //opt.jsonify = true; // TODO: REMOVE!!!! function ename(t){ return encodeURIComponent(t).replace(/\*/g, '%2A') } function atomic(v){ return u !== v && (!v || 'object' != typeof v) } @@ -80,7 +81,6 @@ r.batch = Radix(); r.batch.acks = []; r.batch.ed = 0; - console.debug.i = 1; r.save(batch, function(err, ok){ if(++i > 1){ opt.log('RAD ERR: Radisk has callbacked multiple times, please report this as a BUG at github.com/amark/gun/issues ! ' + i); return } if(err){ opt.log('err', err) } @@ -144,7 +144,8 @@ Therefore it is unavoidable that a read will have to happen, the question is just how long you delay it. */ - r.write = function(file, rad, cb, force){ + r.write = function(file, rad, cb, o){ + o = ('object' == typeof o)? o : {force: o}; var f = function Fractal(){}; f.text = ''; f.count = 0; @@ -154,7 +155,7 @@ if(u !== val){ f.count++ } if(opt.pack <= (val||'').length){ return cb("Record too big!"), true } var enc = Radisk.encode(pre.length) +'#'+ Radisk.encode(k) + (u === val? '' : ':'+ Radisk.encode(val)) +'\n'; - if((opt.chunk < f.text.length + enc.length) && (1 < f.count) && !force){ + if((opt.chunk < f.text.length + enc.length) && (1 < f.count) && !o.force){ f.text = ''; f.limit = Math.ceil(f.count/2); f.count = 0; @@ -166,7 +167,9 @@ } f.write = function(){ var tmp = ename(file); + //var start = (+new Date); // comment this out! opt.store.put(tmp, f.text, function(err){ + //console.log("wrote JSON in", (+new Date) - start); // comment this out! if(err){ return cb(err) } r.list.add(tmp, cb); }); @@ -177,7 +180,7 @@ var name = f.file; f.file = key; f.count = 0; - r.write(name, f.sub, f.next, force); + r.write(name, f.sub, f.next, o); return true; } f.sub(key, val); @@ -186,12 +189,26 @@ if(err){ return cb(err) } f.sub = Radix(); if(!Radix.map(rad, f.slice)){ - r.write(f.file, f.sub, cb, force); + r.write(f.file, f.sub, cb, o); } } + if(opt.jsonify){ return r.write.jsonify(f, file, rad, cb, o) } // temporary testing idea if(!Radix.map(rad, f.each, true)){ f.write() } } + r.write.jsonify = function(f, file, rad, cb, o){ + var raw; + //var start = (+new Date); // comment this out! + try{raw = JSON.stringify(rad.$); + }catch(e){ return cb("Record too big!") } + //console.log("stringified JSON in", (+new Date) - start); // comment this out! + if(opt.chunk < raw.length && !o.force){ + if(Radix.map(rad, f.each, true)){ return } + } + f.text = raw; + f.write(); + } + ;(function(){ var Q = {}; r.read = function(key, cb, o){ @@ -210,10 +227,10 @@ if(!file || file > (o.next || key)){ if(o.next){ g.file = file } if(tmp = Q[g.file]){ - tmp.push({key: key, ack: cb, file: g.file}); + tmp.push({key: key, ack: cb, file: g.file, opt: o}); return true; } - Q[g.file] = [{key: key, ack: cb, file: g.file}]; + Q[g.file] = [{key: key, ack: cb, file: g.file, opt: o}]; r.parse(g.file, g.it); return true; } @@ -221,20 +238,21 @@ } g.it = function(err, disk, info){ if(g.err = err){ opt.log('err', err) } + g.info = info; if(disk){ RAD = g.disk = disk } - o.parsed = (o.parsed || 0) + (info.parsed||0); - o.chunks = (o.chunks || 0) + 1; disk = Q[g.file]; delete Q[g.file]; map(disk, g.ack); } g.ack = function(as){ if(!as.ack){ return } - var tmp = as.key, rad = g.disk || noop, data = rad(tmp), last = rad.last; + var tmp = as.key, o = as.opt, info = g.info, rad = g.disk || noop, data = rad(tmp), last = rad.last; + o.parsed = (o.parsed || 0) + (info.parsed||0); + o.chunks = (o.chunks || 0) + 1; if(!o.some){ o.some = (u !== data) } if(u !== data){ as.ack(g.err, data, o) } else if(!as.file){ !o.some && as.ack(g.err, u, o); return } - if(!last || last === tmp){ !o.some && as.ack(g.err, u, o); return } - if(last > tmp && 0 != last.indexOf(tmp)){ !o.some && as.ack(g.err, u, o); return } + if(/*!last || */last === tmp){ !o.some && as.ack(g.err, u, o); return } + if(last && last > tmp && 0 != last.indexOf(tmp)){ !o.some && as.ack(g.err, u, o); return } if(o.some && o.parsed >= o.limit){ return } o.next = as.file; r.read(tmp, as.ack, o); @@ -271,6 +289,24 @@ }catch(e){ p.err = e } if(p.err){ return map(q, p.ack) } } + info.parsed = data.length; + + //var start = (+new Date); // keep this commented out in production! + if(opt.jsonify){ // temporary testing idea + try{ + var json = JSON.parse(data); + p.disk.$ = json; + //console.log('parsed JSON in', (+new Date) - start); // keep this commented out in production! + map(q, p.ack); + return; + }catch(e){ tmp = e } + if('{' === data[0]){ + p.err = tmp || "JSON error!"; + return map(q, p.ack); + } + } + + //var start = (+new Date); // keep this commented out in production! var tmp = p.split(data), pre = [], i, k, v; if(!tmp || 0 !== tmp[1]){ p.err = "File '"+file+"' does not have root radix! "; @@ -293,8 +329,8 @@ if(u !== k && u !== v){ p.disk(pre.join(''), v) } tmp = p.split(tmp[2]); } + //console.log('parsed JSON in', (+new Date) - start); // keep this commented out in production! //cb(err, p.disk); - info.parsed = data.length; map(q, p.ack); }; p.split = function(t){ diff --git a/lib/store.js b/lib/store.js index a8047202..b0327ab7 100644 --- a/lib/store.js +++ b/lib/store.js @@ -18,8 +18,6 @@ Gun.on('create', function(root){ if(track){ ++acks } //console.log('put:', soul, key, val); val = Radisk.encode(val, null, esc)+'>'+Radisk.encode(Gun.state.is(node, key), null, esc); - //rad(soul+'.'+key, val, (track? ack : u)); - //console.log("PUT!", id, JSON.stringify(soul+esc+key)); rad(soul+esc+key, val, (track? ack : u)); }); function ack(err, ok){ diff --git a/package.json b/package.json index 09ece699..eb858b61 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "gun", - "version": "0.9.9999992", + "version": "0.20190222.0", "description": "A realtime, decentralized, offline-first, graph data synchronization engine.", "main": "index.js", "browser": "gun.js", diff --git a/test/common.js b/test/common.js index 2d95a8a9..2725fd10 100644 --- a/test/common.js +++ b/test/common.js @@ -3679,12 +3679,12 @@ describe('Gun', function(){ this.timeout(5000); var gun = Gun({test_no_peer:true}).get('g/m/no/slow'); //console.log("---------- setup data done -----------"); - var prev, diff, max = 25, total = 9, largest = -1, gone = {}; + var prev, diff, max = 25, total = 9, largest = -1, gone = {}, u; //var prev, diff, max = Infinity, total = 10000, largest = -1, gone = {}; // TODO: It would be nice if we could change these numbers for different platforms/versions of javascript interpreters so we can squeeze as much out of them. gun.get('history').map().on(function(time, index){ - //console.log(">>>", index, time); diff = Gun.time.is() - time; + //console.log(">>>", index, time, diff); //return; expect(gone[index]).to.not.be.ok(); gone[index] = diff; @@ -3695,6 +3695,7 @@ describe('Gun', function(){ var turns = 0; var many = setInterval(function(){ if(turns > total || (diff || 0) > (max + 5)){ + if(u === diff){ return } clearTimeout(many); expect(Gun.num.is(diff)).to.be.ok(); if(done.c){ return } done.c = 1; diff --git a/test/rad/rad.js b/test/rad/rad.js index 64711ecf..e2f597c7 100644 --- a/test/rad/rad.js +++ b/test/rad/rad.js @@ -125,7 +125,7 @@ var names = ["Adalard","Adora","Aia","Albertina","Alfie","Allyn","Amabil","Ammam v = v.toLowerCase(); if(v.indexOf(find) == 0){ all[v] = true } }); - rad(find, function(err, data){ + rad(find, function(err, data, info){ Radix.map(data, function(v,k){ delete all[find+k]; }); @@ -133,7 +133,7 @@ var names = ["Adalard","Adora","Aia","Albertina","Alfie","Allyn","Amabil","Ammam done(); }); }); - + it('read bytes', function(done){ var all = {}, find = 'm', to; names.forEach(function(v){ @@ -159,6 +159,22 @@ var names = ["Adalard","Adora","Aia","Albertina","Alfie","Allyn","Amabil","Ammam var ochunk = 1000; var gun = Gun({chunk: ochunk}); + it('write same', function(done){ + var all = {}, to, start, tmp; + var names = [], c = 285; + while(--c){ names.push('bob') } + names.forEach(function(v,i){ + all[++i] = true; + tmp = v.toLowerCase(); + gun.get('names').get(tmp).put({name: v, age: i}, function(ack){ + expect(ack.err).to.not.be.ok(); + delete all[i]; + if(!Gun.obj.empty(all)){ return } + done(); + }) + }); + }); + it('write contacts', function(done){ var all = {}, to, start, tmp; names.forEach(function(v,i){