mirror of
https://github.com/amark/gun.git
synced 2025-06-05 13:46:43 +00:00
fix SOME of RAD due to GUN queue growing too deep
This commit is contained in:
Mark Nadal
parent
a34e90e868
commit
6af5b6c2ac
29
gun.js
29
gun.js
@ -1235,7 +1235,7 @@
|
||||
gun = gun.$;
|
||||
} else
|
||||
if(key instanceof Function){
|
||||
if(true === cb){ return soul(this, key, cb, as) }
|
||||
if(true === cb){ return soul(this, key, cb, as), this }
|
||||
gun = this;
|
||||
var at = gun._, root = at.root, tmp = root.now, ev;
|
||||
as = cb || {};
|
||||
@ -1292,15 +1292,22 @@
|
||||
}
|
||||
function soul(gun, cb, opt, as){
|
||||
var cat = gun._, acks = 0, tmp;
|
||||
if(tmp = cat.soul || cat.link || cat.dub){ return cb(tmp, as, cat), gun }
|
||||
gun.get(function(msg, ev){
|
||||
if(tmp = cat.soul || cat.link || cat.dub){ return cb(tmp, as, cat) }
|
||||
if(cat.jam){ return cat.jam.push([cb, as]) }
|
||||
cat.jam = [[cb,as]];
|
||||
gun.get(function(msg, eve){
|
||||
if(u === msg.put && (tmp = Object.keys(cat.root.opt.peers).length) && ++acks < tmp){
|
||||
return;
|
||||
}
|
||||
ev.rid(msg);
|
||||
eve.rid(msg);
|
||||
var at = ((at = msg.$) && at._) || {};
|
||||
tmp = at.link || at.soul || rel.is(msg.put) || node_soul(msg.put) || at.dub;
|
||||
cb(tmp, as, msg, ev);
|
||||
tmp = cat.jam; Gun.obj.del(cat, 'jam');
|
||||
Gun.obj.map(tmp, function(as, cb){
|
||||
cb = as[0]; as = as[1];
|
||||
if(!cb){ return }
|
||||
var id = at.link || at.soul || rel.is(msg.put) || node_soul(msg.put) || at.dub;
|
||||
cb(id, as, msg, eve);
|
||||
});
|
||||
}, {out: {get: {'.':true}}});
|
||||
return gun;
|
||||
}
|
||||
@ -1361,9 +1368,9 @@
|
||||
Gun.chain.put = function(data, cb, as){
|
||||
// #soul.has=value>state
|
||||
// ~who#where.where=what>when@was
|
||||
// TODO: BUG! Put probably cannot handle plural chains!
|
||||
// TODO: BUG! Put probably cannot handle plural chains! `!as` is quickfix test.
|
||||
var gun = this, at = (gun._), root = at.root.$, ctx = root._, M = 100, tmp;
|
||||
if(!ctx.puta){ if(tmp = ctx.puts){ if(tmp > M){ // without this, when synchronous, writes to a 'not found' pile up, when 'not found' resolves it recursively calls `put` which incrementally resolves each write. Stack overflow limits can be as low as 10K, so this limit is hardcoded to 1% of 10K.
|
||||
/*if(!ctx.puta && !as){ if(tmp = ctx.puts){ if(tmp > M){ // without this, when synchronous, writes to a 'not found' pile up, when 'not found' resolves it recursively calls `put` which incrementally resolves each write. Stack overflow limits can be as low as 10K, so this limit is hardcoded to 1% of 10K.
|
||||
(ctx.stack || (ctx.stack = [])).push([gun, data, cb, as]);
|
||||
if(ctx.puto){ return }
|
||||
ctx.puto = setTimeout(function drain(){
|
||||
@ -1373,7 +1380,7 @@
|
||||
ctx.stack = ctx.puts = ctx.puto = null;
|
||||
}, 0);
|
||||
return gun;
|
||||
} ++ctx.puts } else { ctx.puts = 1 } }
|
||||
} ++ctx.puts } else { ctx.puts = 1 } }*/
|
||||
as = as || {};
|
||||
as.data = data;
|
||||
as.via = as.$ = as.via || as.$ || gun;
|
||||
@ -1496,6 +1503,7 @@
|
||||
ref = ref.get(path[i]);
|
||||
}
|
||||
if(is){ ref = v }
|
||||
//if(as.not){ (ref._).dub = Gun.text.random() } // This might optimize stuff? Maybe not needed anymore. Make sure it doesn't introduce bugs.
|
||||
var id = (ref._).dub;
|
||||
if(id || (id = Gun.node.soul(at.obj))){
|
||||
ref.back(-1).get(id);
|
||||
@ -1556,6 +1564,7 @@
|
||||
if(at.link || at.soul){ return at.link || at.soul }
|
||||
as.data = obj_put({}, at.get, as.data);
|
||||
});
|
||||
as.not = true; // maybe consider this?
|
||||
}
|
||||
tmp = tmp || at.soul || at.link || at.dub;// || at.get;
|
||||
at = tmp? (at.root.$.get(tmp)._) : at;
|
||||
@ -1815,7 +1824,7 @@
|
||||
// See the next 'opt' code below for actual saving of data.
|
||||
var ev = this.to, opt = root.opt;
|
||||
if(root.once){ return ev.next(root) }
|
||||
//if(false === opt.localStorage){ return ev.next(root) } // we want offline resynce queue regardless!
|
||||
if(false === opt.localStorage){ return ev.next(root) } // we want offline resynce queue regardless! // actually, this doesn't help, per @go1dfish 's observation. Disabling for now, will need better solution later.
|
||||
opt.prefix = opt.file || 'gun/';
|
||||
var gap = Gun.obj.ify(store.getItem('gap/'+opt.prefix)) || {};
|
||||
var empty = Gun.obj.empty, id, to, go;
|
||||
|
||||
2
gun.min.js
vendored
2
gun.min.js
vendored
File diff suppressed because one or more lines are too long
@ -14,12 +14,12 @@
|
||||
opt.chunk = opt.chunk || (1024 * 1024 * 10); // 10MB
|
||||
opt.code = opt.code || {};
|
||||
opt.code.from = opt.code.from || '!';
|
||||
//opt.jsonify = true; // TODO: REMOVE!!!!
|
||||
//opt.jsonify = true; if(opt.jsonify){ console.log("JSON RAD!!!") } // TODO: REMOVE!!!!
|
||||
|
||||
function ename(t){ return encodeURIComponent(t).replace(/\*/g, '%2A') }
|
||||
function atomic(v){ return u !== v && (!v || 'object' != typeof v) }
|
||||
var map = Gun.obj.map;
|
||||
var LOG = false;
|
||||
var LOG = false;//true;
|
||||
|
||||
if(!opt.store){
|
||||
return opt.log("ERROR: Radisk needs `opt.store` interface with `{get: fn, put: fn (, list: fn)}`!");
|
||||
@ -83,12 +83,13 @@
|
||||
r.batch = Radix();
|
||||
r.batch.acks = [];
|
||||
r.batch.ed = 0;
|
||||
//var id = Gun.text.random(2), S = (+new Date); console.log("<<<<<<<<<<<<", id);
|
||||
//console.debug(99); var ID = Gun.text.random(2), S = (+new Date); console.log("[[[[[[[[", ID, batch.acks.length);
|
||||
r.save(batch, function(err, ok){
|
||||
if(++i > 1){ opt.log('RAD ERR: Radisk has callbacked multiple times, please report this as a BUG at github.com/amark/gun/issues ! ' + i); return }
|
||||
if(err){ opt.log('err', err) }
|
||||
//console.log(">>>>>>>>>>>>", id, ((+new Date) - S), batch.acks.length);
|
||||
//console.debug(99); var TMP; console.log("]]]]]]]]", ID, batch.acks.length, (TMP = +new Date) - S, 'more?', thrash.more);
|
||||
map(batch.acks, function(cb){ cb(err, ok) });
|
||||
//console.log("][", +new Date - TMP, thrash.more);
|
||||
thrash.at = null;
|
||||
thrash.ing = false;
|
||||
if(thrash.more){ thrash() }
|
||||
@ -171,9 +172,9 @@
|
||||
}
|
||||
f.write = function(){
|
||||
var tmp = ename(file);
|
||||
var start; LOG && (start = (+new Date)); // comment this out!
|
||||
var start; LOG && (start = +new Date); // comment this out!
|
||||
opt.store.put(tmp, f.text, function(err){
|
||||
LOG && console.log("wrote JSON in", (+new Date) - start); // comment this out!
|
||||
LOG && console.log("wrote to disk in", (+new Date) - start, tmp); // comment this out!
|
||||
if(err){ return cb(err) }
|
||||
r.list.add(tmp, cb);
|
||||
});
|
||||
@ -202,10 +203,10 @@
|
||||
|
||||
r.write.jsonify = function(f, file, rad, cb, o){
|
||||
var raw;
|
||||
var start; LOG && (start = (+new Date)); // comment this out!
|
||||
var start; LOG && (start = +new Date); // comment this out!
|
||||
try{raw = JSON.stringify(rad.$);
|
||||
}catch(e){ return cb("Record too big!") }
|
||||
LOG && console.log("stringified JSON in", (+new Date) - start); // comment this out!
|
||||
LOG && console.log("stringified JSON in", +new Date - start); // comment this out!
|
||||
if(opt.chunk < raw.length && !o.force){
|
||||
if(Radix.map(rad, f.each, true)){ return }
|
||||
}
|
||||
@ -220,7 +221,7 @@
|
||||
var sub = Radix();
|
||||
Radix.map(tree, function(v,k){
|
||||
sub(k,v);
|
||||
}, o)
|
||||
}, o);
|
||||
return sub('');
|
||||
}
|
||||
|
||||
@ -266,7 +267,7 @@
|
||||
}
|
||||
g.ack = function(as){
|
||||
if(!as.ack){ return }
|
||||
var tmp = as.key, o = as.opt, info = g.info, rad = g.disk || noop, data = r.range(rad(tmp), o), last = rad.last;
|
||||
var tmp = as.key, o = as.opt, info = g.info, rad = g.disk || noop, data = r.range(rad(tmp), o), last = rad.last || Radix.map(rad, rev, revo);
|
||||
o.parsed = (o.parsed || 0) + (info.parsed||0);
|
||||
o.chunks = (o.chunks || 0) + 1;
|
||||
if(!o.some){ o.some = (u !== data) }
|
||||
@ -283,6 +284,8 @@
|
||||
if(o.reverse){ g.lex.reverse = true }
|
||||
r.list(g.lex);
|
||||
}
|
||||
function rev(a,b){ return b }
|
||||
var revo = {reverse: true};
|
||||
}());
|
||||
|
||||
;(function(){
|
||||
@ -299,6 +302,7 @@
|
||||
var p = function Parse(){}, info = {};
|
||||
p.disk = Radix();
|
||||
p.read = function(err, data){ var tmp;
|
||||
LOG && console.log('read disk in', +new Date - start, ename(file)); // keep this commented out in
|
||||
delete Q[file];
|
||||
if((p.err = err) || (p.not = !data)){
|
||||
return map(q, p.ack);
|
||||
@ -315,12 +319,12 @@
|
||||
}
|
||||
info.parsed = data.length;
|
||||
|
||||
var start; LOG && (start = (+new Date)); // keep this commented out in production!
|
||||
LOG && (start = +new Date); // keep this commented out in production!
|
||||
if(opt.jsonify){ // temporary testing idea
|
||||
try{
|
||||
var json = JSON.parse(data);
|
||||
p.disk.$ = json;
|
||||
LOG && console.log('parsed JSON in', (+new Date) - start); // keep this commented out in production!
|
||||
LOG && console.log('parsed JSON in', +new Date - start); // keep this commented out in production!
|
||||
map(q, p.ack);
|
||||
return;
|
||||
}catch(e){ tmp = e }
|
||||
@ -329,7 +333,7 @@
|
||||
return map(q, p.ack);
|
||||
}
|
||||
}
|
||||
var start; LOG && (start = (+new Date)); // keep this commented out in production!
|
||||
LOG && (start = +new Date); // keep this commented out in production!
|
||||
var tmp = p.split(data), pre = [], i, k, v;
|
||||
if(!tmp || 0 !== tmp[1]){
|
||||
p.err = "File '"+file+"' does not have root radix! ";
|
||||
@ -352,7 +356,7 @@
|
||||
if(u !== k && u !== v){ p.disk(pre.join(''), v) }
|
||||
tmp = p.split(tmp[2]);
|
||||
}
|
||||
LOG && console.log('parsed JSON in', (+new Date) - start); // keep this commented out in production!
|
||||
LOG && console.log('parsed RAD in', +new Date - start); // keep this commented out in production!
|
||||
//cb(err, p.disk);
|
||||
map(q, p.ack);
|
||||
};
|
||||
@ -372,6 +376,7 @@
|
||||
if(p.err || p.not){ return cb(p.err, u, info) }
|
||||
cb(u, p.disk, info);
|
||||
}
|
||||
var start; LOG && (start = +new Date); // keep this commented out in production!
|
||||
if(raw){ return p.read(null, raw) }
|
||||
opt.store.get(ename(file), p.read);
|
||||
}
|
||||
@ -504,6 +509,7 @@
|
||||
} else {
|
||||
var Gun = require('../gun');
|
||||
var Radix = require('./radix');
|
||||
//var Radix = require('./radix2'); Radisk = require('./radisk2');
|
||||
try{ module.exports = Radisk }catch(e){}
|
||||
}
|
||||
|
||||
|
||||
14
lib/radix.js
14
lib/radix.js
@ -54,10 +54,10 @@
|
||||
Radix.map = function map(radix, cb, opt, pre){ pre = pre || [];
|
||||
var t = ('function' == typeof radix)? radix.$ || {} : radix;
|
||||
if(!t){ return }
|
||||
var keys = (t[_]||no).sort || (t[_] = function $(){ $.sort = Object.keys(t).sort(); return $ }()).sort;
|
||||
var keys = (t[_]||no).sort || (t[_] = function $(){ $.sort = Object.keys(t).sort(); return $ }()).sort, rev;
|
||||
//var keys = Object.keys(t).sort();
|
||||
opt = (true === opt)? {branch: true} : (opt || {});
|
||||
if(opt.reverse){ keys = keys.slice().reverse() }
|
||||
if(rev = opt.reverse){ keys = keys.slice().reverse() }
|
||||
var start = opt.start, end = opt.end;
|
||||
var i = 0, l = keys.length;
|
||||
for(;i < l; i++){ var key = keys[i], tree = t[key], tmp, p, pt;
|
||||
@ -66,6 +66,10 @@
|
||||
pt = p.join('');
|
||||
if(u !== start && pt < (start||'').slice(0,pt.length)){ continue }
|
||||
if(u !== end && (end || '\uffff') < pt){ continue }
|
||||
if(rev){ // children must be checked first when going in reverse.
|
||||
tmp = map(tree, cb, opt, p);
|
||||
if(u !== tmp){ return tmp }
|
||||
}
|
||||
if(u !== (tmp = tree[''])){
|
||||
tmp = cb(tmp, pt, key, pre);
|
||||
if(u !== tmp){ return tmp }
|
||||
@ -75,8 +79,10 @@
|
||||
if(u !== tmp){ return tmp }
|
||||
}
|
||||
pre = p;
|
||||
tmp = map(tree, cb, opt, pre);
|
||||
if(u !== tmp){ return tmp }
|
||||
if(!rev){
|
||||
tmp = map(tree, cb, opt, pre);
|
||||
if(u !== tmp){ return tmp }
|
||||
}
|
||||
pre.pop();
|
||||
}
|
||||
};
|
||||
|
||||
@ -67,14 +67,12 @@ Gun.on('create', function(root){
|
||||
o.limit = (tmp <= (o.pack || (1000 * 100)))? tmp : 1;
|
||||
}
|
||||
if(has['-'] || (soul||{})['-']){ o.reverse = true }
|
||||
//console.log("RAD get:", key, o);
|
||||
var start = (+new Date); // STATS! // console.log("GET!", id, JSON.stringify(key));
|
||||
rad(key||'', function(err, data, o){
|
||||
try{opt.store.stats.get.time[statg % 50] = (+new Date) - start; ++statg;
|
||||
opt.store.stats.get.count++;
|
||||
if(err){ opt.store.stats.get.err = err }
|
||||
}catch(e){} // STATS!
|
||||
//console.log("RAD gat:", err, data, o);
|
||||
if(data){
|
||||
if(typeof data !== 'string'){
|
||||
if(o.atom){
|
||||
@ -85,7 +83,6 @@ Gun.on('create', function(root){
|
||||
}
|
||||
if(!graph && data){ each(data, '') }
|
||||
}
|
||||
//console.log("GOT!", id, JSON.stringify(key), ((+new Date) - start));
|
||||
root.on('in', {'@': id, put: graph, err: err? err : u, rad: Radix});
|
||||
}, o);
|
||||
function each(val, has, a,b){
|
||||
|
||||
@ -127,7 +127,6 @@ function batch(){ var as = this;
|
||||
}, as);
|
||||
if(as.res){ as.res() }
|
||||
} function no(v,k){ if(v){ return true } }
|
||||
//console.debug(999,1); var C = 0; setInterval(function(){ try{ debug.innerHTML = C }catch(e){console.log(e)} }, 500);
|
||||
|
||||
function map(v,k,n, at){ var as = this;
|
||||
var is = Gun.is(v);
|
||||
@ -157,7 +156,7 @@ function soul(id, as, msg, eve){
|
||||
id = at.dub = at.dub || id || Gun.node.soul(cat.obj) || Gun.node.soul(msg.put || at.put) || Gun.val.link.is(msg.put || at.put) || (as.via.back('opt.uuid') || Gun.text.random)(); // TODO: BUG!? Do we really want the soul of the object given to us? Could that be dangerous?
|
||||
if(eve){ eve.stun = true }
|
||||
if(!id){ // polyfill async uuid for SEA
|
||||
at.via.back('opt.uuid')(function(err, id){ // TODO: improve perf without anonymous callback
|
||||
as.via.back('opt.uuid')(function(err, id){ // TODO: improve perf without anonymous callback
|
||||
if(err){ return Gun.log(err) } // TODO: Handle error.
|
||||
solve(at, at.dub = at.dub || id, cat, as);
|
||||
});
|
||||
|
||||
@ -153,6 +153,7 @@ Gun.dup = require('./dup');
|
||||
Gun.on.get = function(msg, gun){
|
||||
var root = gun._, get = msg.get, soul = get[_soul], node = root.graph[soul], has = get[_has], tmp;
|
||||
var next = root.next || (root.next = {}), at = next[soul];
|
||||
// queue concurrent GETs?
|
||||
if(!node){ return root.on('get', msg) }
|
||||
if(has){
|
||||
if('string' != typeof has || !obj_has(node, has)){ return root.on('get', msg) }
|
||||
@ -193,6 +194,7 @@ Gun.dup = require('./dup');
|
||||
at.opt.peers = at.opt.peers || {};
|
||||
obj_map(opt, function each(v,k){
|
||||
if(!obj_has(this, k) || text.is(v) || obj.empty(v)){ this[k] = v ; return }
|
||||
if(v && v.constructor !== Object && !list_is(v)){ return }
|
||||
obj_map(v, each, this[k]);
|
||||
}, at.opt);
|
||||
Gun.on('opt', at);
|
||||
|
||||
@ -3692,7 +3692,6 @@ describe('Gun', function(){
|
||||
expect(gone[index]).to.not.be.ok();
|
||||
gone[index] = diff;
|
||||
largest = (largest < diff)? diff : largest;
|
||||
//console.log(diff, '<', max);
|
||||
expect(diff > max).to.not.be.ok();
|
||||
});
|
||||
var turns = 0;
|
||||
|
||||
@ -113,15 +113,22 @@ var names = ["Adalard","Adora","Aia","Albertina","Alfie","Allyn","Amabil","Ammam
|
||||
|
||||
it('radix reverse', function(done){
|
||||
var r = Radix(), tmp;
|
||||
r('alice', 1);r('bob', 2);r('carl', 3);r('dave', 4);
|
||||
r('alice', 1);r('bob', 2);r('carl', 3);r('carlo',4);
|
||||
r('dave', 5);r('zach',6);r('zachary',7);
|
||||
var by = ['alice','bob','carl','carlo','dave','zach','zachary'];
|
||||
Gun.obj.map(by, function(k,i){
|
||||
r(k,i);
|
||||
});
|
||||
Radix.map(r, function(v,k, a,b){
|
||||
expect(by.pop()).to.be(k);
|
||||
tmp = v;
|
||||
}, {reverse: 1});
|
||||
expect(tmp).to.be(1);
|
||||
expect(by.length).to.be(0);
|
||||
Radix.map(r, function(v,k, a,b){
|
||||
tmp = v;
|
||||
});
|
||||
expect(tmp).to.be(4);
|
||||
expect(tmp).to.be(7);
|
||||
done();
|
||||
});
|
||||
});
|
||||
|
||||
Loading…
x
Reference in New Issue
Block a user