Mirroristas/gun
2
0
Fork 0
mirror of https://github.com/amark/gun.git synced 2025-06-06 06:06:50 +00:00
Code Issues Packages Projects Releases Wiki Activity

Merge pull request #826 from amark/dev

Browse Source 
Dev
This commit is contained in:
Mark Nadal 2019-09-30 00:53:38 -07:00 committed by GitHub
commit b3399a4c01
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
5 changed files with 111 additions and 47 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

55
gun.js
View File

@ -811,16 +811,20 @@
// Maybe... in case the in-memory key we have is a local write
// we still need to trigger a pull/merge from peers.
} else {
var S = +new Date;
node = Gun.obj.copy(node);
console.log(+new Date - S, 'copy node');
}
node = Gun.graph.node(node);
tmp = (at||empty).ack;
var S = +new Date;
root.on('in', {
'@': msg['#'],
how: 'mem',
put: node,
$: gun
});
console.log(+new Date - S, 'root got send');
//if(0 < tmp){ return }
root.on('get', msg);
}
@ -1295,19 +1299,18 @@
if(tmp = cat.soul || cat.link || cat.dub){ return cb(tmp, as, cat) }
if(cat.jam){ return cat.jam.push([cb, as]) }
cat.jam = [[cb,as]];
gun.get(function(msg, eve){
gun.get(function go(msg, eve){
if(u === msg.put && (tmp = Object.keys(cat.root.opt.peers).length) && ++acks < tmp){
return;
}
eve.rid(msg);
var at = ((at = msg.$) && at._) || {};
tmp = cat.jam; Gun.obj.del(cat, 'jam');
Gun.obj.map(tmp, function(as, cb){
cb = as[0]; as = as[1];
if(!cb){ return }
var id = at.link || at.soul || rel.is(msg.put) || node_soul(msg.put) || at.dub;
cb(id, as, msg, eve);
});
var at = ((at = msg.$) && at._) || {}, i = 0, as;
tmp = cat.jam; delete cat.jam; // tmp = cat.jam.splice(0, 100);
//if(tmp.length){ process.nextTick(function(){ go(msg, eve) }) }
while(as = tmp[i++]){ //Gun.obj.map(tmp, function(as, cb){
var cb = as[0], id; as = as[1];
cb && cb(id = at.link || at.soul || rel.is(msg.put) || node_soul(msg.put) || at.dub, as, msg, eve);
} //);
}, {out: {get: {'.':true}}});
return gun;
}
@ -1701,7 +1704,7 @@
return;
}
if(link && u === link.put && (tmp = rel.is(data))){ data = Gun.node.ify({}, tmp) }
eve.rid(msg);
eve.rid? eve.rid(msg) : eve.off();
opt.ok.call(gun || opt.$, data, msg.get);
}
@ -1954,6 +1957,7 @@
;USE(function(module){
var Type = USE('../type');
var puff = (typeof setImmediate !== "undefined")? setImmediate : setTimeout;
function Mesh(root){
var mesh = function(){};
@ -1972,12 +1976,18 @@
if('[' === tmp){
try{msg = JSON.parse(raw);}catch(e){opt.log('DAM JSON parse error', e)}
if(!msg){ return }
var i = 0, m;
var S = +new Date; // STATS!
while(m = msg[i++]){
mesh.hear(m, peer);
}
(mesh.hear.long || (mesh.hear.long = [])).push(+new Date - S);
console.log('hear batch length of', msg.length);
(function go(){
var S = +new Date; // STATS!
var m, c = 100; // hardcoded for now?
while(c-- && (m = msg.shift())){
mesh.hear(m, peer);
}
console.log(+new Date - S, 'hear batch');
(mesh.hear.long || (mesh.hear.long = [])).push(+new Date - S);
if(!msg.length){ return }
puff(go, 0);
}());
return;
}
if('{' === tmp || (Type.obj.is(raw) && (msg = raw))){
@ -1985,6 +1995,7 @@
}catch(e){return opt.log('DAM JSON parse error', e)}
if(!msg){ return }
if(!(id = msg['#'])){ id = msg['#'] = Type.text.random(9) }
if(msg.DBG_s){ console.log(+new Date - msg.DBG_s, 'to hear', id) }
if(dup.check(id)){ return }
dup.track(id, true).it = msg; // GUN core also dedups, so `true` is needed. // Does GUN core need to dedup anymore?
if(!(hash = msg['##']) && u !== msg.put){ hash = msg['##'] = Type.obj.hash(msg.put) }
@ -2000,7 +2011,9 @@
}
return;
}
var S = +new Date;
root.on('in', msg);
!msg.nts && console.log(+new Date - S, 'msg', msg['#']);
return;
}
}
@ -2014,6 +2027,7 @@
if(this.to){ this.to.next(msg) } // compatible with middleware adapters.
if(!msg){ return false }
var id, hash, tmp, raw;
var S = +new Date; //msg.DBG_s = msg.DBG_s || +new Date;
var meta = msg._||(msg._=function(){});
if(!(id = msg['#'])){ id = msg['#'] = Type.text.random(9) }
if(!(hash = msg['##']) && u !== msg.put){ hash = msg['##'] = Type.obj.hash(msg.put) }
@ -2027,12 +2041,15 @@
}
}
}
console.log(+new Date - S, 'mesh say prep');
dup.track(id).it = msg; // track for 9 seconds, default. Earth<->Mars would need more!
if(!peer){ peer = (tmp = dup.s[msg['@']]) && (tmp = tmp.it) && (tmp = tmp._) && (tmp = tmp.via) }
if(!peer && mesh.way){ return mesh.way(msg) }
if(!peer || !peer.id){ message = msg;
if(!Type.obj.is(peer || opt.peers)){ return false }
var S = +new Date;
Type.obj.map(peer || opt.peers, each); // in case peer is a peer list.
console.log(+new Date - S, 'mesh say loop');
return;
}
if(!peer.wire && mesh.wire){ mesh.wire(peer) }
@ -2056,8 +2073,10 @@
peer.batch = peer.tail = null;
if(!tmp){ return }
if(!tmp.length){ return } // if(3 > tmp.length){ return } // TODO: ^
var S = +new Date;
try{tmp = (1 === tmp.length? tmp[0] : JSON.stringify(tmp));
}catch(e){return opt.log('DAM JSON stringify error', e)}
console.log(+new Date - S, 'mesh flush', tmp.length);
if(!tmp){ return }
send(tmp, peer);
}
@ -2067,12 +2086,14 @@
// for now - find better place later.
function send(raw, peer){ try{
var wire = peer.wire;
var S = +new Date;
if(peer.say){
peer.say(raw);
} else
if(wire.send){
wire.send(raw);
}
console.log(+new Date - S, 'wire send', raw.length);
mesh.say.d += raw.length||0; ++mesh.say.c; // STATS!
}catch(e){
(peer.queue = peer.queue || []).push(raw);
@ -2251,6 +2272,8 @@
return wire;
}catch(e){}}
setTimeout(function(){ root.on('out', {dam:'hi'}) },1); // it can take a while to open a socket, so maybe no longer lazy load for perf reasons?
var wait = 2 * 1000;
function reconnect(peer){
clearTimeout(peer.defer);

2
lib/evict.js
View File

@ -20,10 +20,12 @@
var souls = Object.keys(root.graph||empty);
var toss = Math.ceil(souls.length * 0.01);
//var start = Gun.state(), i = toss;
var S = +new Date;
Gun.list.map(souls, function(soul){
if(--toss < 0){ return }
root.gun.get(soul).off();
});
console.log(+new Date - S, 'gc');
//console.log("evicted", i, 'nodes in', ((Gun.state() - start)/1000).toFixed(2), 'sec.');
}
/*

58
lib/radisk.js
View File

@ -11,15 +11,15 @@
opt.pack = opt.pack || (opt.memory? (opt.memory * 1000 * 1000) : 1399000000) * 0.3; // max_old_space_size defaults to 1400 MB.
opt.until = opt.until || opt.wait || 250;
opt.batch = opt.batch || (10 * 1000);
opt.chunk = opt.chunk || (1024 * 1024 * 10); // 10MB
opt.chunk = opt.chunk || (1024 * 1024 * 1); // 1MB
opt.code = opt.code || {};
opt.code.from = opt.code.from || '!';
//opt.jsonify = true; if(opt.jsonify){ console.log("JSON RAD!!!") } // TODO: REMOVE!!!!
opt.jsonify = true;
function ename(t){ return encodeURIComponent(t).replace(/\*/g, '%2A') }
function atomic(v){ return u !== v && (!v || 'object' != typeof v) }
var map = Gun.obj.map;
var LOG = false;//true;
var LOG = true;
if(!opt.store){
return opt.log("ERROR: Radisk needs `opt.store` interface with `{get: fn, put: fn (, list: fn)}`!");
@ -44,9 +44,14 @@
if(val instanceof Function){
var o = cb || {};
cb = val;
var S; LOG && (S = +new Date);
val = r.batch(key);
LOG && console.log(+new Date - S, 'rad mem');
if(u !== val){
cb(u, r.range(val, o), o);
LOG && (S = +new Date);
var v = r.range(val, o);
LOG && console.log(+new Date - S, 'rad range');
cb(u, v, o);
if(atomic(val)){ return }
// if a node is requested and some of it is cached... the other parts might not be.
}
@ -230,19 +235,22 @@
r.read = function(key, cb, o){
o = o || {};
if(RAD && !o.next){ // cache
var S; LOG && (S = +new Date);
var val = RAD(key);
LOG && console.log(+new Date - S, 'rad cached');
//if(u !== val){
//cb(u, val, o);
if(atomic(val)){ cb(u, val, o); return }
// if a node is requested and some of it is cached... the other parts might not be.
//}
}
o.span = (u !== o.start) || (u !== o.end);
o.span = (u !== o.start) || (u !== o.end); // is there a start or end?
var g = function Get(){};
g.lex = function(file){ var tmp;
file = (u === file)? u : decodeURIComponent(file);
tmp = o.next || key || (o.reverse? o.end || '\uffff' : o.start || '');
if(!file || (o.reverse? file < tmp : file > tmp)){
LOG && console.log(+new Date - S, 'rad read lex'); S = +new Date;
if(o.next || o.reverse){ g.file = file }
if(tmp = Q[g.file]){
tmp.push({key: key, ack: cb, file: g.file, opt: o});
@ -263,25 +271,33 @@
g.info = info;
if(disk){ RAD = g.disk = disk }
disk = Q[g.file]; delete Q[g.file];
LOG && console.log(+new Date - S, 'rad read it in, now ack to:', disk.length); S = +new Date;
map(disk, g.ack);
console.log(+new Date - S, 'rad read acked');
}
g.ack = function(as){
if(!as.ack){ return }
var tmp = as.key, o = as.opt, info = g.info, rad = g.disk || noop, data = r.range(rad(tmp), o), last = rad.last || Radix.map(rad, rev, revo);
var key = as.key, o = as.opt, info = g.info, rad = g.disk || noop, data = r.range(rad(key), o), last = rad.last || Radix.map(rad, rev, revo);
o.parsed = (o.parsed || 0) + (info.parsed||0);
o.chunks = (o.chunks || 0) + 1;
if(!o.some){ o.some = (u !== data) }
if(u !== data){ as.ack(g.err, data, o) }
else if(!as.file){ !o.some && as.ack(g.err, u, o); return }
if(!o.span){
if(/*!last || */last === tmp){ !o.some && as.ack(g.err, u, o); return }
if(last && last > tmp && 0 != last.indexOf(tmp)){ !o.some && as.ack(g.err, u, o); return }
o.more = true;
if((!as.file) // if no more places to look
|| (!o.span && last === key) // if our key exactly matches the very last atomic record
|| (!o.span && last && last > key && 0 != last.indexOf(key)) // 'zach' may be lexically larger than 'za', but there still might be more, like 'zane' in the 'za' prefix bucket so do not end here.
){
o.more = u;
as.ack(g.err, data, o);
return
}
if(o.some && o.parsed >= o.limit){ return }
if(u !== data){
as.ack(g.err, data, o); // more might be coming!
if(o.parsed >= o.limit){ return } // even if more, we've hit our limit, asking peer will need to make a new ask with a new starting point.
}
o.next = as.file;
r.read(tmp, as.ack, o);
r.read(key, as.ack, o);
}
if(o.reverse){ g.lex.reverse = true }
LOG && (S = +new Date);
r.list(g.lex);
}
function rev(a,b){ return b }
@ -302,7 +318,7 @@
var p = function Parse(){}, info = {};
p.disk = Radix();
p.read = function(err, data){ var tmp;
LOG && console.log('read disk in', +new Date - start, ename(file)); // keep this commented out in
LOG && console.log('read disk in', +new Date - S, ename(file)); // keep this commented out in
delete Q[file];
if((p.err = err) || (p.not = !data)){
return map(q, p.ack);
@ -319,12 +335,12 @@
}
info.parsed = data.length;
LOG && (start = +new Date); // keep this commented out in production!
if(opt.jsonify){ // temporary testing idea
LOG && (S = +new Date); // keep this commented out in production!
if(opt.jsonify || '{' === data[0]){ // temporary testing idea
try{
var json = JSON.parse(data);
p.disk.$ = json;
LOG && console.log('parsed JSON in', +new Date - start); // keep this commented out in production!
LOG && console.log('parsed JSON in', +new Date - S); // keep this commented out in production!
map(q, p.ack);
return;
}catch(e){ tmp = e }
@ -333,7 +349,7 @@
return map(q, p.ack);
}
}
LOG && (start = +new Date); // keep this commented out in production!
LOG && (S = +new Date); // keep this commented out in production!
var tmp = p.split(data), pre = [], i, k, v;
if(!tmp || 0 !== tmp[1]){
p.err = "File '"+file+"' does not have root radix! ";
@ -356,7 +372,7 @@
if(u !== k && u !== v){ p.disk(pre.join(''), v) }
tmp = p.split(tmp[2]);
}
LOG && console.log('parsed RAD in', +new Date - start); // keep this commented out in production!
LOG && console.log('parsed RAD in', +new Date - S); // keep this commented out in production!
//cb(err, p.disk);
map(q, p.ack);
};
@ -376,7 +392,7 @@
if(p.err || p.not){ return cb(p.err, u, info) }
cb(u, p.disk, info);
}
var start; LOG && (start = +new Date); // keep this commented out in production!
var S; LOG && (S = +new Date); // keep this commented out in production!
if(raw){ return p.read(null, raw) }
opt.store.get(ename(file), p.read);
}

40
lib/store.js
View File

@ -3,7 +3,7 @@ var Gun = (typeof window !== "undefined")? window.Gun : require('../gun');
Gun.on('create', function(root){
if(Gun.TESTING){ root.opt.file = 'radatatest' }
this.to.next(root);
var opt = root.opt, u;
var opt = root.opt, empty = {}, u;
if(false === opt.radisk){ return }
var Radisk = (Gun.window && Gun.window.Radisk) || require('./radisk');
var Radix = Radisk.Radix;
@ -14,14 +14,23 @@ Gun.on('create', function(root){
root.on('put', function(msg){
this.to.next(msg);
var id = msg['#'] || Gun.text.random(3), track = !msg['@'], acks = track? 0 : u; // only ack non-acks.
if(msg.rad && !track){ return } // don't save our own acks
var start = (+new Date); // STATS!
var got = (msg._||empty).rad, now = Gun.state();
var S = (+new Date); // STATS!
Gun.graph.is(msg.put, null, function(val, key, node, soul){
if(!track && got){
var at = (root.next||empty)[soul];
if(!at){ return }
if(u !== got['.']){ at = (at.next||empty)[key] }
if(!at){ return }
at.rad = now;
return;
}
if(track){ ++acks }
//console.log('put:', soul, key, val);
val = Radisk.encode(val, null, esc)+'>'+Radisk.encode(Gun.state.is(node, key), null, esc);
rad(soul+esc+key, val, (track? ack : u));
});
console.log(+new Date - S, 'put loop');
function ack(err, ok){
acks--;
if(ack.err){ return }
@ -31,11 +40,12 @@ Gun.on('create', function(root){
return;
}
if(acks){ return }
try{opt.store.stats.put.time[statp % 50] = (+new Date) - start; ++statp;
try{opt.store.stats.put.time[statp % 50] = (+new Date) - S; ++statp;
opt.store.stats.put.count++;
}catch(e){} // STATS!
//console.log("PAT!", id);
console.log(+new Date - S, 'put'); S = +new Date;
root.on('in', {'@': id, ok: 1});
console.log(+new Date - S, 'put sent');
}
});
@ -67,36 +77,48 @@ Gun.on('create', function(root){
o.limit = (tmp <= (o.pack || (1000 * 100)))? tmp : 1;
}
if(has['-'] || (soul||{})['-']){ o.reverse = true }
var start = (+new Date); // STATS! // console.log("GET!", id, JSON.stringify(key));
if(tmp = (root.next||empty)[soul]){
if(tmp && tmp.rad){ return }
if(o.atom){ tmp = (tmp.next||empty)[o.atom] }
if(tmp && tmp.rad){ return }
}
var S = (+new Date); // STATS! // console.log("GET!", id, JSON.stringify(key));
rad(key||'', function(err, data, o){
try{opt.store.stats.get.time[statg % 50] = (+new Date) - start; ++statg;
try{opt.store.stats.get.time[statg % 50] = (+new Date) - S; ++statg;
opt.store.stats.get.count++;
if(err){ opt.store.stats.get.err = err }
}catch(e){} // STATS!
//if(u === data && o.chunks > 1){ return } // if we already sent a chunk, ignore ending empty responses. // this causes tests to fail.
console.log(+new Date - S, 'got'); S = +new Date;
if(data){
if(typeof data !== 'string'){
if(o.atom){
data = u;
Radix.map(data, each)
} else {
Radix.map(data, each)
}
}
if(!graph && data){ each(data, '') }
}
root.on('in', {'@': id, put: graph, err: err? err : u, rad: Radix});
console.log(+new Date - S, 'got prep'); S = +new Date;
root.on('in', {'@': id, put: graph, '%': o.more? 1 : u, err: err? err : u, _: each});
console.log(+new Date - S, 'got sent');
}, o);
console.log(+new Date - S, 'get call');
function each(val, has, a,b){
if(!val){ return }
has = (key+has).split(esc);
var soul = has.slice(0,1)[0];
has = has.slice(-1)[0];
o.count = (o.count || 0) + val.length;
tmp = val.lastIndexOf('>');
var tmp = val.lastIndexOf('>');
var state = Radisk.decode(val.slice(tmp+1), null, esc);
val = Radisk.decode(val.slice(0,tmp), null, esc);
(graph = graph || {})[soul] = Gun.state.ify(graph[soul], has, state, val, soul);
if(o.limit && o.limit <= o.count){ return true }
}
each.rad = get;
});
opt.store.stats = {get:{time:{}, count:0}, put: {time:{}, count:0}}; // STATS!
var statg = 0, statp = 0; // STATS!

3
test/rad/rad.js
View File

@ -190,12 +190,13 @@ var names = ["Adalard","Adora","Aia","Albertina","Alfie","Allyn","Amabil","Ammam
if(opt.end < v){ return }
if(v.indexOf(find) == 0){ all[v] = true }
});
rad(find, function(err, data){
rad(find, function(err, data, o){
Radix.map(data, function(v,k){
//console.log(find+k, v);
delete all[find+k];
});
if(!Gun.obj.empty(all)){ return }
if(!data){ return } // in case there is "more" that returned empty
done();
}, opt);
});