From ba2b207dd66efeda7779d4d457afd56b652e2208 Mon Sep 17 00:00:00 2001 From: Mark Nadal Date: Wed, 22 Jan 2020 06:56:57 -0800 Subject: [PATCH 1/2] hopefully increases limits? --- gun.js | 2 +- lib/evict.js | 5 +++-- lib/server.js | 2 +- lib/store.js | 7 ++++++- 4 files changed, 11 insertions(+), 5 deletions(-) diff --git a/gun.js b/gun.js index 63452fe8..608eabe1 100644 --- a/gun.js +++ b/gun.js @@ -619,7 +619,7 @@ var Type = USE('./type'); function Dup(opt){ var dup = {s:{}}; - opt = opt || {max: 1000, age: /*1000 * 9};//*/ 1000 * 9 * 3}; + opt = opt || {max: 1000, age: /*1000 * 9};//*/ 1000 * 60 * 2}; dup.check = function(id){ var tmp; if(!(tmp = dup.s[id])){ return false } if(tmp.pass){ return tmp.pass = false } diff --git a/lib/evict.js b/lib/evict.js index 3186eb77..ff66b366 100644 --- a/lib/evict.js +++ b/lib/evict.js @@ -16,7 +16,8 @@ function check(){ var used = util().rss / 1024 / 1024; var hused = heap().used_heap_size / 1024 / 1024; - if(hused < ev.max && used < ev.max){ return } + //if(hused < ev.max && used < ev.max){ return } + if(used < ev.max){ return } console.LOG && Gun.log('evict memory:', hused.toFixed(), used.toFixed(), ev.max.toFixed()); GC();//setTimeout(GC, 1); } @@ -28,7 +29,7 @@ if(--toss < 0){ return } root.gun.get(soul).off(); }); - root.dup.drop(1000 * 9); // clean up message tracker + root.dup.drop(1000 * 45); // clean up message tracker } /* root.on('in', function(msg){ diff --git a/lib/server.js b/lib/server.js index ad8c8a5c..7b8be474 100644 --- a/lib/server.js +++ b/lib/server.js @@ -2,7 +2,7 @@ var Gun = require('../gun'), u; Gun.serve = require('./serve'); //process.env.GUN_ENV = process.env.GUN_ENV || 'debug'; - //console.LOG = true; // only do this for dev. + console.LOG = true; // only do this for dev. Gun.on('opt', function(root){ if(u === root.opt.super){ root.opt.super = true } if(u === root.opt.faith){ root.opt.faith = true } // HNPERF: This should probably be off, but we're testing performance improvements, please audit. diff --git a/lib/store.js b/lib/store.js index c923d7b2..734ea585 100644 --- a/lib/store.js +++ b/lib/store.js @@ -79,11 +79,16 @@ Gun.on('create', function(root){ } if(has['-'] || (soul||{})['-']){ o.reverse = true } if((tmp = (root.next||empty)[soul]) && tmp.put){ + var SPUT = tmp.put; if(o.atom){ tmp = (tmp.next||empty)[o.atom] ; - if(tmp && tmp.rad){ return } + if(tmp && tmp.rad){ + LOG && Gun.log("still cached atom", JSON.stringify(get), Object.keys(SPUT||{}).length); + return; + } } else if(tmp && tmp.rad){ + LOG && Gun.log("still cached", JSON.stringify(get), Object.keys(SPUT||{}).length); return; } } From 8f874d6f329b8bf9f61fc4d00b5f858684b0d264 Mon Sep 17 00:00:00 2001 From: Mark Nadal Date: Wed, 22 Jan 2020 22:39:02 -0800 Subject: [PATCH 2/2] try relocate mislocated data on reads --- gun.js | 2 +- lib/evict.js | 2 +- lib/radisk.js | 145 ++++++++++++++++++++++++++++++++++++++------------ lib/radix.js | 13 +++-- lib/rfs.js | 8 +++ package.json | 2 +- 6 files changed, 132 insertions(+), 40 deletions(-) diff --git a/gun.js b/gun.js index 608eabe1..63452fe8 100644 --- a/gun.js +++ b/gun.js @@ -619,7 +619,7 @@ var Type = USE('./type'); function Dup(opt){ var dup = {s:{}}; - opt = opt || {max: 1000, age: /*1000 * 9};//*/ 1000 * 60 * 2}; + opt = opt || {max: 1000, age: /*1000 * 9};//*/ 1000 * 9 * 3}; dup.check = function(id){ var tmp; if(!(tmp = dup.s[id])){ return false } if(tmp.pass){ return tmp.pass = false } diff --git a/lib/evict.js b/lib/evict.js index ff66b366..8e0023bf 100644 --- a/lib/evict.js +++ b/lib/evict.js @@ -29,7 +29,7 @@ if(--toss < 0){ return } root.gun.get(soul).off(); }); - root.dup.drop(1000 * 45); // clean up message tracker + root.dup.drop(1000 * 9); // clean up message tracker } /* root.on('in', function(msg){ diff --git a/lib/radisk.js b/lib/radisk.js index 8f3c552a..8c873546 100644 --- a/lib/radisk.js +++ b/lib/radisk.js @@ -108,47 +108,84 @@ 6. Merge and write all of those to the in-memory file and back to disk. 7. If file too large, split. More details needed here. */ + /* NEW APPROACH: + 1. For each item in radix memory + 2. Add it to a radix bucket corresponding to directory of files + 3. Iterate over each bucket + 4. Resume old approach. + */ r.save = function(rad, cb){ - var s = function Span(){}; - s.find = function(tree, key){ - if(key < s.start){ return } - s.start = key; - r.list(s.lex); - return true; + if(r.save.ing){ + r.save.ing.push({rad: rad, ack: cb}); + return; } - s.lex = function(file){ - file = (u === file)? u : decodeURIComponent(file); - if(!file || file > s.start){ - s.mix(s.file || opt.code.from, s.start, s.end = file, s.file); + //console.only(99); var ID = Gun.text.random(2), S = (+new Date); console.log("[[[[[[[[", ID); + r.save.ing = []; + var ack = cb; + var s = function Span(err, ok){ + var tmp = r.save.ing; + //console.only(99); var TMP; console.log("]]]]]]]]", ID, (TMP = +new Date) - S, 'more?', !!tmp); + r.save.ing = null; + map(tmp, function(q){ // if many, not the most efficient to requeue, but works for now. + if(!q || !q.rad || !q.ack){ return } + r.save(q.rad, q.ack); + }) + ack(err, ok); + }; + cb = s; + s.files = {}; + s.i = 0; // TODO: revise? Using counter for critical path not my favorite. + s.place = function(tree, key){ + var go = function(file, last){ + file = decodeURIComponent(file || last || opt.code.from); + (s.files[file] || (s.files[file] = Radix()))(key, tree); + if(!(--s.i)){ s.go() } // TODO: See above, revise? return true; } - s.file = file; + go.reverse = 1; + go.end = key; + r.list(go); + ++s.i; // TODO: See above, revise? } - s.mix = function(file, start, end, f){ - s.start = s.end = s.file = u; + s.go = function(){ + if(s.gone){ return } s.gone = true; + s.seq = []; + map(s.files, function(mem, file){ s.seq.push({file: file, mem: mem}) }); + s.files = null; + s.c = 0; + s.merge(s.c); + } + s.merge = function(i){ + i = i || 0; + //var at = s.seq[i]; + var at = s.seq.shift(); + if(!at){ + if(s.ok){ return cb(null, s.ok) } + return cb("No file to save data to."); + } + var file = at.file, mem = at.mem; r.parse(file, function(err, disk){ - if(err){ return cb(err) } - if(!disk && f){ // corrupt file? - r.list.bad(f); // remove from dir list + if(err){ return cb(err) } + if(!disk && file !== opt.code.from){ // corrupt file? + r.list.bad(file); // remove from dir list r.save(rad, cb); // try again return; } disk = disk || Radix(); - Radix.map(rad, function(val, key){ - if(key < start){ return } - if(end && end < key){ return s.start = key } + Radix.map(mem, function(val, key){ // PLUGIN: consider adding HAM as an extra layer of protection disk(key, val); // merge batch[key] -> disk[key] }); - r.write(file, disk, s.next); - }); + r.write(file, disk, s.pop); + }) } - s.next = function(err, ok){ - if(s.err = err){ return cb(err) } - if(s.start){ return Radix.map(rad, s.find) } - cb(err, ok); + s.pop = function(err, ok){ + if(s.err = err || s.err){ return cb(err) } + s.ok = ok || s.ok || 1; + s.merge(++s.c); } - Radix.map(rad, s.find); + Radix.map(rad, s.place); + if(!s.i){ s.go() }; // TODO: See above, revise? } /* @@ -253,7 +290,6 @@ file = (u === file)? u : decodeURIComponent(file); tmp = o.next || key || (o.reverse? o.end || '\uffff' : o.start || ''); if(!file || (o.reverse? file < tmp : file > tmp)){ - console.only(2, 'RAD LEX:', JSON.stringify(tmp), 'in', JSON.stringify(g.file), !file, JSON.stringify(file) > JSON.stringify(tmp), file > tmp, JSON.stringify(file), o); LOG && opt.log(S, +new Date - S, 'rad read lex'); S = +new Date; if(o.next || o.reverse){ g.file = file } if(tmp = Q[g.file]){ @@ -265,7 +301,7 @@ g.it(null, u, {}); return true; } - r.parse(g.file, g.it); + r.parse(g.file, g.check); return true; } g.file = file; @@ -308,6 +344,49 @@ o.next = as.file; r.read(key, as.ack, o); } + g.check = function(err, disk, info){ + g.it(err, disk, info); + var good = true; + Radix.map(disk, function(val, key){ + // assume in memory for now, since both write/read already call r.list which will init it. + var go = function(file){ + if(info.file !== file){ + good = false + } + return true; + } + go.reverse = 1; + go.end = key; + r.list(go); + }); + if(good){ return } + var id = Gun.text.random(3); console.log("MISLOCATED DATA", id); + r.save(disk, function ack(err, ok){ + if(err){ return r.save(disk, ack) } // ad infinitum??? + console.log("MISLOCATED CORRECTED", id); + }); + } + /*g.check2 = function(err, disk, info){ + if(err || !disk){ return g.it(err, disk, info) } + var good = true; + Radix.map(disk, function(val, key){ + // assume in memory for now, since both write/read already call r.list which will init it. + var go = function(file){ + if(info.file !== file){ good = false } + return true; + } + go.reverse = 1; + go.end = key; + r.list(go); + }); + if(good){ return g.it(err, disk, info) } + var id = Gun.text.random(3); console.log("MISLOCATED DATA", id); + r.save(disk, function ack(err, ok){ + if(err){ return r.save(disk, ack) } // ad infinitum??? + console.log("MISLOCATED CORRECTED", id); + r.read(key, cb, o); + }); + }*/ if(o.reverse){ g.lex.reverse = true } LOG && (S = +new Date); r.list(g.lex); @@ -327,7 +406,7 @@ var Q = {}, s = String.fromCharCode(31); r.parse = function(file, cb, raw){ var q; if(q = Q[file]){ return q.push(cb) } q = Q[file] = [cb]; - var p = function Parse(){}, info = {}; + var p = function Parse(){}, info = {file: ename(file)}; p.disk = Radix(); p.read = function(err, data){ var tmp; LOG && opt.log(S, +new Date - S, 'read disk', ename(file)); @@ -414,11 +493,11 @@ var dir, q, f = String.fromCharCode(28), ef = ename(f); r.list = function(cb){ if(dir){ - var tmp = {reverse: (cb.reverse)? 1 : 0}; + var last, tmp = {reverse: (cb.reverse)? 1 : 0, start: cb.start, end: cb.end}; Radix.map(dir, function(val, key){ if(!val){ return } - return cb(key); - }, tmp) || cb(); + return cb(last = key); + }, tmp) || cb(u, last); return; } if(q){ return q.push(cb) } q = [cb]; @@ -450,7 +529,7 @@ } if(disk){ r.list.drain(disk); - //return; + return; } if(!opt.store.list){ r.list.drain(Radix()); diff --git a/lib/radix.js b/lib/radix.js index e4533f32..441f9540 100644 --- a/lib/radix.js +++ b/lib/radix.js @@ -58,21 +58,26 @@ //var keys = Object.keys(t).sort(); opt = (true === opt)? {branch: true} : (opt || {}); if(rev = opt.reverse){ keys = keys.slice().reverse() } - var start = opt.start, end = opt.end; + var start = opt.start, end = opt.end, END = '\uffff'; var i = 0, l = keys.length; for(;i < l; i++){ var key = keys[i], tree = t[key], tmp, p, pt; if(!tree || '' === key || _ === key){ continue } p = pre.slice(); p.push(key); pt = p.join(''); if(u !== start && pt < (start||'').slice(0,pt.length)){ continue } - if(u !== end && (end || '\uffff') < pt){ continue } + if(u !== end && (end || END) < pt){ continue } if(rev){ // children must be checked first when going in reverse. tmp = map(tree, cb, opt, p); if(u !== tmp){ return tmp } } if(u !== (tmp = tree[''])){ - tmp = cb(tmp, pt, key, pre); - if(u !== tmp){ return tmp } + var yes = 1; + if(u !== start && pt < (start||'')){ yes = 0 } + if(u !== end && pt > (end || END)){ yes = 0 } + if(yes){ + tmp = cb(tmp, pt, key, pre); + if(u !== tmp){ return tmp } + } } else if(opt.branch){ tmp = cb(u, pt, key, pre); diff --git a/lib/rfs.js b/lib/rfs.js index 6ed7d131..8a2e805b 100644 --- a/lib/rfs.js +++ b/lib/rfs.js @@ -56,6 +56,14 @@ function Store(opt){ } }); }; + + store.list = function(cb, match, params, cbs){ + var dir = fs.readdirSync(opt.file); + dir.forEach(function(file){ + cb(file); + }) + cb(); + }; return store; } diff --git a/package.json b/package.json index 01c9a35f..e31efe13 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "gun", - "version": "0.2020.115", + "version": "0.2020.116", "description": "A realtime, decentralized, offline-first, graph data synchronization engine.", "main": "index.js", "browser": "browser.js",