mirror of
https://github.com/amark/gun.git
synced 2025-03-30 15:08:33 +00:00
cleaned up Book results & sorting & caching
This commit is contained in:
Mark Nadal
parent
3688ba1cc6
commit
203bd40932
60
gun.js
60
gun.js
@ -183,7 +183,7 @@
|
||||
}
|
||||
function got(word, page){
|
||||
var b = page.book, l, has, a, i;
|
||||
if(l = from(page)){ has = l[got.i = i = spot(word, l, B.decode)]; } // TODO: POTENTIAL BUG! This assumes that each word on a page uses the same serializer/formatter/structure.
|
||||
if(l = from(page)){ has = l[got.i = i = spot(word, l, B.decode)]; } // TODO: POTENTIAL BUG! This assumes that each word on a page uses the same serializer/formatter/structure. // TOOD: BUG!!! Not actually, but if we want to do non-exact radix-like closest-word lookups on a page, we need to check limbo & potentially sort first.
|
||||
// parseless may return -1 from actual value, so we may need to test both. // TODO: Double check? I think this is correct.
|
||||
if(has && word == has.word){ return (b.all[word] = has).is }
|
||||
if('string' != typeof has){ has = l[got.i = i+=1] }
|
||||
@ -208,14 +208,14 @@
|
||||
|
||||
function from(a, t, l){
|
||||
if('string' != typeof a.from){ return a.from }
|
||||
//(l = a.from = (t = a.from||'').substring(1, t.length-1).split(t[0])).toString = join; // slot
|
||||
(l = a.from = slot(t = t||a.from||'')).toString = join;
|
||||
//(l = a.from = (t = a.from||'').substring(1, t.length-1).split(t[0])); // slot
|
||||
(l = a.from = slot(t = t||a.from||''));
|
||||
return l;
|
||||
}
|
||||
function list(each){ each = each || function(x){return x}
|
||||
// TODO: BUG!!! in limbo items need to get situated before calling this, if there are any. (obviously, we shouldn't do it again if limbo has previously been sorted).
|
||||
var i = 0, l = from(this)||[], w, r = [], p = this.book.parse || function(){};
|
||||
while(w = l[i++]){ r.push(each(slot(w)[1],p(w)||w,this)) }
|
||||
var i = 0, l = sort(this), w, r = [], p = this.book.parse || function(){};
|
||||
//while(w = l[i++]){ r.push(each(slot(w)[1], p(w)||w, this)) }
|
||||
while(w = l[i++]){ r.push(each(this.get(w = w.word||p(w)||w), w, this)) } // TODO: BUG! PERF?
|
||||
return r;
|
||||
}
|
||||
|
||||
@ -230,7 +230,7 @@
|
||||
// MUST be an insert:
|
||||
has = b.all[word] = {word: word, is: is, page: page, substring: subt, toString: tot};
|
||||
page.first = (page.first < word)? page.first : word;
|
||||
if(!page.limbo){ (page.limbo = []).toString = join }
|
||||
if(!page.limbo){ (page.limbo = []) }
|
||||
page.limbo.push(has);
|
||||
b(word, is);
|
||||
page.size += size(word) + size(is);
|
||||
@ -240,24 +240,18 @@
|
||||
|
||||
function split(p, b){ // TODO: use closest hash instead of half.
|
||||
//console.time();
|
||||
// TODO: BUG???? May need to do a SORTED merge with FROM.
|
||||
var i = 0, L = p.limbo, tmp;
|
||||
//while(tmp = L[i++]){ }
|
||||
var L = p.limbo = sort(p.limbo), l = L.length, i = l/2 >> 0, j = i, half = L[j], tmp;
|
||||
var L = sort(p), l = L.length, i = l/2 >> 0, j = i, half = L[j], tmp;
|
||||
//console.timeEnd();
|
||||
var next = {limbo: [], first: half.substring(), size: 0, substring: sub, toString: to, book: b, get: b, read: list}, nl = next.limbo;
|
||||
nl.toString = join;
|
||||
var next = {first: half.substring(), size: 0, substring: sub, toString: to, book: b, get: b, read: list}, f = next.from = [];
|
||||
//console.time();
|
||||
while(tmp = L[i++]){
|
||||
nl.push(tmp);
|
||||
f.push(tmp);
|
||||
next.size += (tmp.is||'').length||1;
|
||||
tmp.page = next;
|
||||
}
|
||||
//console.timeEnd();
|
||||
//console.time();
|
||||
p.limbo = p.limbo.slice(0, j);
|
||||
//console.timeEnd(); console.time();
|
||||
p.from = p.from.slice(0, j);
|
||||
p.size -= next.size;
|
||||
p.sort = 0;
|
||||
b.list.splice(spot(next.first, b.list)+1, 0, next); // TODO: BUG! Make sure next.first is decoded text. // TODO: BUG! spot may need parse too?
|
||||
//console.timeEnd();
|
||||
if(b.split){ b.split(next, p) }
|
||||
@ -285,27 +279,29 @@
|
||||
}
|
||||
function sub(i,j){ return (this.first||this.word||B.decode((from(this)||'')[0]||'')).substring(i,j) }
|
||||
function to(){ return this.text = this.text || text(this) }
|
||||
function join(){ return this.join('|') }
|
||||
function text(p){ var l = p.limbo; // TODO: BUG??? Shouldn't any stringify cause limbo to be reset?
|
||||
if(!l){ return (typeof p.from == 'string')? (p.from||'')+'' : '|'+p.from+'|' }
|
||||
if(!p.from){ return p.limbo = null, '|'+((l && sort(l).join('|'))||'')+'|' } // TODO: p.limbo should be reset each time we "flush".
|
||||
return '|'+mix(l, from(p), p).join('|')+'|'; // commenting out this sub-portion of code fixed a more basic test, but will probably cause a bug with a FROM + MEMORY.
|
||||
function text(p){ // PERF: read->[*] : text->"*" no edit waste 1 time perf.
|
||||
if(p.limbo){ sort(p) } // TODO: BUG? Empty page meaning? undef, '', '||'?
|
||||
return ('string' == typeof p.from)? p.from : '|'+(p.from||[]).join('|')+'|';
|
||||
}
|
||||
function mix(l, f, p){ // TODO: IMPROVE PERFORMANCE!!!! l[j] = i is 5X+ faster than .push(
|
||||
var j = 0, i;
|
||||
|
||||
function sort(p, l){
|
||||
var f = p.from = ('string' == typeof p.from)? slot(p.from) : p.from||[];
|
||||
if(!(l = l || p.limbo)){ return f }
|
||||
return mix(p).sort(function(a,b){
|
||||
return (a.word||B.decode(''+a)) < (b.word||B.decode(''+b))? -1:1;
|
||||
});
|
||||
}
|
||||
function mix(p, l){ // TODO: IMPROVE PERFORMANCE!!!! l[j] = i is 5X+ faster than .push(
|
||||
l = l || p.limbo || []; p.limbo = null;
|
||||
var j = 0, i, f = p.from;
|
||||
while(i = l[j++]){
|
||||
if(got(i.word, p)){
|
||||
f[got.i] = i;
|
||||
f[got.i] = i; // TODO: Trick: allow for a GUN'S HAM CRDT hook here.
|
||||
} else {
|
||||
f.push(i);
|
||||
}
|
||||
}
|
||||
return sort(f);
|
||||
}
|
||||
function sort(l){ //return l.sort();
|
||||
return l.sort(function(a,b){
|
||||
return (a.word||B.decode(''+a)) < (b.word||B.decode(''+b))? -1:1;
|
||||
});
|
||||
return f;
|
||||
}
|
||||
|
||||
B.encode = function(d, s, u){ s = s || "|"; u = u || String.fromCharCode(32);
|
||||
|
||||
40
rad.js
40
rad.js
@ -6,7 +6,7 @@
|
||||
var log = opt.log || nope;
|
||||
|
||||
var has = (sT.RAD.has || (sT.RAD.has = {}))[opt.file];
|
||||
if(has){ return has }
|
||||
if(has){ return has } // TODO: BUG? Not reuses same instance?
|
||||
var r = function rad(word, is, reply){ r.word = word;
|
||||
if(!b){ start(word, is, reply); return r }
|
||||
if(is === undefined || 'function' == typeof is){ // THIS IS A READ:
|
||||
@ -35,29 +35,18 @@
|
||||
})
|
||||
}
|
||||
|
||||
async function write(word, reply){
|
||||
log('write() word', word);
|
||||
function write(word, reply){
|
||||
var p = b.page(word), tmp;
|
||||
if(tmp = p.saving){ reply && tmp.push(reply); return } p.saving = [reply];
|
||||
var S = +new Date; log(" writing", p.substring(), 'since last', S - p.saved, RAD.c, 'records', env.count++, 'mid-swap.');
|
||||
if(tmp=p.saving){(reply||!tmp.length)&&(p.saving=tmp.concat(reply));return} // TODO: PERF! Rogowski points out concat is slow. BUG??? I HAVE NO clue how/why this if statement being called from recursion yet not set to 0.
|
||||
p.saving = ('function' == typeof reply)? [reply] : reply || [];
|
||||
get(p, function(err, disk){
|
||||
if(err){ log("ERR! in write() get() cb ", err); return }
|
||||
log(' get() - p.saving ', (p.saving || []).length);
|
||||
if(p.from && disk){
|
||||
log(" get() merge: p.from ", p.toString().slice(0, 40), " disk.length", disk?.length || 0);
|
||||
}
|
||||
if(err){ log("ERR! in write() get() cb ", err); return } // TODO: BUG!!! Unhandled, no callbacks called.
|
||||
p.from = disk || p.from;
|
||||
// p.list = p.text = p.from = 0;
|
||||
// p.first = p.first.word || p.first;
|
||||
tmp = p.saving; p.saving = [];
|
||||
put(p, '' + p, function(err, ok){
|
||||
env.count--; p.saved = +new Date; log(" ...wrote %d bytes in %dms", ('' + p).length, (p.saved = +new Date) - S);
|
||||
// TODO: BUG: Confirmed! Only calls back first. Need to fix + use perf hack from old RAD.
|
||||
put(p, ''+p, function(err, ok){
|
||||
sT.each(tmp, function(cb){ cb && cb(err, ok) });
|
||||
if(!p.saving.length){ p.saving = 0; return; } //p.saving = 0; // what?
|
||||
// log({ tmp });
|
||||
console.log("hm?", word, reply+'');
|
||||
write(word, reply);
|
||||
tmp = p.saving; p.saving = 0;
|
||||
if(tmp.length){ write(word, tmp) }
|
||||
});
|
||||
}, p);
|
||||
}
|
||||
@ -71,13 +60,13 @@
|
||||
function get(file, cb){
|
||||
var tmp;
|
||||
if(!file){ return } // TODO: HANDLE ERROR!!
|
||||
if(file.from){ cb(null, file.from); return } // IS THIS LINE SAFE? ADD TESTS!
|
||||
if(file.from){ cb(null, file.from); return }
|
||||
if(b&&1==b.list.length){ file.first = (file.first < '!')? file.first : '!'; } // TODO: BUG!!!! This cleanly makes for a common first file, but SAVING INVISIBLE ASCII KEYS IS COMPLETELY UNTESTED and guaranteed to have bugs/corruption issues.
|
||||
if(tmp = put[file = fname(file)]){ cb(u, tmp.data); return }
|
||||
if(tmp = get[file]){ tmp.push(cb); return } get[file] = [cb];
|
||||
RAD.get(file, function(err, data){
|
||||
tmp = get[file]||''; delete get[file];
|
||||
var i = -1, f; while (f = tmp[++i]){ f(err, data) } // TODO: BUG! CPU SCHEDULE?
|
||||
sT.each(tmp, function(cb){ cb && cb(err, data) });
|
||||
}, opt);
|
||||
};
|
||||
|
||||
@ -104,7 +93,6 @@
|
||||
return t;
|
||||
}
|
||||
b.split = function(next, page){
|
||||
log("SPLIT!!!!", b.list.length);
|
||||
put(' ', '' + b.list, function(err, ok){
|
||||
if(err){ console.log("ERR!"); return }
|
||||
// ??
|
||||
@ -117,12 +105,6 @@
|
||||
//function fname(p){ return opt.file + '/' + ename(p.substring()) }
|
||||
function fname(p){ return ename(p.substring()) }
|
||||
|
||||
|
||||
function valid(word, is, reply){
|
||||
if(is !== is){ reply(word +" cannot be NaN!"); return }
|
||||
return true;
|
||||
}
|
||||
|
||||
function valid(word, is, reply){
|
||||
if(is !== is){ reply(word +" cannot be NaN!"); return }
|
||||
return true;
|
||||
@ -210,7 +192,7 @@
|
||||
cb(401)
|
||||
}
|
||||
RAD.get = async function(file, cb, opt){ get && get(file, cb, opt);
|
||||
var t = (await (await fetch('http://localhost:8766/gun/1data/'+file)).text());
|
||||
var t = (await (await fetch('http://localhost:8765/gun/authorsData/'+file)).text());
|
||||
if('404' == t){ cb(); return }
|
||||
cb(null, t);
|
||||
}
|
||||
|
||||
60
src/book.js
60
src/book.js
@ -45,7 +45,7 @@ function get(word){
|
||||
}
|
||||
function got(word, page){
|
||||
var b = page.book, l, has, a, i;
|
||||
if(l = from(page)){ has = l[got.i = i = spot(word, l, B.decode)]; } // TODO: POTENTIAL BUG! This assumes that each word on a page uses the same serializer/formatter/structure.
|
||||
if(l = from(page)){ has = l[got.i = i = spot(word, l, B.decode)]; } // TODO: POTENTIAL BUG! This assumes that each word on a page uses the same serializer/formatter/structure. // TOOD: BUG!!! Not actually, but if we want to do non-exact radix-like closest-word lookups on a page, we need to check limbo & potentially sort first.
|
||||
// parseless may return -1 from actual value, so we may need to test both. // TODO: Double check? I think this is correct.
|
||||
if(has && word == has.word){ return (b.all[word] = has).is }
|
||||
if('string' != typeof has){ has = l[got.i = i+=1] }
|
||||
@ -70,14 +70,14 @@ function spot(word, sorted, parse){ parse = parse || spot.no || (spot.no = funct
|
||||
|
||||
function from(a, t, l){
|
||||
if('string' != typeof a.from){ return a.from }
|
||||
//(l = a.from = (t = a.from||'').substring(1, t.length-1).split(t[0])).toString = join; // slot
|
||||
(l = a.from = slot(t = t||a.from||'')).toString = join;
|
||||
//(l = a.from = (t = a.from||'').substring(1, t.length-1).split(t[0])); // slot
|
||||
(l = a.from = slot(t = t||a.from||''));
|
||||
return l;
|
||||
}
|
||||
function list(each){ each = each || function(x){return x}
|
||||
// TODO: BUG!!! in limbo items need to get situated before calling this, if there are any. (obviously, we shouldn't do it again if limbo has previously been sorted).
|
||||
var i = 0, l = from(this)||[], w, r = [], p = this.book.parse || function(){};
|
||||
while(w = l[i++]){ r.push(each(slot(w)[1],p(w)||w,this)) }
|
||||
var i = 0, l = sort(this), w, r = [], p = this.book.parse || function(){};
|
||||
//while(w = l[i++]){ r.push(each(slot(w)[1], p(w)||w, this)) }
|
||||
while(w = l[i++]){ r.push(each(this.get(w = w.word||p(w)||w), w, this)) } // TODO: BUG! PERF?
|
||||
return r;
|
||||
}
|
||||
|
||||
@ -92,7 +92,7 @@ function set(word, is){
|
||||
// MUST be an insert:
|
||||
has = b.all[word] = {word: word, is: is, page: page, substring: subt, toString: tot};
|
||||
page.first = (page.first < word)? page.first : word;
|
||||
if(!page.limbo){ (page.limbo = []).toString = join }
|
||||
if(!page.limbo){ (page.limbo = []) }
|
||||
page.limbo.push(has);
|
||||
b(word, is);
|
||||
page.size += size(word) + size(is);
|
||||
@ -102,24 +102,18 @@ function set(word, is){
|
||||
|
||||
function split(p, b){ // TODO: use closest hash instead of half.
|
||||
//console.time();
|
||||
// TODO: BUG???? May need to do a SORTED merge with FROM.
|
||||
var i = 0, L = p.limbo, tmp;
|
||||
//while(tmp = L[i++]){ }
|
||||
var L = p.limbo = sort(p.limbo), l = L.length, i = l/2 >> 0, j = i, half = L[j], tmp;
|
||||
var L = sort(p), l = L.length, i = l/2 >> 0, j = i, half = L[j], tmp;
|
||||
//console.timeEnd();
|
||||
var next = {limbo: [], first: half.substring(), size: 0, substring: sub, toString: to, book: b, get: b, read: list}, nl = next.limbo;
|
||||
nl.toString = join;
|
||||
var next = {first: half.substring(), size: 0, substring: sub, toString: to, book: b, get: b, read: list}, f = next.from = [];
|
||||
//console.time();
|
||||
while(tmp = L[i++]){
|
||||
nl.push(tmp);
|
||||
f.push(tmp);
|
||||
next.size += (tmp.is||'').length||1;
|
||||
tmp.page = next;
|
||||
}
|
||||
//console.timeEnd();
|
||||
//console.time();
|
||||
p.limbo = p.limbo.slice(0, j);
|
||||
//console.timeEnd(); console.time();
|
||||
p.from = p.from.slice(0, j);
|
||||
p.size -= next.size;
|
||||
p.sort = 0;
|
||||
b.list.splice(spot(next.first, b.list)+1, 0, next); // TODO: BUG! Make sure next.first is decoded text. // TODO: BUG! spot may need parse too?
|
||||
//console.timeEnd();
|
||||
if(b.split){ b.split(next, p) }
|
||||
@ -147,27 +141,29 @@ function tot(){ var tmp = {};
|
||||
}
|
||||
function sub(i,j){ return (this.first||this.word||B.decode((from(this)||'')[0]||'')).substring(i,j) }
|
||||
function to(){ return this.text = this.text || text(this) }
|
||||
function join(){ return this.join('|') }
|
||||
function text(p){ var l = p.limbo; // TODO: BUG??? Shouldn't any stringify cause limbo to be reset?
|
||||
if(!l){ return (typeof p.from == 'string')? (p.from||'')+'' : '|'+p.from+'|' }
|
||||
if(!p.from){ return p.limbo = null, '|'+((l && sort(l).join('|'))||'')+'|' } // TODO: p.limbo should be reset each time we "flush".
|
||||
return '|'+mix(l, from(p), p).join('|')+'|'; // commenting out this sub-portion of code fixed a more basic test, but will probably cause a bug with a FROM + MEMORY.
|
||||
function text(p){ // PERF: read->[*] : text->"*" no edit waste 1 time perf.
|
||||
if(p.limbo){ sort(p) } // TODO: BUG? Empty page meaning? undef, '', '||'?
|
||||
return ('string' == typeof p.from)? p.from : '|'+(p.from||[]).join('|')+'|';
|
||||
}
|
||||
function mix(l, f, p){ // TODO: IMPROVE PERFORMANCE!!!! l[j] = i is 5X+ faster than .push(
|
||||
var j = 0, i;
|
||||
|
||||
function sort(p, l){
|
||||
var f = p.from = ('string' == typeof p.from)? slot(p.from) : p.from||[];
|
||||
if(!(l = l || p.limbo)){ return f }
|
||||
return mix(p).sort(function(a,b){
|
||||
return (a.word||B.decode(''+a)) < (b.word||B.decode(''+b))? -1:1;
|
||||
});
|
||||
}
|
||||
function mix(p, l){ // TODO: IMPROVE PERFORMANCE!!!! l[j] = i is 5X+ faster than .push(
|
||||
l = l || p.limbo || []; p.limbo = null;
|
||||
var j = 0, i, f = p.from;
|
||||
while(i = l[j++]){
|
||||
if(got(i.word, p)){
|
||||
f[got.i] = i;
|
||||
f[got.i] = i; // TODO: Trick: allow for a GUN'S HAM CRDT hook here.
|
||||
} else {
|
||||
f.push(i);
|
||||
}
|
||||
}
|
||||
return sort(f);
|
||||
}
|
||||
function sort(l){ //return l.sort();
|
||||
return l.sort(function(a,b){
|
||||
return (a.word||B.decode(''+a)) < (b.word||B.decode(''+b))? -1:1;
|
||||
});
|
||||
return f;
|
||||
}
|
||||
|
||||
B.encode = function(d, s, u){ s = s || "|"; u = u || String.fromCharCode(32);
|
||||
|
||||
@ -374,6 +374,25 @@ var names = ["Adalard","Adora","Aia","Albertina","Alfie","Allyn","Amabil","Ammam
|
||||
|
||||
});
|
||||
|
||||
describe('API usage checks', function(){
|
||||
var opt = {file: 'search'}
|
||||
var search = RAD(opt);
|
||||
var b = Book();
|
||||
it('read results from in-memory data', async done => {
|
||||
b('hello', '1data');
|
||||
var r = b.page('wat').read();
|
||||
expect(r).to.be.eql(['1data']);
|
||||
b('hello', '1dataZ');
|
||||
r = b.page('wat').read();
|
||||
expect(r).to.be.eql(['1dataZ']);
|
||||
b('new', '2data');
|
||||
r = b.page('wat').read();
|
||||
expect(r).to.be.eql(['1dataZ','2data']);
|
||||
done();
|
||||
});
|
||||
|
||||
});
|
||||
|
||||
console.log("Performance Tests: 2023 Nov 12, 60M put/sec, 120M get/sec, 1M get/sec with splits.");
|
||||
|
||||
});
|
||||
|
||||
Loading…
x
Reference in New Issue
Block a user