diff --git a/gun.js b/gun.js index fc9f3f46..b7a66d2d 100644 --- a/gun.js +++ b/gun.js @@ -74,7 +74,7 @@ Type.list.index = 1; // change this to 0 if you want non-logical, non-mathematical, non-matrix, non-convenient array notation Type.obj = {is: function(o){ return o? (o instanceof Object && o.constructor === Object) || Object.prototype.toString.call(o).match(/^\[object (\w+)\]$/)[1] === 'Object' : false }} Type.obj.put = function(o, f, v){ return (o||{})[f] = v, o } - Type.obj.has = function(o, f){ return o && Object.prototype.hasOwnProperty.call(o, f) } + Type.obj.has = function(o, f){ return o && Object.hasOwnProperty(o, f) } Type.obj.del = function(o, k){ if(!o){ return } o[k] = null; @@ -126,7 +126,7 @@ var u, i = 0, x, r, ll, lle, f = fn_is(c); t.r = null; if(keys && obj_is(l)){ - ll = Object.keys(l); lle = true; + ll = keys(l); lle = true; } if(list_is(l) || ll){ x = (ll || l).length; @@ -995,6 +995,9 @@ if(!obj_is(at.opt.peers)){ at.opt.peers = {}} at.opt.peers = obj_to(tmp, at.opt.peers); } + at.opt.uuid = at.opt.uuid || function(){ + return state().toString(36).replace('.','') + text_rand(12); + } at.opt.wsc = at.opt.wsc || {protocols:[]} at.opt.peers = at.opt.peers || {}; obj_to(opt, at.opt); // copies options on to `at.opt` only if not already taken. @@ -1003,10 +1006,10 @@ } }()); - var text_is = Gun.text.is; var list_is = Gun.list.is; + var text = Gun.text, text_is = text.is, text_rand = text.random; var obj = Gun.obj, obj_is = obj.is, obj_has = obj.has, obj_to = obj.to, obj_map = obj.map, obj_copy = obj.copy; - var _soul = Gun._.soul, _field = Gun._.field, rel_is = Gun.val.rel.is; + var state = Gun.state, _soul = Gun._.soul, _field = Gun._.field, rel_is = Gun.val.rel.is; var empty = {}, u; console.debug = function(i, s){ return (console.debug.i && i === console.debug.i && console.debug.i++) && (console.log.apply(console, arguments) || s) }; @@ -1066,7 +1069,7 @@ // language, consider implementing an easier API to build. var Gun = require('./root'); Gun.chain.chain = function(){ - var at = this._, chain = new this.constructor(this), cat = chain._; + var at = this._, chain = new this.constructor(this), cat = chain._, root; cat.root = root = at.root; cat.id = ++root._.once; cat.back = this; @@ -2003,12 +2006,11 @@ Gun.on('opt', function(at){ var opt = at.opt, peers = opt.peers; - var prefix = opt.file || opt.prefix || 'gun/'; - var wait = opt.wait || 1, batch = opt.batch || 10000; + opt.file = opt.file || opt.prefix || 'gun/'; // support old option name. this.to.next(at); if(at.once){ return } if(false === at.opt.file){ return } - var graph = at.graph, acks = {}, async = {}, count = 0, to; + var graph = at.graph, acks = {}, batch = {}, count = 0, to; at.on('put', function(at){ this.to.next(at); @@ -2016,12 +2018,11 @@ if(!at['@']){ acks[at['#']] = 1; } // only ack non-acks. count += 1; console.log('lS PUT', Gun.obj.copy(async)); - if(count >= batch){ + if(count >= opt.batch){ return flush(); } if(to){ return } - clearTimeout(to); - to = setTimeout(flush, wait); + to = setTimeout(flush, opt.wait); }); function map(node, soul){ @@ -2029,31 +2030,31 @@ } function flush(){ - var err; - count = 0; - clearTimeout(to); - to = false; - var ack = acks; - acks = {}; - var all = async; - async = {}; - Gun.obj.map(all, function(node, soul){ - // Since localStorage only has 5MB, it is better that we keep only - // the data that the user is currently interested in. - node = graph[soul] || all[soul] || node; - console.log("WRITE:", Gun.obj.copy(node)); - try{store.setItem(prefix + soul, JSON.stringify(node)); - }catch(e){ err = e || "localStorage failure" } + var err; + count = 0; + clearTimeout(to); + to = false; + var ack = acks; + acks = {}; + var all = async; + async = {}; + Gun.obj.map(all, function(node, soul){ + // Since localStorage only has 5MB, it is better that we keep only + // the data that the user is currently interested in. + node = graph[soul] || all[soul] || node; + console.log("WRITE:", Gun.obj.copy(node)); + try{store.setItem(opt.file + soul, JSON.stringify(node)); + }catch(e){ err = e || "localStorage failure" } + }); + if(!err && !Gun.obj.empty(peers)){ return } // only ack if there are no peers. + Gun.obj.map(ack, function(yes, id){ + at.on('in', { + '@': id, + err: err, + ok: 0 // localStorage isn't reliable, so make its `ok` code be a low number. }); - if(!err && !Gun.obj.empty(peers)){ return } // only ack if there are no peers. - Gun.obj.map(ack, function(yes, id){ - at.on('in', { - '@': id, - err: err, - ok: 0 // localStorage isn't reliable, so make its `ok` code be a low number. - }); - }); - } + }); + } at.on('get', function(at){ this.to.next(at); @@ -2062,7 +2063,7 @@ if(!lex || !(soul = lex[Gun._.soul])){ return } //if(0 >= at.cap){ return } var field = lex['.']; - data = async[soul] || Gun.obj.ify(store.getItem(prefix + soul) || null) || async[soul] || u; + data = async[soul] || Gun.obj.ify(store.getItem(opt.file + soul) || null) || async[soul] || u; console.debug(2, 'lS GET', data, async[soul]); if(data && field){ data = Gun.state.to(data, field); diff --git a/lib/radisk.js b/lib/radisk.js new file mode 100644 index 00000000..c2cfe6ca --- /dev/null +++ b/lib/radisk.js @@ -0,0 +1,196 @@ +var fs = require('fs'); +var Gun = require('../gun'); +var Radix = require('./radix'); + +function Radisk(opt){ + /* + Any and all storage adapters should... + 1. If not busy, write to disk immediately. + 2. If busy, batch to disk. (Improves performance, reduces potential disk corruption) + 3. If a batch exceeds a certain number of writes, force to disk. (This caps total performance, but reduces potential loss) + */ + var radisk = function(key, val, cb){ + if(val instanceof Function){ + cb = val; + val = radisk.batch(key); + if(u !== val){ + return cb(null, val); + } + if(radisk.was){ + val = radisk.was(key); + if(u !== val){ + return cb(null, val); + } + } + console.log("READ FROM DISK"); + return cb(null, val); + } + radisk.batch(key, val); + if(cb){ radisk.batch.acks.push(cb) } + if(!count++){ return thrash() } // (1) + if(opt.batch <= count){ return thrash() } // (3) + clearTimeout(to); // (2) + to = setTimeout(thrash, opt.wait); + }; + radisk.batch = Radix(); + radisk.batch.acks = []; + var count = 0, wait, to, u; + + opt = opt || {}; + opt.file = opt.file || 'radata'; + opt.size = opt.size || (1024 * 1024 * 10); // 10MB + opt.batch = opt.batch || 10 * 1000; + opt.wait = opt.wait || 1; + opt.nest = opt.nest || ' '; + + console.log("Warning: Radix storage engine has not been tested with all types of values and keys yet."); + if(!fs.existsSync(opt.file)){ fs.mkdirSync(opt.file) } + + var thrash = function(){ + if(wait){ return } + clearTimeout(to); + wait = true; + var was = radisk.was = radisk.batch; + radisk.batch = null; + radisk.batch = Radix(); + radisk.batch.acks = []; + chunk(radisk.was, function(err, ok){ + radisk.was = null; + wait = false; + var tmp = count; + count = 0; + Gun.obj.map(was.acks, function(cb){cb(err, ok)}); + if(1 < tmp){ thrash() } + }); + } + + /* + 1. Find the first radix item in memory. + 2. Use that as the starting index in the directory of files. + 3. Find the first file that is lexically larger than it, + 4. Read the previous file to that into memory + 5. Scan through the in memory radix for all values lexically less than the limit. + 6. Merge and write all of those to the in-memory file and back to disk. + 7. If file to large, split. More details needed here. + */ + function chunk(radix, cb){ + var step = { + check: function(tree, key){ + if(key < step.start){ return } + step.start = key; + fs.readdir(opt.file, step.match); + return true; + }, + match: function(err, dir){ + step.dir = dir; + if(!dir.length){ + step.file = '0'; + return step.merge(null, Radix()); + } + Gun.obj.map(dir, step.lex); + read(step.file, step.merge); + }, + lex: function(file){ + if(file > step.start){ + return step.end = file; + } + step.file = file; + }, + merge: function(err, disk){ + if(err){ return console.log("ERROR!!!", err) } + step.disk = disk; + Radix.map(radix, step.add); + write(step.file, step.disk, step.done); + }, + add: function(val, key){ + if(key < step.start){ return } + if(step.end && step.end < key){ return step.next = key; } + step.disk(key, val); + }, + done: function(err){ + if(err){ console.log("ERROR!!!", err) } + if(!step.next){ + return cb(err); + } + step.start = step.next; + step.end = step.next = step.file = u; + Radix.map(radix, step.check); + } + } + Radix.map(radix, step.check); + } + + var write = function(file, radix, cb){ + var step = { + rest: "", + count: 0, + file: file, + each: function(val, key, k, pre){ + step.count++; + if(opt.size < step.rest.length){ + step.rest = ""; + step.limit = Math.ceil(step.count/2); + step.count = 0; + step.sub = Radix(); + Radix.map(radix, step.slice); + return true; + } + var i = pre.length; + while(i--){ step.rest += opt.nest }; + step.rest += k + (u === val? '' : '=' + val) + '\n'; + }, + dump: function(){ + var rest = step.rest; + step.rest = ""; + fs.writeFile(opt.file +'/'+ file, rest, cb); + }, + slice: function(val, key){ + if(key < step.file){ return } + if(step.limit < (++step.count)){ + var name = step.file; + step.file = key; + step.count = 0; + write(name, step.sub, step.next); + return true; + } + step.sub(key, val); + }, + next: function(err){ + if(err){ console.log("ERR!!!!") } + step.sub = Radix(); + if(!Radix.map(radix, step.slice)){ + write(step.file, step.sub, cb); + } + } + }; + if(!Radix.map(radix, step.each, true)){ step.dump() } + } + + var read = function(file, cb){ + var step = { + nest: 0, + rad: Radix(), + data: function(err, data){ + if(err){ return console.log("ERROR READING FILE!", err) } + step.pre = []; + Gun.obj.map(data.toString().split('\n'), step.split); // TODO: Escape! + cb(null, step.rad); + }, + split: function(line){ var LINE = line; + var nest = -1; while(opt.nest === line[++nest]){}; + if(nest){ line = line.slice(nest) } + if(nest <= step.nest){ step.pre = step.pre.slice(0, nest - step.nest - 1) } + line = line.split('='); step.pre.push(line[0]); + if(1 < line.length){ step.rad(step.pre.join(''), line[1]) } + step.nest = nest; + } + } + fs.readFile(opt.file +'/'+ file, step.data); + } + + radisk.read = read; + + return radisk; +} + +module.exports = Radisk; \ No newline at end of file diff --git a/lib/radix.js b/lib/radix.js index b875e7c9..1dcd10e4 100644 --- a/lib/radix.js +++ b/lib/radix.js @@ -1,10 +1,9 @@ -var fs = require('fs'); -var Gun = require('gun'); +var Gun = require('../gun'); var gbm = Gun.obj.map, no = {}, u; function Radix(){ var radix = function(key, val, t){ - t = t || tree; + t = t || radix._ || (radix._ = {}); var i = 0, l = key.length-1, k = key[i], at, tmp; while(!(at = t[k]) && i < l){ k += key[++i]; @@ -36,75 +35,31 @@ function Radix(){ if(u === val){ return (u === (tmp = at.$))? at._ : tmp } at.$ = val; } else { - return radix(key.slice(++i), val, at._); + return radix(key.slice(++i), val, at._ || (at._ = {})); } } - var tree = radix._ = {}; return radix; -} - - - -(function(){ - var radix = Radix(); - radix('user/marknadal', 'asdf'); - radix('user/ambercazzell', 'dafs'); - radix('user/taitforrest', 'sadf'); - radix('user/taitveronika', 'fdsa'); - radix('user/marknadal', 'foo'); - radix('user/table', 'foo'); - console.log("__________________"); - console.log(radix._); - console.log(radix('user/table')); - //fs.writeFileSync('radix-data.json', JSON.stringify(radix._, null, 2)); -}()); -(function(){return; - var radix = Radix(); - var gtr = Gun.text.random; - var c = 0, l = 10; - function bench(){ - if(c > 1000){ return clearInterval(it), console.log(radix._, Object.keys(radix._).length), fs.writeFileSync('radix-data.json', JSON.stringify(radix._, null, 2)); } - for(var i = 0; i < l; i++){ - radix(gtr(7), c++); - } - //if(c % 1000 === 0){ console.log(radix._) } - } - var it = setInterval(bench, 1); -}()); - -function Radisk(){ - var radix = Radix(); - var radisk = function(key, val, cb){ - if(val instanceof Function){ - cb = val; - val = radix(key); - if(u !== val){ - return cb(null, val); +}; +;(function(){ + Radix.map = function map(radix, cb, opt, pre){ pre = pre || []; + var _ = radix._ || radix, keys = Object.keys(_).sort(), i = 0, l = keys.length; + for(;i < l; i++){ var key = keys[i], tree = _[key], tmp; + if(u !== (tmp = tree.$)){ + tmp = cb(tmp, pre.join('') + key, key, pre); + if(u !== tmp){ return tmp } + } else + if(opt){ + cb(u, pre.join(''), key, pre); + } + if(tmp = tree._){ + pre.push(key); + tmp = map(tmp, cb, opt, pre); + if(u !== tmp){ return tmp } + pre.pop(); } - return console.log("read from disk"); } - radix(key, val); - if(!to){ return flush(to = true) } - clearTimeout(to); - to = setTimeout(flush, opt.wait) }; - var opt = {file: 'radix', size: 1024 * 1024 * 2, wait: 10 * 1000, batch: 10 * 1000}, to; + Object.keys = Object.keys || function(o){ return gbm(o, function(v,k,t){t(k)}) } +}()); - var flush = function(){ - - } - var read = function(path){ - return fs.readFileSync(nodePath.join(dir, path)).toString(); - } - - var write = function(path, data){ - return fs.writeFileSync(nodePath.join(dir, path), data); - } - - var mk = function(path){ - path = nodePath.join(dir, path); - if(fs.existsSync(path)){ return } - fs.mkdirSync(path); - } - return radisk; -} \ No newline at end of file +module.exports = Radix; \ No newline at end of file diff --git a/lib/radix_.js b/lib/radix_.js deleted file mode 100644 index c75dcba0..00000000 --- a/lib/radix_.js +++ /dev/null @@ -1,104 +0,0 @@ -function radix(r){ - r = r || {}; - var u, n = null, c = 0; - function get(p){ - var v = match(p, r); - return v; - } - function match(path, tree, v){ - if(!Gun.obj.map(tree, function(val, key){ - if(key[0] !== path[0]){ return } - var i = 1; - while(key[i] === path[i] && path[i]){ i++ } - if(key = key.slice(i)){ // recurse - console.log("match", key, i) - v = {sub: tree, pre: path.slice(0, i), val: val, post: key, path: path.slice(i) }; - } else { // replace - console.log("matching", path, key, i); - v = match(path.slice(i), val); - } - return true; - })){ console.log("matched", tree, path); v = {sub: tree, path: path} } // insert - return v; - } - function rebalance(ctx, val){ - console.log("rebalance", ctx, val); - if(!ctx.post){ return ctx.sub[ctx.path] = val } - ctx.sub[ctx.pre] = ctx.sub[ctx.pre] || (ctx.post? {} : ctx.val || {}); - ctx.sub[ctx.pre][ctx.path] = val; - if(ctx.post){ - ctx.sub[ctx.pre][ctx.post] = ctx.val; - delete ctx.sub[ctx.pre + ctx.post]; - } - } - function set(p, val){ - rebalance(match(p, r), val); - console.log('-------------------------'); - return r; - } - return function(p, val){ - return (1 < arguments.length)? set(p, val) : get(p || ''); - } -} // IT WORKS!!!!!! - -var rad = radix({}); - -rad('user/marknadal', {'#': 'asdf'}); -rad('user/ambercazzell', {'#': 'dafs'}); -rad('user/taitforrest', {'#': 'sadf'}); -rad('user/taitveronika', {'#': 'fdsa'}); -rad('user/marknadal', {'#': 'foo'}); - -/* - -function radix(r){ - var u, n = null, c = 0; - r = r || {}; - function get(){ - - } - function match(p, l, cb){ - cb = cb || function(){}; - console.log("LETS DO THIS", p, l); - if(!Gun.obj.map(l, function(v, k){ - if(k[0] === p[0]){ - var i = 1; - while(k[i] === p[i] && p[i]){ i++ } - k = k.slice(i); - if(k){ - cb(p.slice(0, i), v, k, l, p.slice(i)); - } else { - match(p.slice(i), v, cb); - } - return 1; - } - })){ cb(p, l, null, l) } - } - function set(p, val){ - match(p, r, function(pre, data, f, rr, pp){ - if(f === null){ - rr[pre] = val; - return; - } - console.log("Match?", c, pre, data, f); - rr[pre] = r[pre] || (f? {} : data || {}); - rr[pre][pp] = val; - if(f){ - rr[pre][f] = data; - delete rr[pre + f]; - } - }); - return r; - } - return function(p, val){ - return (1 < arguments.length)? set(p, val) : get(p || ''); - } -} // IT WORKS!!!!!! - -var rad = radix({}); - -rad('user/marknadal', {'#': 'asdf'}); -//rad('user/ambercazzell', {'#': 'dafs'}); -//rad('user/taitforrest', {'#': 'sadf'}); -//rad('user/taitveronika', {'#': 'fdsa'}); -*/ \ No newline at end of file diff --git a/test/ptsd/radisk.js b/test/ptsd/radisk.js new file mode 100644 index 00000000..a72f12e3 --- /dev/null +++ b/test/ptsd/radisk.js @@ -0,0 +1,71 @@ +var Radix = require('../../lib/radix'); +var Radisk = require('../../lib/radisk'); +var Gun = require('../../gun'); +var fs = require('fs'); + +var TOTAL = 1000000; +var c = 0; +var acked = 0; +var start; +var diff; + +(function(){//return; + var radix = Radisk(); + var gtr = Gun()._.opt.uuid; + var l = 500000; + var last = start; + var t = Gun.time.is; + var at = c; + var toc, alldone = function(){ + acked++; + if(acked < TOTAL){ return } + diff = (Gun.time.is() - start) / 1000; + clearTimeout(toc); + toc = setTimeout(CHECK,1000); + } + function bench(){ + if(c >= (TOTAL)){ return clearInterval(it); } + for(var i = 0; i < l; i++){ + radix(gtr(), ++c, alldone); + if(c % 50000 === 0){ + var now = t(); + console.log(c);//, (now - last)/1000); + at = c; + last = now; + } + } + } + start = Gun.time.is(); + var it = setInterval(bench, 1); +}()); + +function CHECK(){ + console.log(Math.floor(c / diff), 'disk writes per second'); + var disk = Radisk(); + var all = {}; + var to; + var i = TOTAL; + /*while(--i){ + all[i] = true; + }*/ + var dir = fs.readdirSync('radata'); + dir.forEach(function(file){ + disk.read(file, function(err, rad){ + Radix.map(rad, function(val, key){ + all[key] = false; + clearTimeout(to); + to = setTimeout(function(){ + var len = Object.keys(all).length; + console.log("how many?", len); + if(len < TOTAL){ return } + var missing = []; + var fail = Gun.obj.map(all, function(val, key){ + if(val){ missing.push(key); return true } + }); + //console.log(all); + console.log("DONE!", 'Verify ALL writes:', fail? '!!!FAIL!!!!' : 'YES');// '. Missing:', missing); + },1000); + }) + }) + }) +} \ No newline at end of file