From 6164d4c75341d6434bc5c7e31e2e483aaa1352fa Mon Sep 17 00:00:00 2001 From: Mark Nadal Date: Tue, 17 Jul 2018 15:50:42 -0700 Subject: [PATCH] adjust defaults --- examples/http.js | 4 +-- lib/bye.js | 12 +++---- lib/erase.js | 12 +++---- lib/file.js | 21 +++++------ lib/forget.js | 14 ++++---- lib/radisk.js | 86 +++++++++++++++++++++++++++++++++++--------- lib/rs3.js | 9 +++-- lib/store.js | 23 ++++++------ test/panic/radisk.js | 4 +-- 9 files changed, 119 insertions(+), 66 deletions(-) diff --git a/examples/http.js b/examples/http.js index e0dc12eb..defc3122 100644 --- a/examples/http.js +++ b/examples/http.js @@ -1,4 +1,4 @@ -var port = process.env.OPENSHIFT_NODEJS_PORT || process.env.VCAP_APP_PORT || process.env.PORT || process.argv[2] || 8080; +var port = process.env.OPENSHIFT_NODEJS_PORT || process.env.VCAP_APP_PORT || process.env.PORT || process.argv[2] || 8080 || 8765; var Gun = require('../'); @@ -19,4 +19,4 @@ var gun = Gun({ server.listen(port); -console.log('Server started on port ' + port + ' with /gun'); +console.log('Server started on port ' + port + ' with /gun'); \ No newline at end of file diff --git a/lib/bye.js b/lib/bye.js index dc3ea64a..e3f34aaa 100644 --- a/lib/bye.js +++ b/lib/bye.js @@ -1,18 +1,18 @@ var Gun = (typeof window !== "undefined")? window.Gun : require('../gun'); -Gun.on('opt', function(ctx){ - this.to.next(ctx); - if(ctx.once){ return } +Gun.on('opt', function(root){ + this.to.next(root); + if(root.once){ return } console.log("WARNING: `lib/bye` is out of date!"); - ctx.on('in', function(msg){ + root.on('in', function(msg){ if(!msg.peer || !msg.BYE){ return this.to.next(msg) } var peer = msg.peer(); (peer.bye = peer.bye || []).push(msg.BYE); }) - ctx.on('bye', function(peer){ + root.on('bye', function(peer){ this.to.next(peer); if(!peer.bye){ return } - var gun = ctx.gun; + var gun = root.gun; Gun.obj.map(peer.bye, function(data){ Gun.obj.map(data, function(put, soul){ gun.get(soul).put(put); diff --git a/lib/erase.js b/lib/erase.js index 73bd271c..4b4cd1be 100644 --- a/lib/erase.js +++ b/lib/erase.js @@ -1,13 +1,13 @@ var Gun = (typeof window !== "undefined")? window.Gun : require('../gun'); -Gun.on('opt', function(ctx){ - this.to.next(ctx); - if(ctx.once){ return } - ctx.on('put', function(msg){ +Gun.on('opt', function(root){ + this.to.next(root); + if(root.once){ return } + root.on('put', function(msg){ Gun.graph.is(msg.put, null, function(val, key, node, soul){ if(null !== val){ return } // TODO: Refactor this to use `.off()`? - var tmp = ctx.graph[soul]; + var tmp = root.graph[soul]; if(tmp){ delete tmp[key]; } @@ -15,7 +15,7 @@ Gun.on('opt', function(ctx){ if(tmp){ delete tmp[key]; } - tmp = ctx.next; + tmp = root.next; if(tmp && (tmp = tmp[soul]) && (tmp = tmp.put)){ delete tmp[key]; tmp = tmp._ && tmp._['>']; diff --git a/lib/file.js b/lib/file.js index 18881334..74ec7f01 100644 --- a/lib/file.js +++ b/lib/file.js @@ -6,14 +6,15 @@ var Gun = require('../gun'), fs = require('fs'); -Gun.on('opt', function(ctx){ - this.to.next(ctx); - var opt = ctx.opt; - if(ctx.once){ return } +Gun.on('create', function(root){ + this.to.next(root); + var opt = root.opt; + //if(true !== opt.localStorage){ return } if(false === opt.localStorage){ return } + if(process.env.RAD_ENV){ return } if(process.env.AWS_S3_BUCKET){ return } opt.file = String(opt.file || 'data.json'); - var graph = ctx.graph, acks = {}, count = 0, to; + var graph = root.graph, acks = {}, count = 0, to; var disk = Gun.obj.ify((fs.existsSync || require('path').existsSync)(opt.file)? fs.readFileSync(opt.file).toString() : null) || {}; @@ -24,7 +25,7 @@ Gun.on('opt', function(ctx){ 'intended for local development testing only!' ); - ctx.on('put', function(at){ + root.on('put', function(at){ this.to.next(at); Gun.graph.is(at.put, null, map); if(!at['@']){ acks[at['#']] = true; } // only ack non-acks. @@ -36,7 +37,7 @@ Gun.on('opt', function(ctx){ to = setTimeout(flush, opt.wait || 1); }); - ctx.on('get', function(at){ + root.on('get', function(at){ this.to.next(at); var lex = at.get, soul, data, opt, u; //setTimeout(function(){ @@ -48,7 +49,7 @@ Gun.on('opt', function(ctx){ if(data && field){ data = Gun.state.to(data, field); } - ctx.on('in', {'@': at['#'], put: Gun.graph.node(data)}); + root.on('in', {'@': at['#'], put: Gun.graph.node(data)}); //},11); }); @@ -68,7 +69,7 @@ Gun.on('opt', function(ctx){ var tmp = count; count = 0; Gun.obj.map(ack, function(yes, id){ - ctx.on('in', { + root.on('in', { '@': id, err: err, ok: err? u : 1 @@ -84,7 +85,7 @@ Gun.on('opt', function(ctx){ if(!Gun.text.match(soul, rgx)){ return } if(has){ node = Gun.state.to(node, has) } (put = {})[soul] = node; - ctx.on('in', {put: put, '@': at['#']}); + root.on('in', {put: put, '@': at['#']}); }); } }); \ No newline at end of file diff --git a/lib/forget.js b/lib/forget.js index 75283ae9..7cedd03f 100644 --- a/lib/forget.js +++ b/lib/forget.js @@ -1,15 +1,15 @@ ;(function(){ var Gun = (typeof window !== "undefined")? window.Gun : require('../gun'); - Gun.on('opt', function(ctx){ - once(ctx); - this.to.next(ctx); + Gun.on('opt', function(root){ + once(root); + this.to.next(root); }); - function once(ctx){ - if(ctx.once){ return } - var forget = ctx.opt.forget = ctx.opt.forget || {}; - ctx.on('put', function(msg){ + function once(root){ + if(root.once){ return } + var forget = root.opt.forget = root.opt.forget || {}; + root.on('put', function(msg){ Gun.graph.is(msg.put, function(node, soul){ if(!Gun.obj.has(forget, soul)){ return } delete msg.put[soul]; diff --git a/lib/radisk.js b/lib/radisk.js index 30be77de..2cfe7506 100644 --- a/lib/radisk.js +++ b/lib/radisk.js @@ -22,7 +22,7 @@ return Gun.log("ERROR: Radisk needs `store.get` interface with `(file, cb)`!"); } if(!opt.store.list){ - return Gun.log("ERROR: Radisk needs a streaming `store.list` interface with `(cb)`!"); + Gun.log("WARNING: `store.list` interface might be needed!"); } /* @@ -94,7 +94,7 @@ s.find = function(tree, key){ if(key < s.start){ return } s.start = key; - opt.store.list(s.lex); + r.list(s.lex); return true; } s.lex = function(file){ @@ -109,6 +109,7 @@ s.start = s.end = s.file = u; r.parse(file, function(err, disk){ if(err){ return cb(err) } + disk = disk || Radix(); Radix.map(rad, function(val, key){ if(key < start){ return } if(end && end < key){ return s.start = key } @@ -132,7 +133,7 @@ Therefore it is unavoidable that a read will have to happen, the question is just how long you delay it. */ - r.write = function(file, rad, cb){ + r.write = function(file, rad, cb, force){ var f = function Fractal(){}; f.text = ''; f.count = 0; @@ -140,7 +141,7 @@ f.each = function(val, key, k, pre){ f.count++; var enc = Radisk.encode(pre.length) +'#'+ Radisk.encode(k) + (u === val? '' : '='+ Radisk.encode(val)) +'\n'; - if((opt.chunk < f.text.length + enc.length) && !(1 >= rad.count)){ + if((opt.chunk < f.text.length + enc.length) && !(1 >= rad.count) && !force){ f.text = ''; f.limit = Math.ceil(f.count/2); f.count = 0; @@ -150,7 +151,13 @@ } f.text += enc; } - f.write = function(){ opt.store.put(ename(file), f.text, cb) } + f.write = function(){ + var tmp = ename(file); + r.list.add(tmp, function(err, ok){ + if(err){ return cb(err) } + opt.store.put(tmp, f.text, cb); + }); + } f.slice = function(val, key){ if(key < f.file){ return } if(f.limit < (++f.count)){ @@ -210,18 +217,65 @@ if(last > tmp && 0 > last.indexOf(tmp)){ return } r.read(tmp, as.ack, as.file); } - opt.store.list(g.lex); + r.list(g.lex); } - var lq; - r.list = function(cb){ - if(lq){ return lq.push(cb) } - lq = [cb]; - opt.store.list(); - } - r.list.it = function(file){ - - } + ;(function(){ + var dir, dq, f = String.fromCharCode(28), ef = ename(f); + r.list = function(cb){ + if(dir){ + Radix.map(dir, function(val, key){ + return cb(key); + }) || cb(); + return; + } + if(dq){ return dq.push(cb) } + dq = [cb]; + r.parse(f, r.list.init); + } + r.list.add = function(file, cb){ + var has = dir(file); + if(has || file === ef){ + return cb(u, 1); + } + dir(file, true); + r.write(f, dir, function(err, ok){ + if(err){ return cb(err) } + cb(u, 1); + }, true); + } + r.list.init = function(err, disk){ + if(err){ + Gun.log('list', err); + setTimeout(function(){ r.parse(f, r.list.init) }, 1000); + return; + } + if(disk){ + r.list.drain(disk); + return; + } + if(!opt.store.list){ + r.list.drain(Radix()); + return; + } + // import directory. + opt.store.list(function(file){ + dir = dir || Radix(); + if(!file){ return r.list.drain(dir) } + r.list.add(file, noop); + }); + } + r.list.drain = function(rad){ + r.list.dir = dir = rad; + var tmp = dq; + dq = null; + Gun.list.map(tmp, function(cb){ + Radix.map(dir, function(val, key){ + return cb(key); + }) || cb(); + }); + } + }()); /* Let us start by assuming we are the only process that is changing the directory or bucket. Not because we do not want @@ -234,7 +288,7 @@ p.disk = Radix(); p.read = function(err, data){ var tmp; if(err){ return cb(err) } - if(!data){ return cb(u, p.disk) } + if(!data){ return cb(u, u) } var tmp = p.split(data), pre = [], i, k, v; while(tmp){ k = v = u; diff --git a/lib/rs3.js b/lib/rs3.js index aa3ed870..ac63343f 100644 --- a/lib/rs3.js +++ b/lib/rs3.js @@ -4,12 +4,11 @@ var fs = require('fs'); var Radix = Radisk.Radix; var u, AWS; -Gun.on('opt', function(ctx){ - this.to.next(ctx); - var opt = ctx.opt; - if(ctx.once){ return } +Gun.on('create', function(root){ + this.to.next(root); + var opt = root.opt; if(!process.env.AWS_S3_BUCKET){ return } - opt.batch = opt.batch || (1000 * 10); + opt.batch = opt.batch || (1000 * 1); opt.until = opt.until || (1000 * 15); opt.chunk = opt.chunk || (1024 * 1024 * 10); // 10MB diff --git a/lib/store.js b/lib/store.js index 8485b0be..18ef10ce 100644 --- a/lib/store.js +++ b/lib/store.js @@ -1,22 +1,21 @@ var Gun = (typeof window !== "undefined")? window.Gun : require('../gun'); -Gun.on('opt', function(ctx){ - this.to.next(ctx); - var opt = ctx.opt, u; +Gun.on('create', function(root){ + this.to.next(root); + var opt = root.opt, u; if(typeof window !== "undefined"){ opt.window = window; } - if(ctx.once){ return } - if(false !== opt.localStorage && !(!opt.window && process.env.AWS_S3_BUCKET)){ return } // TODO: Remove this after migration. - if(false === opt.radisk){ return } - console.log("BUG WARNING: Radix Storage Engine (RAD) has a known rare edge case, if data gets split between file chunks, a GET may only return the first chunk!!!"); + if(true !== opt.radisk && (!opt.window && !process.env.RAD_ENV && !process.env.AWS_S3_BUCKET)){ return } + //if(true !== opt.radisk){ return } + //if(false === opt.radisk){ return } var Radisk = (opt.window && opt.window.Radisk) || require('./radisk'); var Radix = Radisk.Radix; opt.store = opt.store || (!opt.window && require('./rfs')(opt)); var rad = Radisk(opt); - ctx.on('put', function(msg){ + root.on('put', function(msg){ this.to.next(msg); var id = msg['#'], track = !msg['@'], acks = track? 0 : u; // only ack non-acks. if(msg.rad && !track){ return } // don't save our own acks @@ -29,15 +28,15 @@ Gun.on('opt', function(ctx){ acks--; if(ack.err){ return } if(ack.err = err){ - ctx.on('in', {'@': id, err: err}); + root.on('in', {'@': id, err: err}); return; } if(acks){ return } - ctx.on('in', {'@': id, ok: 1}); + root.on('in', {'@': id, ok: 1}); } }); - ctx.on('get', function(msg){ + root.on('get', function(msg){ this.to.next(msg); var id = msg['#'], soul = msg.get['#'], key = msg.get['.']||'', tmp = soul+'.'+key, node; rad(tmp, function(err, val){ @@ -45,7 +44,7 @@ Gun.on('opt', function(ctx){ Radix.map(val, each); if(!node){ each(val, key) } } - ctx.on('in', {'@': id, '#': key, put: Gun.graph.node(node), err: err? err : u, rad: Radix}); + root.on('in', {'@': id, '#': key, put: Gun.graph.node(node), err: err? err : u, rad: Radix}); }); function each(val, key){ tmp = val.lastIndexOf('>'); diff --git a/test/panic/radisk.js b/test/panic/radisk.js index 0799dd28..8825798e 100644 --- a/test/panic/radisk.js +++ b/test/panic/radisk.js @@ -7,7 +7,7 @@ var config = { burst: 10, wait: 1, dir: __dirname, - chunk: 1024 * 100, + chunk: 1024 * 1024 * 10, notrad: false, route: { '/': __dirname + '/index.html', @@ -41,7 +41,7 @@ var browsers = clients.excluding(servers); var alice = browsers.pluck(1); var bob = browsers.excluding(alice).pluck(1); -describe("Make sure the Radix Storage Engine (RSE) works.", function(){ +describe("Make sure the Radix Storage Engine (RAD) works.", function(){ //this.timeout(5 * 60 * 1000); this.timeout(100 * 60 * 1000);