diff --git a/CHANGELOG.md b/CHANGELOG.md index 06f5e0bd..591c9816 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,11 @@ # CHANGELOG +## 0.8.x + +Adapter interfaces have changed from `Gun.on('event', cb)` to `gun.on('event', cb)`, this will force adapters to be instance specific. + +`.path()` and `.not()` have been officially removed from the core bundle, you can bundle them yourself at `lib/path.js` and `lib/not.js` if you still need them. + ## 0.7.x Small breaking change to `.val(cb)`: diff --git a/examples/engage/index.html b/examples/engage/index.html new file mode 100644 index 00000000..fabfeb9e --- /dev/null +++ b/examples/engage/index.html @@ -0,0 +1,78 @@ +
+
+ + + +
+
+ + + \ No newline at end of file diff --git a/gun.js b/gun.js index b865d46b..c0fc6247 100644 --- a/gun.js +++ b/gun.js @@ -184,6 +184,9 @@ this.to.back = this.back; this.next = onto._.next; this.back.to = this.to; + if(this.the.last === this.the){ + delete this.on.tag[this.the.tag]; + } }), to: onto._, next: arg, @@ -588,7 +591,7 @@ var Type = require('./type'); function Dup(opt){ var dup = {s:{}}; - opt = opt || {max: 1000, age: 1000 * 60 * 2}; + opt = opt || {max: 1000, age: 1000 * 9};//1000 * 60 * 2}; dup.check = function(id){ return dup.s[id]? dup.track(id) : false; } @@ -761,14 +764,21 @@ Gun.on.ask = function(cb, as){ if(!this.on){ return } var id = text_rand(9); - if(cb){ this.on(id, cb, as) } + if(cb){ + var to = this.on(id, cb, as); + to.err = setTimeout(function(){ + to.next({err: "Error: No ACK received yet."}); + to.off(); + }, 1000 * 9); // TODO: Make configurable!!! + } return id; } Gun.on.ack = function(at, reply){ if(!at || !reply || !this.on){ return } - var id = at['#'] || at; - if(!this.tag || !this.tag[id]){ return } + var id = at['#'] || at, tmp = (this.tag||empty)[id]; + if(!tmp){ return } this.on(id, reply); + clearTimeout(tmp.err); return true; } }()); @@ -1292,13 +1302,13 @@ function batch(){ var as = this; if(!as.graph || obj_map(as.stun, no)){ return } (as.res||iife)(function(){ + var cat = (as.gun.back(-1)._), ask = cat.ask(function(ack){ + this.off(); // One response is good enough for us currently. Later we may want to adjust this. + if(!as.ack){ return } + as.ack(ack, this); + }, as.opt); (as.ref._).on('out', { - cap: 3, - gun: as.ref, put: as.out = as.env.graph, opt: as.opt, - '#': as.gun.back(-1)._.ask(function(ack){ this.off(); // One response is good enough for us currently. Later we may want to adjust this. - if(!as.ack){ return } - as.ack(ack, this); - }, as.opt) + gun: as.ref, put: as.out = as.env.graph, opt: as.opt, '#': ask }); }, as); if(as.res){ as.res() } @@ -1519,6 +1529,22 @@ } return gun; } + Gun.chain.off = function(){ + var gun = this, at = gun._, tmp; + var back = at.back || {}, cat = back._; + if(!cat){ return } + if(tmp = cat.next){ + if(tmp[at.get]){ + obj_del(tmp, at.get); + } else { + + } + } + if(tmp = at.soul){ + obj_del(cat.root._.graph, tmp); + } + return gun; + } var obj = Gun.obj, obj_has = obj.has, obj_del = obj.del, obj_to = obj.to; var rel = Gun.val.rel; var empty = {}, noop = function(){}, u; @@ -1591,6 +1617,11 @@ if(typeof window !== 'undefined'){ root = window } var store = root.localStorage || {setItem: noop, removeItem: noop, getItem: noop}; + /* + NOTE: Both `lib/file.js` and `lib/memdisk.js` are based on this design! + If you update anything here, consider updating the other adapters as well. + */ + Gun.on('opt', function(ctx){ this.to.next(ctx); var opt = ctx.opt; diff --git a/lib/memdisk.js b/lib/memdisk.js new file mode 100644 index 00000000..795e3200 --- /dev/null +++ b/lib/memdisk.js @@ -0,0 +1,68 @@ +// Take caution running this in production, it ONLY saves to disk what is in memory. + +var Gun = require('../gun'), +fs = require('fs'); + +Gun.on('opt', function(ctx){ + this.to.next(ctx); + var opt = ctx.opt; + if(ctx.once){ return } + opt.file = String(opt.file || 'data.json'); + var graph = ctx.graph, acks = {}, count = 0, to; + var disk = Gun.obj.ify((fs.existsSync || require('path').existsSync)(opt.file)? + fs.readFileSync(opt.file).toString() + : null) || {}; + + Gun.obj.map(disk, function(node, soul){ + graph[soul] = node; // TODO: Check if soul is already on graph? + }); + + ctx.on('put', function(at){ + this.to.next(at); + if(!at['@']){ acks[at['#']] = true; } // only ack non-acks. + count += 1; + if(count >= (opt.batch || 10000)){ + return flush(); + } + if(to){ return } + to = setTimeout(flush, opt.wait || 1); + }); + + ctx.on('get', function(at){ + this.to.next(at); + var gun = at.gun, lex = at.get, soul, data, opt, u; + //setTimeout(function(){ + if(!lex || !(soul = lex[Gun._.soul])){ return } + //if(0 >= at.cap){ return } + var field = lex['.']; + data = graph[soul] || u; + if(data && field){ + data = Gun.state.to(data, field); + } + gun.on('in', {'@': at['#'], put: Gun.graph.node(data)}); + //},11); + }); + + var wait; + var flush = function(){ + if(wait){ return } + wait = true; + clearTimeout(to); + to = false; + var ack = acks; + acks = {}; + fs.writeFile(opt.file, JSON.stringify(graph), function(err, ok){ + wait = false; + var tmp = count; + count = 0; + Gun.obj.map(ack, function(yes, id){ + ctx.on('in', { + '@': id, + err: err, + ok: 0 // memdisk should not be relied upon as permanent storage. + }); + }); + if(1 < tmp){ flush() } + }); + } +}); \ No newline at end of file diff --git a/lib/s3.js b/lib/s3.js index 6b2b7c22..4a0ccc1a 100644 --- a/lib/s3.js +++ b/lib/s3.js @@ -1,29 +1,27 @@ ;(function(){ - if(!process.env.AWS_S3_BUCKET){ return } - var Gun = require('../gun'); var S3 = require('./aws'); - // TODO: BUG! Mark, upgrade S3 in v0.8.1! And try to integrate with Radix Storage Engine! + // TODO: BUG! Mark, upgrade S3 in v0.8.X! And try to integrate with Radix Storage Engine!!! - Gun.on('opt', function(at){ - var opt = at.opt.s3 || (at.opt.s3 = {}); - var s3 = opt.store || S3(opt); - opt.store = s3; - this.to.next(at); - if(!s3 || !s3.config){ return } - if(at.once){ return } - var root = at.gun.back(-1); - opt.prefix = opt.prefix || ''; + Gun.on('opt', function(ctx){ + this.to.next(ctx); + var opt = ctx.opt; + if(ctx.once){ return } + if(!process.env.AWS_S3_BUCKET){ return } + console.log("S3 STORAGE ENGINE IS BROKEN IN 0.8! DO NOT USE UNTIL FIXED!"); + var s3 = opt.store || S3(opt.s3 = opt.s3 || {}); + opt.s3.store = s3; + if(!s3 || !s3.config){ return Gun.log("No S3 config!") } + opt.file = opt.file || opt.prefix || ''; opt.batch = opt.batch || 10; opt.throttle = opt.throttle || process.env.AWS_S3_THROTTLE || 15; opt.disconnect = opt.disconnect || 5; - Gun.on('get', function(at){ - if(!at.get){ return } + ctx.on('get', function(at){ + this.to.next(at); var id = at['#'], soul = at.get['#'], field = at.get['.']; var key = opt.prefix+soul; - //console.log("g3t", soul); s3.GET(key, function(err, data, text, meta){ meta = meta || {}; if(err && err.statusCode == 404){ @@ -44,32 +42,17 @@ if(data && field){ node = Gun.state.ify({}, field, Gun.state.is(node, field), node[field], soul); } - //console.log("got", soul, node); + console.log("got", soul, node); root.on('in', {'@': id, put: Gun.graph.node(node)}); }); }); - Gun.on('put', function(at){ + ctx.on('put', function(at){ + this.to.next(at); var id = at['#'], check = {}, next = s3.next, err, u; - Gun.graph.is(at.put, function(node, soul){ - check[soul] = 1; - /*if(!graph[soul]){ - // need to read before writing - return; - }*/ - async[soul] = node; - batch[soul] = (batch[soul] || 0) + 1; - s3.on(next + ':' + soul, function(arg){ - var reply = arg[1]; - err = arg[0]; - check[soul] = 0; - this.off(); - Gun.obj.del(async, soul); - if(Gun.obj.map(check, function(v){ - if(v){ return true } - })){ return } - root.on('in', {'@': id, err: err, ok: err? u : reply}); - }); + Gun.graph.is(at.put, null, function(val, key, node, soul){ + batch[soul] = Gun.state.to(node, key, disk[soul]); }); + if(!at['@']){ acks[at['#']] = true; } // only ack non-acks. s3.batching += 1; if(opt.batch < s3.batching){ return now(); @@ -88,7 +71,7 @@ s3.next = Gun.time.is(); s3.wait = null; Gun.obj.map(keep, function put(exists, soul, count){ - //console.log("s3ving...", soul); + console.log("s3ving...", soul); var node = root._.graph[soul] || async[soul]; // the batch does not actually have the nodes, but what happens when we do cold data? Could this be gone? s3.PUT(opt.prefix+soul, node, function(err, reply){ if(count < 5 && (err || !reply)){ @@ -100,7 +83,8 @@ }); }); } - var graph = {}, batch = {}, ids = {}, async = {}; + var graph = {}, batch = {}, acks = {}, ids = {}, async = {}; + var count = 0; s3.next = s3.next || Gun.time.is(); s3.on = s3.on || Gun.on; }); @@ -113,7 +97,6 @@ process.env.AWS_SECRET_ACCESS_KEY = 'fdsa'; process.env.fakes3 = 'http://localhost:4567'; process.env.AWS_S3_THROTTLE = 0; - require('../test/abc'); }()); -}()); \ No newline at end of file +}()); diff --git a/test/common.js b/test/common.js index f3b256f7..022d5058 100644 --- a/test/common.js +++ b/test/common.js @@ -3545,6 +3545,25 @@ describe('Gun', function(){ done(); }); }); + return; + it.only('Memory management', function(done){ + this.timeout(9999999); + var gun = Gun(), c = 100000, big = "big"; + while(--c){big += "big"} + c = 0; + setInterval(function(){ + var key = Gun.text.random(5); + gun.get(key).put({data: big}); + setTimeout(function(){ + gun.get(key).off(); + },10); + if(typeof process === 'undefined'){ return } + var mem = process.memoryUsage(); + console.log(((mem.heapUsed / mem.heapTotal) * 100).toFixed(0) + '% memory'); + console.log(Object.keys(gun._.graph).length, 'item in memory graph:', Object.keys(gun._.graph)); + },25); + }); + return; it.only('Custom extensions are chainable', function(done){ Gun.chain.filter = function(filter){ diff --git a/test/ptsd/memdisk.html b/test/ptsd/memdisk.html new file mode 100644 index 00000000..e7c6d5fe --- /dev/null +++ b/test/ptsd/memdisk.html @@ -0,0 +1,28 @@ + + \ No newline at end of file diff --git a/test/ptsd/memdisk.js b/test/ptsd/memdisk.js new file mode 100644 index 00000000..513a46a9 --- /dev/null +++ b/test/ptsd/memdisk.js @@ -0,0 +1,31 @@ +var Gun = require('../../gun'); +require('../../lib/memdisk'); +//require('../../lib/file'); + +var gun = Gun(); +var TOTAL = 10000000; +var c = 1000, big = "big"; +while(--c){big += "big"} +c = 0; + +var to = setInterval(function(){ + if(TOTAL < c){ return clearTimeout(to) } + var i = 100; + while(--i){ + it(i); + } +},2); + +function it(i){ + c++; + var key = Gun.text.random(5); + gun.get(key).put({data: Math.random() + big + Math.random()}); + setTimeout(function(){ + gun.get(key).off(); + },5); + if(c % 5000){ return } + if(typeof process === 'undefined'){ return } + //try{global.gc()}catch(e){console.log(e)} + var mem = process.memoryUsage(); + console.log(((mem.heapUsed / mem.heapTotal) * 100).toFixed(0) + '% memory with', Object.keys(gun._.graph).length, 'memory nodes, put', c); +} \ No newline at end of file diff --git a/test/ptsd/radisk.js b/test/ptsd/radisk.js index a72f12e3..d4a6ad8d 100644 --- a/test/ptsd/radisk.js +++ b/test/ptsd/radisk.js @@ -26,7 +26,7 @@ var diff; function bench(){ if(c >= (TOTAL)){ return clearInterval(it); } for(var i = 0; i < l; i++){ - radix(gtr(), ++c, alldone); + radix(++c, gtr(), alldone); if(c % 50000 === 0){ var now = t(); console.log(c);//, (now - last)/1000);