From a634b37b1a452e65a2f741d46c301bda591ebc88 Mon Sep 17 00:00:00 2001 From: Daniel Raeder Date: Sun, 29 May 2022 22:00:52 -0500 Subject: [PATCH 1/7] Heroku fixes (#1243) * Update app.json Removed 'stack' parameter to enable using the latest Heroku stack * Update package.json Remove prepare parameter to fix Heroku deploys --- app.json | 1 - package.json | 1 - 2 files changed, 2 deletions(-) diff --git a/app.json b/app.json index 680bf0bc..b1dc3dbb 100644 --- a/app.json +++ b/app.json @@ -1,6 +1,5 @@ { "name": "gun-server", - "stack": "heroku-18", "website": "http://gun.eco/", "repository": "https://github.com/amark/gun", "logo": "https://avatars3.githubusercontent.com/u/8811914", diff --git a/package.json b/package.json index dd6cbfef..691a8582 100644 --- a/package.json +++ b/package.json @@ -8,7 +8,6 @@ "ios": "browser.ios.js", "android": "browser.android.js", "scripts": { - "prepare": "npm run unbuildSea && npm run unbuild", "start": "node --prof examples/http.js", "debug": "node --prof-process --preprocess -j isolate*.log > v8data.json && rm isolate*.log && echo 'drag & drop ./v8data.json into https://mapbox.github.io/flamebearer/'", "https": "HTTPS_KEY=test/https/server.key HTTPS_CERT=test/https/server.crt npm start", From 8facbcc095bb496acc64a7901ebebcbc4ecc0075 Mon Sep 17 00:00:00 2001 From: Daniel Raeder Date: Mon, 30 May 2022 22:00:14 -0500 Subject: [PATCH 2/7] Adds Heroku deploy unit testing thanks to @bmatusiak (#1244) * Add Heroku deploy unit testing thanks to @bmatusiak 1243.js is a reference to PR #1243 * Update 1243.js Corrected some indentation, removed unnecessary require --- test/bug/1243.js | 118 +++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 118 insertions(+) create mode 100644 test/bug/1243.js diff --git a/test/bug/1243.js b/test/bug/1243.js new file mode 100644 index 00000000..8deaaff2 --- /dev/null +++ b/test/bug/1243.js @@ -0,0 +1,118 @@ +/** + * Heroku CLI REQUIRED! + * + * NOTE: Does not work with npm installed heroku-cli + * - Uninstall with: npm uninstall heroku -g + * - Install Heroku with: curl https://cli-assets.heroku.com/install.sh | sh + * + * 1. Login Heroku with: heroku login + * 2. After login you can run test + * like: $ mocha test/bug/1243.js + * +*/ + +const expect = require('../expect'); +const path = require('path'); +const http = require("https"); + +const outputData = false; + +function request(hostname, done){ + http.get('https://'+hostname+'/',{ + timeout: 1000 * 60 * 5 + }, (res) => { + done(res.statusCode); + }); +} + +function spawn(cmd, done) { + + const spawn = require('child_process').spawn; + let args = cmd.split(" "); + cmd = args.shift(); + + const s = spawn(cmd, args); + let stderrOUT = ""; + + s.stdout.on('data', (data) => { + saveData(data.toString()); + }); + + s.stderr.on('data', (data) => { + saveData(data.toString()); + }); + + function saveData(out){ + stderrOUT += out.toString(); + if(outputData) console.log(out.toString()); + } + + s.on('exit', (code) => { + done(stderrOUT); + }); +} + + function makeid(length) { + let result = ''; + let characters = 'abcdefghijklmnopqrstuvwxyz'; + let charactersLength = characters.length; + for (let i = 0; i < length; i++) { + result += characters.charAt(Math.floor(Math.random() * + charactersLength)); + } + return result; + } + + describe('Heroku deploy', function() { + const heroku_name = 'test-gun-' + makeid(5); + const dir_cwd = path.join(__dirname, "../.."); + + it('create herokuapp '+ heroku_name, function(done) { + this.timeout(15 * 1000); + let c = 'heroku create ' + heroku_name; + spawn(c, (stdout) => { + if (stdout.indexOf("done") > -1) { + done(); + } + }) + }) + + it('add git remote', function(done) { + this.timeout(15 * 1000); + let c = 'heroku git:remote -a '+ heroku_name; + spawn(c, (stdout) => { + if (stdout.indexOf("set git") > -1) { + done(); + } + }) + }) + + it('git push heroku', function(done) { + this.timeout(1000 * 60 * 2); // 2 min + let c = 'git push heroku master --force'; + spawn(c, (stdout) => { + if (stdout.indexOf("deployed to Heroku") > -1) { + done(); + } + }) + }) + + it('fetch heroku app https', function(done) { + this.timeout(1000 * 60 * 5); // 2 min + request(heroku_name+".herokuapp.com",( statusCode )=>{ + expect(statusCode).to.be(200); + done(); + }) + }) + + it('destroy herokuapp ' + heroku_name, function(done) { + this.timeout(15 * 1000); + + let c = 'heroku apps:destroy ' + heroku_name + ' --confirm=' + heroku_name; + spawn(c, (stdout) => { + if (stdout.indexOf("done") > -1) { + done(); + } + }) + }) + }) \ No newline at end of file From 42720c57ea99a57a8b7dcb34c4827779f6e87b05 Mon Sep 17 00:00:00 2001 From: Bradley Matusiak Date: Mon, 30 May 2022 23:02:25 -0400 Subject: [PATCH 3/7] unbuild is no longer silent (#1239) * prepare to sync gun lib unbuild on npm install * Update unbuild.js * Update package.json * Update package.json --- lib/unbuild.js | 32 ++++++++++++++++++++++++++++---- 1 file changed, 28 insertions(+), 4 deletions(-) diff --git a/lib/unbuild.js b/lib/unbuild.js index 0e4a8e20..197fb723 100644 --- a/lib/unbuild.js +++ b/lib/unbuild.js @@ -19,6 +19,13 @@ var mk = function(path){ fs.mkdirSync(path); } +var rn = function(path, newPath){ + path = nodePath.join(dir, path) + newPath = nodePath.join(dir, newPath) + if(fs.existsSync(newPath)){ return } + fs.renameSync(path, newPath); +} + var between = function(text, start, end){ end = end || start; var s = text.indexOf(start); @@ -65,15 +72,19 @@ var undent = function(code, n){ var arg = process.argv[2] || 'gun'; + var g; if('gun' === arg){ - rm('./src'); + g = 'gun'; + rn('./src','./old_src'); mk('./src'); mk('./src/polyfill'); mk('./src/adapters'); } else { - rm('./'+arg); + g = arg; + rn('./'+arg,'./old_'+arg); mk('./'+arg); } + console.log("unbuild:", arg+'.js') var f = read(arg+'.js'); var code = next(f); @@ -81,7 +92,7 @@ var undent = function(code, n){ code = next("/* UNBUILD */"); - if('gun' === arg){ + if('gun' === g){ write('src/polyfill/unbuild.js', undent(code, 1)); arg = ''; } @@ -92,7 +103,20 @@ var undent = function(code, n){ var file = path(arg); if(!file){ return } code = code.replace(/\bUSE\(/g, 'require('); - write(file, undent(code)); + code = undent(code); + var rcode; + try{ rcode = read('old_'+file); } catch(e){} + // console.log(rcode); + if(rcode != code){ + console.log("unbuild:","update",file); + } + write(file, code); recurse(); }()); + if('gun' === g){ + rm('./old_src'); + }else{ + rm('./old_'+g); + } + }()); From fa1e1578f7bfb2305dbe079fd12815ccda9167c8 Mon Sep 17 00:00:00 2001 From: abenezermario <58272831+abenezermario@users.noreply.github.com> Date: Mon, 30 May 2022 23:06:43 -0400 Subject: [PATCH 4/7] fixed s3 (#1245) --- lib/rs3.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/rs3.js b/lib/rs3.js index 3aefc4dd..80c348a6 100644 --- a/lib/rs3.js +++ b/lib/rs3.js @@ -95,7 +95,7 @@ function Store(opt){ cb.end = true; cb(); } // Gun.obj.map(cbs, cbe); // lets see if fixes heroku - if(!IT){ Gun.obj.del(c.l, 1); return } + if(!IT){ delete c.l[1]; return } params.ContinuationToken = data.NextContinuationToken; store.list(cb, match, params, cbs); }); From de46cccc1ecda3f206671d8dd724c40f5a764ea2 Mon Sep 17 00:00:00 2001 From: Bradley Matusiak Date: Fri, 3 Jun 2022 15:27:38 -0400 Subject: [PATCH 5/7] Unbuild update (#1248) * unbuild gun * unbuild sea --- sea/auth.js | 6 +++--- sea/base64.js | 2 +- sea/certify.js | 23 ++++++++++++----------- sea/create.js | 2 +- sea/index.js | 10 +++++----- sea/recall.js | 2 +- sea/shim.js | 2 +- sea/verify.js | 2 +- src/localStorage.js | 41 ++++++++++++++++++++++++++--------------- src/mesh.js | 22 ++++++++++++---------- src/on.js | 12 ++++++++++-- src/put.js | 9 +++++++-- src/root.js | 26 +++++++++++++++++++------- src/shim.js | 3 ++- src/valid.js | 14 ++++++++------ src/websocket.js | 1 + 16 files changed, 110 insertions(+), 67 deletions(-) diff --git a/sea/auth.js b/sea/auth.js index b5473d7e..cdbdaea5 100644 --- a/sea/auth.js +++ b/sea/auth.js @@ -66,7 +66,7 @@ at = user._ = root.get('~'+pair.pub)._; at.opt = upt; // add our credentials in-memory only to our root user instance - user.is = {pub: pair.pub, epub: pair.epub, alias: alias || pair}; + user.is = {pub: pair.pub, epub: pair.epub, alias: alias || pair.pub}; at.sea = act.pair; cat.ing = false; try{if(pass && u == (obj_ify(cat.root.graph['~'+pair.pub].auth)||'')[':']){ opt.shuffle = opt.change = pass; } }catch(e){} // migrate UTF8 & Shuffle! @@ -74,7 +74,7 @@ if(SEA.window && ((gun.back('user')._).opt||opt).remember){ // TODO: this needs to be modular. try{var sS = {}; - sS = window.sessionStorage; + sS = window.sessionStorage; // TODO: FIX BUG putting on `.is`! sS.recall = true; sS.pair = JSON.stringify(pair); // auth using pair is more reliable than alias/pass }catch(e){} @@ -154,4 +154,4 @@ }catch(e){o={}}; return o; } - + \ No newline at end of file diff --git a/sea/base64.js b/sea/base64.js index ccaea7f2..5c4b4a1d 100644 --- a/sea/base64.js +++ b/sea/base64.js @@ -2,7 +2,7 @@ var u; if(u+''== typeof btoa){ if(u+'' == typeof Buffer){ - try{ global.Buffer = require("buffer", 1).Buffer }catch(e){ console.log("Please add `buffer` to your package.json!") } + try{ global.Buffer = require("buffer", 1).Buffer }catch(e){ console.log("Please `npm install buffer` or add it to your package.json !") } } global.btoa = function(data){ return Buffer.from(data, "binary").toString("base64") }; global.atob = function(data){ return Buffer.from(data, "base64").toString("binary") }; diff --git a/sea/certify.js b/sea/certify.js index 2d09ec01..b6145d01 100644 --- a/sea/certify.js +++ b/sea/certify.js @@ -3,23 +3,21 @@ // This is to certify that a group of "certificants" can "put" anything at a group of matched "paths" to the certificate authority's graph SEA.certify = SEA.certify || (async (certificants, policy = {}, authority, cb, opt = {}) => { try { /* + The Certify Protocol was made out of love by a Vietnamese code enthusiast. Vietnamese people around the world deserve respect! IMPORTANT: A Certificate is like a Signature. No one knows who (authority) created/signed a cert until you put it into their graph. "certificants": '*' or a String (Bob.pub) || an Object that contains "pub" as a key || an array of [object || string]. These people will have the rights. "policy": A string ('inbox'), or a RAD/LEX object {'*': 'inbox'}, or an Array of RAD/LEX objects or strings. RAD/LEX object can contain key "?" with indexOf("*") > -1 to force key equals certificant pub. This rule is used to check against soul+'/'+key using Gun.text.match or String.match. "authority": Key pair or priv of the certificate authority. "cb": A callback function after all things are done. - "opt": If opt.expiry (a timestamp) is set, SEA won't sync data after opt.expiry. If opt.blacklist is set, SEA will look for blacklist before syncing. + "opt": If opt.expiry (a timestamp) is set, SEA won't sync data after opt.expiry. If opt.block is set, SEA will look for block before syncing. */ console.log('SEA.certify() is an early experimental community supported method that may change API behavior without warning in any future version.') certificants = (() => { var data = [] if (certificants) { - if ((typeof certificants === 'string' || Array.isArray(certificants)) && certificants.indexOf('*') !== -1) return '*' - if (typeof certificants === 'string') { - return certificants - } - + if ((typeof certificants === 'string' || Array.isArray(certificants)) && certificants.indexOf('*') > -1) return '*' + if (typeof certificants === 'string') return certificants if (Array.isArray(certificants)) { if (certificants.length === 1 && certificants[0]) return typeof certificants[0] === 'object' && certificants[0].pub ? certificants[0].pub : typeof certificants[0] === 'string' ? certificants[0] : null certificants.map(certificant => { @@ -31,7 +29,7 @@ if (typeof certificants === 'object' && certificants.pub) return certificants.pub return data.length > 0 ? data : null } - return null + return })() if (!certificants) return console.log("No certificant found.") @@ -39,8 +37,11 @@ const expiry = opt.expiry && (typeof opt.expiry === 'number' || typeof opt.expiry === 'string') ? parseFloat(opt.expiry) : null const readPolicy = (policy || {}).read ? policy.read : null const writePolicy = (policy || {}).write ? policy.write : typeof policy === 'string' || Array.isArray(policy) || policy["+"] || policy["#"] || policy["."] || policy["="] || policy["*"] || policy[">"] || policy["<"] ? policy : null - const readBlacklist = ((opt || {}).blacklist || {}).read && (typeof opt.blacklist.read === 'string' || opt.blacklist.read['#']) ? opt.blacklist.read : null - const writeBlacklist = typeof (opt || {}).blacklist === 'string' || (((opt || {}).blacklist || {}).write || {})['#'] ? opt.blacklist : ((opt || {}).blacklist || {}).write && (typeof opt.blacklist.write === 'string' || opt.blacklist.write['#']) ? opt.blacklist.write : null + // The "blacklist" feature is now renamed to "block". Why ? BECAUSE BLACK LIVES MATTER! + // We can now use 3 keys: block, blacklist, ban + const block = (opt || {}).block || (opt || {}).blacklist || (opt || {}).ban || {} + const readBlock = block.read && (typeof block.read === 'string' || (block.read || {})['#']) ? block.read : null + const writeBlock = typeof block === 'string' ? block : block.write && (typeof block.write === 'string' || block.write['#']) ? block.write : null if (!readPolicy && !writePolicy) return console.log("No policy found.") @@ -50,8 +51,8 @@ ...(expiry ? {e: expiry} : {}), // inject expiry if possible ...(readPolicy ? {r: readPolicy } : {}), // "r" stands for read, which means read permission. ...(writePolicy ? {w: writePolicy} : {}), // "w" stands for write, which means write permission. - ...(readBlacklist ? {rb: readBlacklist} : {}), // inject READ blacklist if possible - ...(writeBlacklist ? {wb: writeBlacklist} : {}), // inject WRITE blacklist if possible + ...(readBlock ? {rb: readBlock} : {}), // inject READ block if possible + ...(writeBlock ? {wb: writeBlock} : {}), // inject WRITE block if possible }) const certificate = await SEA.sign(data, authority, null, {raw:1}) diff --git a/sea/create.js b/sea/create.js index 15d0c190..da317481 100644 --- a/sea/create.js +++ b/sea/create.js @@ -80,7 +80,7 @@ if(!act.h.ok || !act.i.ok){ return } cat.ing = false; cb({ok: 0, pub: act.pair.pub}); // callback that the user has been created. (Note: ok = 0 because we didn't wait for disk to ack) - if(noop === cb){ pair? gun.auth(pair) : gun.auth(alias, pass) } // if no callback is passed, auto-login after signing up. + if(noop === cb){ pair ? gun.auth(pair) : gun.auth(alias, pass) } // if no callback is passed, auto-login after signing up. } root.get('~@'+alias).once(act.a); return gun; diff --git a/sea/index.js b/sea/index.js index 66fcbc13..a6fe7aba 100644 --- a/sea/index.js +++ b/sea/index.js @@ -100,12 +100,12 @@ if ((String.match(path, lex['#']) && String.match(key, lex['.'])) || (!lex['.'] && String.match(path, lex['#'])) || (!lex['#'] && String.match(key, lex['.'])) || String.match((path ? path + '/' + key : key), lex['#'] || lex)) { // is Certificant forced to present in Path if (lex['+'] && lex['+'].indexOf('*') > -1 && path && path.indexOf(certificant) == -1 && key.indexOf(certificant) == -1) return no(`Path "${path}" or key "${key}" must contain string "${certificant}".`) - // path is allowed, but is there any WRITE blacklist? Check it out - if (data.wb && (typeof data.wb === 'string' || ((data.wb || {})['#']))) { // "data.wb" = path to the WRITE blacklist - var root = at.$.back(-1) + // path is allowed, but is there any WRITE block? Check it out + if (data.wb && (typeof data.wb === 'string' || ((data.wb || {})['#']))) { // "data.wb" = path to the WRITE block + var root = eve.as.root.$.back(-1) if (typeof data.wb === 'string' && '~' !== data.wb.slice(0, 1)) root = root.get('~' + pub) - return root.get(data.wb).get(certificant).once(value => { - if (value && (value === 1 || value === true)) return no("Certificant blacklisted.") + return root.get(data.wb).get(certificant).once(value => { // TODO: INTENT TO DEPRECATE. + if (value && (value === 1 || value === true)) return no(`Certificant ${certificant} blocked.`) return cb(data) }) } diff --git a/sea/recall.js b/sea/recall.js index a8a82e93..c8e5976a 100644 --- a/sea/recall.js +++ b/sea/recall.js @@ -7,7 +7,7 @@ if(SEA.window){ try{ var sS = {}; - sS = window.sessionStorage; + sS = window.sessionStorage; // TODO: FIX BUG putting on `.is`! if(sS){ (root._).opt.remember = true; ((gun.back('user')._).opt||opt).remember = true; diff --git a/sea/shim.js b/sea/shim.js index c0d18a64..e955e736 100644 --- a/sea/shim.js +++ b/sea/shim.js @@ -40,7 +40,7 @@ api.ossl = api.subtle = new WebCrypto({directory: 'ossl'}).subtle // ECDH } catch(e){ - console.log("Please add `@peculiar/webcrypto` to your package.json!"); + console.log("Please `npm install @peculiar/webcrypto` or add it to your package.json !"); }} module.exports = api diff --git a/sea/verify.js b/sea/verify.js index e2e4ec97..b431badf 100644 --- a/sea/verify.js +++ b/sea/verify.js @@ -40,7 +40,7 @@ }}); module.exports = SEA.verify; - // legacy & ossl leak mitigation: + // legacy & ossl memory leak mitigation: var knownKeys = {}; var keyForPair = SEA.opt.slow_leak = pair => { diff --git a/src/localStorage.js b/src/localStorage.js index f8a35099..60c98137 100644 --- a/src/localStorage.js +++ b/src/localStorage.js @@ -7,13 +7,18 @@ if(!store){ Gun.log("Warning: No localStorage exists to persist data to!"); store = {setItem: function(k,v){this[k]=v}, removeItem: function(k){delete this[k]}, getItem: function(k){return this[k]}}; } + +var parse = JSON.parseAsync || function(t,cb,r){ var u; try{ cb(u, JSON.parse(t,r)) }catch(e){ cb(e) } } +var json = JSON.stringifyAsync || function(v,cb,r,s){ var u; try{ cb(u, JSON.stringify(v,r,s)) }catch(e){ cb(e) } } + Gun.on('create', function lg(root){ this.to.next(root); - var opt = root.opt, graph = root.graph, acks = [], disk, to; + var opt = root.opt, graph = root.graph, acks = [], disk, to, size, stop; if(false === opt.localStorage){ return } opt.prefix = opt.file || 'gun/'; - try{ disk = lg[opt.prefix] = lg[opt.prefix] || JSON.parse(store.getItem(opt.prefix)) || {}; // TODO: Perf! This will block, should we care, since limited to 5MB anyways? + try{ disk = lg[opt.prefix] = lg[opt.prefix] || JSON.parse(size = store.getItem(opt.prefix)) || {}; // TODO: Perf! This will block, should we care, since limited to 5MB anyways? }catch(e){ disk = lg[opt.prefix] = {}; } + size = (size||'').length; root.on('get', function(msg){ this.to.next(msg); @@ -31,24 +36,30 @@ Gun.on('create', function lg(root){ root.on('put', function(msg){ this.to.next(msg); // remember to call next middleware adapter - var put = msg.put, soul = put['#'], key = put['.'], tmp; // pull data off wire envelope + var put = msg.put, soul = put['#'], key = put['.'], id = msg['#'], tmp; // pull data off wire envelope disk[soul] = Gun.state.ify(disk[soul], key, put['>'], put[':'], soul); // merge into disk object - if(!msg['@']){ acks.push(msg['#']) } // then ack any non-ack write. // TODO: use batch id. + if(stop && size > (4999880)){ root.on('in', {'@': id, err: "localStorage max!"}); return; } + if(!msg['@']){ acks.push(id) } // then ack any non-ack write. // TODO: use batch id. if(to){ return } - //flush();return; - to = setTimeout(flush, opt.wait || 1); // that gets saved as a whole to disk every 1ms + to = setTimeout(flush, 9+(size / 333)); // 0.1MB = 0.3s, 5MB = 15s }); function flush(){ + if(!acks.length && ((setTimeout.turn||'').s||'').length){ setTimeout(flush,99); return; } // defer if "busy" && no saves. var err, ack = acks; clearTimeout(to); to = false; acks = []; - try{store.setItem(opt.prefix, JSON.stringify(disk)); - }catch(e){ - Gun.log((err = (e || "localStorage failure")) + " Consider using GUN's IndexedDB plugin for RAD for more storage space, https://gun.eco/docs/RAD#install"); - root.on('localStorage:error', {err: err, get: opt.prefix, put: disk}); - } - if(!err && !Object.empty(opt.peers)){ return } // only ack if there are no peers. // Switch this to probabilistic mode - setTimeout.each(ack, function(id){ - root.on('in', {'@': id, err: err, ok: 0}); // localStorage isn't reliable, so make its `ok` code be a low number. - }); + json(disk, function(err, tmp){ + try{!err && store.setItem(opt.prefix, tmp); + }catch(e){ err = stop = e || "localStorage failure" } + if(err){ + Gun.log(err + " Consider using GUN's IndexedDB plugin for RAD for more storage space, https://gun.eco/docs/RAD#install"); + root.on('localStorage:error', {err: err, get: opt.prefix, put: disk}); + } + size = tmp.length; + + if(!err && !Object.empty(opt.peers)){ return } // only ack if there are no peers. // Switch this to probabilistic mode + setTimeout.each(ack, function(id){ + root.on('in', {'@': id, err: err, ok: 0}); // localStorage isn't reliable, so make its `ok` code be a low number. + },0,99); + }) } }); diff --git a/src/mesh.js b/src/mesh.js index 74f6b95f..27644552 100644 --- a/src/mesh.js +++ b/src/mesh.js @@ -1,6 +1,11 @@ require('./shim'); +var noop = function(){} +var parse = JSON.parseAsync || function(t,cb,r){ var u, d = +new Date; try{ cb(u, JSON.parse(t,r), json.sucks(+new Date - d)) }catch(e){ cb(e) } } +var json = JSON.stringifyAsync || function(v,cb,r,s){ var u, d = +new Date; try{ cb(u, JSON.stringify(v,r,s), json.sucks(+new Date - d)) }catch(e){ cb(e) } } +json.sucks = function(d){ if(d > 99){ console.log("Warning: JSON blocking CPU detected. Add `gun/lib/yson.js` to fix."); json.sucks = noop } } + function Mesh(root){ var mesh = function(){}; var opt = root.opt || {}; @@ -10,8 +15,6 @@ function Mesh(root){ opt.pack = opt.pack || (opt.max * 0.01 * 0.01); opt.puff = opt.puff || 9; // IDEA: do a start/end benchmark, divide ops/result. var puff = setTimeout.turn || setTimeout; - var parse = JSON.parseAsync || function(t,cb,r){ var u; try{ cb(u, JSON.parse(t,r)) }catch(e){ cb(e) } } - var json = JSON.stringifyAsync || function(v,cb,r,s){ var u; try{ cb(u, JSON.stringify(v,r,s)) }catch(e){ cb(e) } } var dup = root.dup, dup_check = dup.check, dup_track = dup.track; @@ -38,7 +41,7 @@ function Mesh(root){ var P = opt.puff; (function go(){ var S = +new Date; - var i = 0, m; while(i < P && (m = msg[i++])){ hear(m, peer) } + var i = 0, m; while(i < P && (m = msg[i++])){ mesh.hear(m, peer) } msg = msg.slice(i); // slicing after is faster than shifting during. console.STAT && console.STAT(S, +new Date - S, 'hear loop'); flush(peer); // force send all synchronously batched acks. @@ -90,7 +93,6 @@ function Mesh(root){ mesh.leap = mesh.last = null; // warning! mesh.leap could be buggy. } var tomap = function(k,i,m){m(k,true)}; - var noop = function(){}; hear.c = hear.d = 0; ;(function(){ @@ -105,7 +107,7 @@ function Mesh(root){ console.STAT && console.STAT(S, +new Date - S, 'say json+hash'); msg._.$put = t; msg['##'] = h; - say(msg, peer); + mesh.say(msg, peer); delete msg._.$put; }, sort); } @@ -147,7 +149,7 @@ function Mesh(root){ loop = 1; var wr = meta.raw; meta.raw = raw; // quick perf hack var i = 0, p; while(i < 9 && (p = (pl||'')[i++])){ if(!(p = ps[p])){ continue } - say(msg, p); + mesh.say(msg, p); } meta.raw = wr; loop = 0; pl = pl.slice(i); // slicing after is faster than shifting during. @@ -221,7 +223,7 @@ function Mesh(root){ function res(err, raw){ if(err){ return } // TODO: Handle!! meta.raw = raw; //if(meta && (raw||'').length < (999 * 99)){ meta.raw = raw } // HNPERF: If string too big, don't keep in memory. - say(msg, peer); + mesh.say(msg, peer); } } }()); @@ -239,7 +241,6 @@ function Mesh(root){ } // for now - find better place later. function send(raw, peer){ try{ - //console.log('SAY:', peer.id, (raw||'').slice(0,250), ((raw||'').length / 1024 / 1024).toFixed(4)); var wire = peer.wire; if(peer.say){ peer.say(raw); @@ -253,7 +254,8 @@ function Mesh(root){ }} mesh.hi = function(peer){ - var tmp = peer.wire || {}; + var wire = peer.wire, tmp; + if(!wire){ mesh.wire((peer.length && {url: peer, id: peer}) || peer); return } if(peer.id){ opt.peers[peer.url || peer.id] = peer; } else { @@ -262,7 +264,7 @@ function Mesh(root){ delete dup.s[peer.last]; // IMPORTANT: see https://gun.eco/docs/DAM#self } peer.met = peer.met || +(new Date); - if(!tmp.hied){ root.on(tmp.hied = 'hi', peer) } + if(!wire.hied){ root.on(wire.hied = 'hi', peer) } // @rogowski I need this here by default for now to fix go1dfish's bug tmp = peer.queue; peer.queue = []; setTimeout.each(tmp||[],function(msg){ diff --git a/src/on.js b/src/on.js index c60be5ef..c2fc1386 100644 --- a/src/on.js +++ b/src/on.js @@ -63,13 +63,21 @@ Gun.chain.once = function(cb, opt){ opt = opt || {}; // avoid rewriting if('string' == typeof tmp){ return } // TODO: BUG? Will this always load? clearTimeout((cat.one||'')[id]); // clear "not found" since they only get set on cat. clearTimeout(one[id]); one[id] = setTimeout(once, opt.wait||99); // TODO: Bug? This doesn't handle plural chains. - function once(){ + function once(f){ if(!at.has && !at.soul){ at = {put: data, get: key} } // handles non-core messages. if(u === (tmp = at.put)){ tmp = ((msg.$$||'')._||'').put } - if('string' == typeof Gun.valid(tmp)){ tmp = root.$.get(tmp)._.put; if(tmp === u){return} } + if('string' == typeof Gun.valid(tmp)){ + tmp = root.$.get(tmp)._.put; + if(tmp === u && !f){ + one[id] = setTimeout(function(){ once(1) }, opt.wait||99); // TODO: Quick fix. Maybe use ack count for more predictable control? + return + } + } + //console.log("AND VANISHED", data); if(eve.stun){ return } if('' === one[id]){ return } one[id] = ''; if(cat.soul || cat.has){ eve.off() } // TODO: Plural chains? // else { ?.off() } // better than one check? cb.call($, tmp, at.get); + clearTimeout(one[id]); // clear "not found" since they only get set on cat. // TODO: This was hackily added, is it necessary or important? Probably not, in future try removing this. Was added just as a safety for the `&& !f` check. }; }, {on: 1}); return gun; diff --git a/src/put.js b/src/put.js index 3eeda5d8..54bd2c78 100644 --- a/src/put.js +++ b/src/put.js @@ -28,12 +28,13 @@ Gun.chain.put = function(data, cb, as){ // I rewrote it :) } k && (to.path || (to.path = [])).push(k); if(!(v = valid(d)) && !(g = Gun.is(d))){ - if(!Object.plain(d)){ (as.ack||noop).call(as, as.out = {err: as.err = Gun.log("Invalid data: " + ((d && (tmp = d.constructor) && tmp.name) || typeof d) + " at " + (as.via.back(function(at){at.get && tmp.push(at.get)}, tmp = []) || tmp.join('.'))+'.'+(to.path||[]).join('.'))}); as.ran(as); return } + if(!Object.plain(d)){ ran.err(as, "Invalid data: "+ check(d) +" at " + (as.via.back(function(at){at.get && tmp.push(at.get)}, tmp = []) || tmp.join('.'))+'.'+(to.path||[]).join('.')); return } var seen = as.seen || (as.seen = []), i = seen.length; while(i--){ if(d === (tmp = seen[i]).it){ v = d = tmp.link; break } } } if(k && v){ at.node = state_ify(at.node, k, s, d) } // handle soul later. else { + if(!as.seen){ ran.err(as, "Data at root of graph must be a node (an object)."); return } as.seen.push(cat = {it: d, link: {}, todo: g? [] : Object.keys(d).sort().reverse(), path: (to.path||[]).slice(), up: at}); // Any perf reasons to CPU schedule this .keys( ? at.node = state_ify(at.node, k, s, cat.link); !g && cat.todo.length && to.push(cat); @@ -113,11 +114,14 @@ function ran(as){ setTimeout.each(Object.keys(stun = stun.add||''), function(cb){ if(cb = stun[cb]){cb()} }); // resume the stunned reads // Any perf reasons to CPU schedule this .keys( ? }).hatch = tmp; // this is not official yet ^ //console.log(1, "PUT", as.run, as.graph); - (as.via._).on('out', {put: as.out = as.graph, opt: as.opt, '#': ask, _: tmp}); + (as.via._).on('out', {put: as.out = as.graph, ok: as.ok || as.opt, opt: as.opt, '#': ask, _: tmp}); }; ran.end = function(stun,root){ stun.end = noop; // like with the earlier id, cheaper to make this flag a function so below callbacks do not have to do an extra type check. if(stun.the.to === stun && stun === stun.the.last){ delete root.stun } stun.off(); +}; ran.err = function(as, err){ + (as.ack||noop).call(as, as.out = { err: as.err = Gun.log(err) }); + as.ran(as); } function get(as){ @@ -141,6 +145,7 @@ function get(as){ return; } } +function check(d, tmp){ return ((d && (tmp = d.constructor) && tmp.name) || typeof d) } var u, empty = {}, noop = function(){}, turn = setTimeout.turn, valid = Gun.valid, state_ify = Gun.state.ify; var iife = function(fn,as){fn.call(as||empty)} diff --git a/src/root.js b/src/root.js index 23a3403c..7a3bb29f 100644 --- a/src/root.js +++ b/src/root.js @@ -39,6 +39,7 @@ Gun.ask = require('./ask'); return gun; } function universe(msg){ + // TODO: BUG! msg.out = null being set! //if(!F){ var eve = this; setTimeout(function(){ universe.call(eve, msg,1) },Math.random() * 100);return; } // ADD F TO PARAMS! if(!msg){ return } if(msg.out === universe){ this.to.next(msg); return } @@ -70,7 +71,7 @@ Gun.ask = require('./ask'); } ctx.latch = root.hatch; ctx.match = root.hatch = []; var put = msg.put; - var DBG = ctx.DBG = msg.DBG, S = +new Date; + var DBG = ctx.DBG = msg.DBG, S = +new Date; CT = CT || S; if(put['#'] && put['.']){ /*root && root.on('put', msg);*/ return } // TODO: BUG! This needs to call HAM instead. DBG && (DBG.p = S); ctx['#'] = msg['#']; @@ -107,13 +108,14 @@ Gun.ask = require('./ask'); if(!valid(val)){ err = ERR+cut(key)+"on"+cut(soul)+"bad "+(typeof val)+cut(val); break } //ctx.all++; //ctx.ack[soul+key] = ''; ham(val, key, soul, state, msg); + ++C; // courtesy count; } if((kl = kl.slice(i)).length){ turn(pop); return } ++ni; kl = null; pop(o); }()); } Gun.on.put = put; // TODO: MARK!!! clock below, reconnect sync, SEA certify wire merge, User.auth taking multiple times, // msg put, put, say ack, hear loop... - // WASIS BUG! first .once( undef 2nd good. .off othe rpeople: .open + // WASIS BUG! local peer not ack. .off other people: .open function ham(val, key, soul, state, msg){ var ctx = msg._||'', root = ctx.root, graph = root.graph, lot, tmp; var vertex = graph[soul] || empty, was = state_is(vertex, key, 1), known = vertex[key]; @@ -132,6 +134,7 @@ Gun.ask = require('./ask'); } ctx.stun++; // TODO: 'forget' feature in SEA tied to this, bad approach, but hacked in for now. Any changes here must update there. var aid = msg['#']+ctx.all++, id = {toString: function(){ return aid }, _: ctx}; id.toJSON = id.toString; // this *trick* makes it compatible between old & new versions. + root.dup.track(id)['#'] = msg['#']; // fixes new OK acks for RPC like RTC. DBG && (DBG.ph = DBG.ph || +new Date); root.on('put', {'#': id, '@': msg['@'], put: {'#': soul, '.': key, ':': val, '>': state}, _: ctx}); } @@ -156,16 +159,23 @@ Gun.ask = require('./ask'); if(!(msg = ctx.msg) || ctx.err || msg.err){ return } msg.out = universe; ctx.root.on('out', msg); + + CF(); // courtesy check; } function ack(msg){ // aggregate ACKs. var id = msg['@'] || '', ctx; - if(!(ctx = id._)){ return } + if(!(ctx = id._)){ + var dup = (dup = msg.$) && (dup = dup._) && (dup = dup.root) && (dup = dup.dup); + if(!(dup = dup.check(id))){ return } + msg['@'] = dup['#'] || msg['@']; + return; + } ctx.acks = (ctx.acks||0) + 1; if(ctx.err = msg.err){ msg['@'] = ctx['#']; fire(ctx); // TODO: BUG? How it skips/stops propagation of msg if any 1 item is error, this would assume a whole batch/resync has same malicious intent. } - if(!ctx.stop && !ctx.crack){ ctx.crack = ctx.match && ctx.match.push(function(){back(ctx)}) } // handle synchronous acks + if(!ctx.stop && !ctx.crack){ ctx.crack = ctx.match && ctx.match.push(function(){back(ctx)}) } // handle synchronous acks. NOTE: If a storage peer ACKs synchronously then the PUT loop has not even counted up how many items need to be processed, so ctx.STOP flags this and adds only 1 callback to the end of the PUT loop. back(ctx); } function back(ctx){ @@ -177,6 +187,7 @@ Gun.ask = require('./ask'); var ERR = "Error: Invalid graph!"; var cut = function(s){ return " '"+(''+s).slice(0,9)+"...' " } var L = JSON.stringify, MD = 2147483647, State = Gun.state; + var C = 0, CT, CF = function(){if(C>999 && (C/-(CT - (CT = +new Date))>1)){Gun.window && console.log("Warning: You're syncing 1K+ records a second, faster than DOM can update - consider limiting query.");CF=function(){C=0}}}; }()); @@ -250,19 +261,20 @@ Gun.ask = require('./ask'); if(!Object.plain(opt)){ opt = {} } if(!Object.plain(at.opt)){ at.opt = opt } if('string' == typeof tmp){ tmp = [tmp] } + if(!Object.plain(at.opt.peers)){ at.opt.peers = {}} if(tmp instanceof Array){ - if(!Object.plain(at.opt.peers)){ at.opt.peers = {}} + opt.peers = {}; tmp.forEach(function(url){ var p = {}; p.id = p.url = url; - at.opt.peers[url] = at.opt.peers[url] || p; + opt.peers[url] = at.opt.peers[url] = at.opt.peers[url] || p; }) } - at.opt.peers = at.opt.peers || {}; obj_each(opt, function each(k){ var v = this[k]; if((this && this.hasOwnProperty(k)) || 'string' == typeof v || Object.empty(v)){ this[k] = v; return } if(v && v.constructor !== Object && !(v instanceof Array)){ return } obj_each(v, each); }); + at.opt.from = opt; Gun.on('opt', at); at.opt.uuid = at.opt.uuid || function uuid(l){ return Gun.state().toString(36).replace('.','') + String.random(l||12) } return gun; diff --git a/src/shim.js b/src/shim.js index 88a27f51..a7a6b15a 100644 --- a/src/shim.js +++ b/src/shim.js @@ -48,8 +48,9 @@ Object.keys = Object.keys || function(o){ } ;(function(){ // max ~1ms or before stack overflow var u, sT = setTimeout, l = 0, c = 0, sI = (typeof setImmediate !== ''+u && setImmediate) || sT; // queueMicrotask faster but blocks UI + sT.hold = sT.hold || 9; sT.poll = sT.poll || function(f){ //f(); return; // for testing - if((1 >= (+new Date - l)) && c++ < 3333){ f(); return } + if((sT.hold >= (+new Date - l)) && c++ < 3333){ f(); return } sI(function(){ l = +new Date; f() },c=0) } }()); diff --git a/src/valid.js b/src/valid.js index b9403eca..84d21990 100644 --- a/src/valid.js +++ b/src/valid.js @@ -1,3 +1,4 @@ + // Valid values are a subset of JSON: null, binary, number (!Infinity), text, // or a soul relation. Arrays need special algorithms to handle concurrency, // so they are not supported directly. Use an extension that supports them if @@ -5,10 +6,11 @@ module.exports = function (v) { // "deletes", nulling out keys. return v === null || - "string" === typeof v || - "boolean" === typeof v || - // we want +/- Infinity to be, but JSON does not support it, sad face. - // can you guess what v === v checks for? ;) - ("number" === typeof v && v != Infinity && v != -Infinity && v === v) || - (!!v && "string" == typeof v["#"] && Object.keys(v).length === 1 && v["#"]); + "string" === typeof v || + "boolean" === typeof v || + // we want +/- Infinity to be, but JSON does not support it, sad face. + // can you guess what v === v checks for? ;) + ("number" === typeof v && v != Infinity && v != -Infinity && v === v) || + (!!v && "string" == typeof v["#"] && Object.keys(v).length === 1 && v["#"]); } + \ No newline at end of file diff --git a/src/websocket.js b/src/websocket.js index be2263d5..7d0bd7b9 100644 --- a/src/websocket.js +++ b/src/websocket.js @@ -46,6 +46,7 @@ Gun.on('opt', function(root){ var wait = 2 * 999; function reconnect(peer){ clearTimeout(peer.defer); + if(!opt.peers[peer.url]){ return } if(doc && peer.retry <= 0){ return } peer.retry = (peer.retry || opt.retry+1 || 60) - ((-peer.tried + (peer.tried = +new Date) < wait*4)?1:0); peer.defer = setTimeout(function to(){ From 87652467b55b3c22ea5cfb17d9e6eafa3b6bd63c Mon Sep 17 00:00:00 2001 From: Bradley Matusiak Date: Mon, 6 Jun 2022 18:02:36 -0400 Subject: [PATCH 6/7] chain fork (#1247) * chain fork * removed test * removed unbuild * moved chain fork to libs folder --- lib/fork.js | 81 +++++++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 81 insertions(+) create mode 100644 lib/fork.js diff --git a/lib/fork.js b/lib/fork.js new file mode 100644 index 00000000..3d60d9a0 --- /dev/null +++ b/lib/fork.js @@ -0,0 +1,81 @@ +/* +describe('API Chain Features', function(){ + + describe('Gun.chain.fork', function(){ + var gun = Gun(); + var fork; + it('create fork', function(done){ + fork = gun.fork().wire(); + done(); + }); + it('put data via fork', function(done){ + fork.get("fork-test").get("fork").put("test123").once(()=>done()); + }); + it('get data via main', function(done){ + gun.get("fork-test").get("fork").once((data)=>{ + expect(data).to.be("test123"); + done(); + }); + }); + it('put data via main', function(done){ + gun.get("fork-test").get("main").put("test321").once(()=>done()); + }); + it('get data via fork', function(done){ + fork.get("fork-test").get("main").once((data)=>{ + expect(data).to.be("test321"); + done(); + }); + }); + }) + +}) +*/ +(function (Gun, u) { + /** + * + * credits: + * github:bmatusiak + * + */ + Gun.chain.fork = function(g) { + var gun = this._; + var w = {}, + mesh = () => { + var root = gun.root, + opt = root.opt; + return opt.mesh || Gun.Mesh(root); + } + w.link = function() { + if (this._l) return this._l; + this._l = { + send: (msg) => { + if (!this.l || !this.l.onmessage) + throw 'not attached'; + this.l.onmessage(msg); + } + } + return this._l; + }; + w.attach = function(l) { + if (this.l) + throw 'already attached'; + var peer = { wire: l }; + l.onmessage = function(msg) { + mesh().hear(msg.data || msg, peer); + }; + mesh().hi(this.l = l && peer); + }; + w.wire = function(opts) { + var f = new Gun(opts); + f.fork(w); + return f; + }; + if (g) { + w.attach(g.link()); + g.attach(w.link()); + } + return w; + }; + + +})((typeof window !== "undefined") ? window.Gun : require('../gun')) \ No newline at end of file From 9c9a5fd293d6cefad3947d7311ee761a43aedc84 Mon Sep 17 00:00:00 2001 From: Bradley Matusiak Date: Mon, 6 Jun 2022 18:04:29 -0400 Subject: [PATCH 7/7] Lex builder (#1249) * lex builder * updated lex builder * added args to map * added match to lex-builder * update match * moved matach and gave Lex to Global Gun * fix match in lex --- lib/lex.js | 93 ++++++++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 93 insertions(+) create mode 100644 lib/lex.js diff --git a/lib/lex.js b/lib/lex.js new file mode 100644 index 00000000..b814870e --- /dev/null +++ b/lib/lex.js @@ -0,0 +1,93 @@ +(function (Gun, u) { + /** + * + * credits: + * github:bmatusiak + * + */ + var lex = (gun) => { + function Lex() {} + + Lex.prototype = Object.create(Object.prototype, { + constructor: { + value: Lex + } + }); + Lex.prototype.toString = function () { + return JSON.stringify(this); + } + Lex.prototype.more = function (m) { + this[">"] = m; + return this; + } + Lex.prototype.less = function (le) { + this["<"] = le; + return this; + } + Lex.prototype.in = function () { + var l = new Lex(); + this["."] = l; + return l; + } + Lex.prototype.of = function () { + var l = new Lex(); + this.hash(l) + return l; + } + Lex.prototype.hash = function (h) { + this["#"] = h; + return this; + } + Lex.prototype.prefix = function (p) { + this["*"] = p; + return this; + } + Lex.prototype.return = function (r) { + this["="] = r; + return this; + } + Lex.prototype.limit = function (l) { + this["%"] = l; + return this; + } + Lex.prototype.reverse = function (rv) { + this["-"] = rv || 1; + return this; + } + Lex.prototype.includes = function (i) { + this["+"] = i; + return this; + } + Lex.prototype.map = function (...args) { + return gun.map(this, ...args); + } + Lex.prototype.match = lex.match; + + return new Lex(); + }; + + lex.match = function(t,o){ var tmp, u; + o = o || this || {}; + if('string' == typeof o){ o = {'=': o} } + if('string' !== typeof t){ return false } + tmp = (o['='] || o['*'] || o['>'] || o['<']); + if(t === tmp){ return true } + if(u !== o['=']){ return false } + tmp = (o['*'] || o['>']); + if(t.slice(0, (tmp||'').length) === tmp){ return true } + if(u !== o['*']){ return false } + if(u !== o['>'] && u !== o['<']){ + return (t >= o['>'] && t <= o['<'])? true : false; + } + if(u !== o['>'] && t >= o['>']){ return true } + if(u !== o['<'] && t <= o['<']){ return true } + return false; + } + + Gun.Lex = lex; + + Gun.chain.lex = function () { + return lex(this); + } + +})((typeof window !== "undefined") ? window.Gun : require('../gun')) \ No newline at end of file