Mirroristas/gun
2
0
Fork 0
mirror of https://github.com/amark/gun.git synced 2025-03-30 15:08:33 +00:00
Code Issues Packages Projects Releases Wiki Activity

RAD rindexed radisk!

Browse Source
This commit is contained in:
Mark Nadal 2019-03-23 15:21:58 -07:00
parent 116a2521df
commit bc742aab0d
12 changed files with 184 additions and 127 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

8
gun.js
View File

@ -1361,7 +1361,7 @@
// ~who#where.where=what>when@was
// TODO: BUG! Put probably cannot handle plural chains!
var gun = this, at = (gun._), root = at.root.$, ctx = root._, tmp;
if(tmp = ctx.puts){
/*if(tmp = ctx.puts){
if(tmp > 100){ // without this, when synchronous, writes to a 'not found' pile up, when 'not found' resolves it recursively calls `put` which incrementally resolves each write. Stack overflow limits can be as low as 10K, so this limit is hardcoded to 1% of 10K.
(ctx.stack || (ctx.stack = [])).push([gun, data, cb, as]);
clearTimeout(ctx.puto);
@ -1375,7 +1375,7 @@
return gun;
}
++ctx.puts;
} else { ctx.puts = 1 }
} else { ctx.puts = 1 }*/
as = as || {};
as.data = data;
as.via = as.$ = as.via || as.$ || gun;
@ -1470,7 +1470,9 @@
if(!ack.lack){ this.off() } // One response is good enough for us currently. Later we may want to adjust this.
if(!as.ack){ return }
as.ack(ack, this);
// --C;
}, as.opt);
//C++;
// NOW is a hack to get synchronous replies to correctly call.
// and STOP is a hack to get async behavior to correctly call.
// neither of these are ideal, need to be fixed without hacks,
@ -1485,6 +1487,7 @@
}, as);
if(as.res){ as.res() }
} function no(v,k){ if(v){ return true } }
//console.debug(999,1); var C = 0; setInterval(function(){ try{ debugg.innerHTML = C }catch(e){console.log(e)} }, 500);
function map(v,k,n, at){ var as = this;
var is = Gun.is(v);
@ -1673,6 +1676,7 @@
}
function val(msg, eve, to){
if(!msg.$){ eve.off(); return }
var opt = this.as, cat = opt.at, gun = msg.$, at = gun._, data = at.put || msg.put, link, tmp;
if(tmp = msg.$$){
link = tmp = (msg.$$._);

11
lib/radisk.js
View File

@ -9,8 +9,8 @@
if(has){ return has }
opt.pack = opt.pack || (opt.memory? (opt.memory * 1000 * 1000) : 1399000000) * 0.3; // max_old_space_size defaults to 1400 MB.
opt.until = opt.until || opt.wait || 9;
opt.batch = opt.batch || 10 * 1000;
opt.until = opt.until || opt.wait || 250;
opt.batch = opt.batch || (10 * 1000);
opt.chunk = opt.chunk || (1024 * 1024 * 10); // 10MB
opt.code = opt.code || {};
opt.code.from = opt.code.from || '!';
@ -63,7 +63,8 @@
r.batch(key, val);
if(cb){ r.batch.acks.push(cb) }
if(++r.batch.ed >= opt.batch){ return r.thrash() } // (2)
clearTimeout(r.batch.to); // (1)
if(r.batch.to){ return }
//clearTimeout(r.batch.to); // (1) // THIS LINE IS EVIL! NEVER USE IT! ALSO NEVER DELETE THIS SO WE NEVER MAKE THE SAME MISTAKE AGAIN!
r.batch.to = setTimeout(r.thrash, opt.until || 1);
}
@ -82,9 +83,11 @@
r.batch = Radix();
r.batch.acks = [];
r.batch.ed = 0;
//var id = Gun.text.random(2), S = (+new Date); console.log("<<<<<<<<<<<<", id);
r.save(batch, function(err, ok){
if(++i > 1){ opt.log('RAD ERR: Radisk has callbacked multiple times, please report this as a BUG at github.com/amark/gun/issues ! ' + i); return }
if(err){ opt.log('err', err) }
//console.log(">>>>>>>>>>>>", id, ((+new Date) - S), batch.acks.length);
map(batch.acks, function(cb){ cb(err, ok) });
thrash.at = null;
thrash.ing = false;
@ -285,7 +288,7 @@
if(opt.pack <= data.length){
p.err = "Chunk too big!";
} else {
data = data.toString();
data = data.toString(); // If it crashes, it crashes here. How!?? We check size first!
}
}catch(e){ p.err = e }
if(p.err){ return map(q, p.ack) }

136
lib/rindexed.js
View File

@ -10,126 +10,46 @@
opt = opt || {};
opt.file = String(opt.file || 'radata');
if(Gun.TESTING){ opt.file = 'radatatest' }
var db = null;
opt.chunk = opt.chunk || (1024 * 1024); // 1MB
var db = null, u;
try{opt.indexedDB = opt.indexedDB || indexedDB}catch(e){}
try{if(!opt.indexedDB || 'file:' == location.protocol){
var store = {}, s = {}, u;
var store = {}, s = {};
store.put = function(f, d, cb){ s[f] = d; cb(null, 1) };
store.get = function(f, cb){ cb(null, s[f] || u) };
console.log('Warning: No indexedDB exists to persist data to!');
return store;
}}catch(e){}
// Initialize indexedDB. Version 1.
var request = opt.indexedDB.open(opt.file, 1);
var store = function Store(){};
store.start = function(){
var o = indexedDB.open(opt.file, 1);
o.onupgradeneeded = function(eve){ (eve.target.result).createObjectStore(opt.file) }
o.onsuccess = function(){ db = o.result }
o.onerror = function(eve){ console.log(eve||1); }
}; store.start();
// Create schema. onupgradeneeded is called only when DB is first created or when the DB version increases.
request.onupgradeneeded = function(event){
var db = event.target.result;
db.createObjectStore(opt.file);
store.put = function(key, data, cb){
if(!db){ setTimeout(function(){ store.put(key, data, cb) },1); return }
var tx = db.transaction([opt.file], 'readwrite');
var obj = tx.objectStore(opt.file);
var req = obj.put(data, ''+key);
req.onsuccess = obj.onsuccess = tx.onsuccess = function(){ cb(null, 1) }
req.onabort = obj.onabort = tx.onabort = function(eve){ cb(eve||'put.tx.abort') }
req.onerror = obj.onerror = tx.onerror = function(eve){ cb(eve||'put.tx.error') }
}
// onsuccess is called when the DB is ready.
request.onsuccess = function(){
db = request.result;
store.get = function(key, cb){
if(!db){ setTimeout(function(){ store.get(key, cb) },9); return }
var tx = db.transaction([opt.file], 'readonly');
var obj = tx.objectStore(opt.file);
var req = obj.get(''+key);
req.onsuccess = function(){ cb(null, req.result) }
req.onabort = function(eve){ cb(eve||4) }
req.onerror = function(eve){ cb(eve||5) }
}
request.onerror = function(event){
console.log('ERROR: RAD IndexedDB generic error:', event);
};
var store = function Store(){}, u;
store.put = function(file, data, cb){
cb = cb || function(){};
var doPut = function(){
// Start a transaction. The transaction will be automaticallt closed when the last success/error handler took no new action.
var transaction = db.transaction([opt.file], 'readwrite');
// Add or update data.
var radStore = transaction.objectStore(opt.file);
var putRequest = radStore.put(data, file);
putRequest.onsuccess = radStore.onsuccess = transaction.onsuccess = function(){
//console.log('RAD IndexedDB put transaction was succesful.');
cb(null, 1);
};
putRequest.onabort = radStore.onabort = transaction.onabort = function(){
var es = 'ERROR: RAD IndexedDB put transaction was aborted.';
console.log(es);
cb(es, undefined);
};
putRequest.onerror = radStore.onerror = transaction.onerror = function(event){
var es = 'ERROR: RAD IndexedDB put transaction was in error: ' + JSON.stringify(event)
console.log(es);
cb(es, undefined);
};
}
if(!db){
waitDbReady(doPut, 100, function(){
var es = 'ERROR: Timeout: RAD IndexedDB not ready.';
console.log(es);
cb(es, undefined);
}, 10)
} else {
doPut();
}
};
store.get = function(file, cb){
cb = cb || function(){};
var doGet = function(){
// Start a transaction. The transaction will be automaticallt closed when the last success/error handler took no new action.
var transaction = db.transaction([opt.file], 'readwrite');
// Read data.
var radStore = transaction.objectStore(opt.file);
var getRequest = radStore.get(file);
getRequest.onsuccess = function(){
//console.log('RAD IndexedDB get transaction was succesful.');
cb(null, getRequest.result);
};
getRequest.onabort = function(){
var es = 'ERROR: RAD IndexedDB get transaction was aborted.';
console.log(es);
cb(es, undefined);
};
getRequest.onerror = function(event){
var es = 'ERROR: RAD IndexedDB get transaction was in error: ' + JSON.stringify(event)
console.log(es);
cb(es, undefined);
};
}
if(!db){
waitDbReady(doGet, 100, function(){
var es = 'ERROR: Timeout: RAD IndexedDB not ready.';
console.log(es);
cb(es, undefined);
}, 10)
} else {
doGet();
}
};
var waitDbReady = function(readyFunc, checkInterval, timeoutFunc, timeoutSecs){
var startTime = new Date();
var checkFunc = function(){
if(db){
readyFunc();
} else {
if((new Date() - startTime) / 1000 >= timeoutSecs){
timeoutFunc();
} else {
setTimeout(checkFunc, checkInterval);
}
}
};
checkFunc();
try{
//if(/^((?!chrome|android).)*safari/i.test(navigator.userAgent)){ return }
setTimeout(function(){ db && db.close(); db = null }, 1000 * 15); // reset webkit bug
}catch(e){}
};
setInterval(function(){ db && db.close(); db = null; store.start() }, 1000 * 15); // reset webkit bug?
return store;
}

28
lib/rls.js Normal file
View File

@ -0,0 +1,28 @@
;(function(){
var Gun = (typeof window !== "undefined")? window.Gun : require('../gun');
Gun.on('create', function(root){
this.to.next(root);
root.opt.store = root.opt.store || Store(root.opt);
});
function Store(opt){
opt = opt || {};
opt.file = String(opt.file || 'radata');
if(Gun.TESTING){ opt.file = 'radatatest' }
var ls = localStorage;
var store = function Store(){};
store.put = function(key, data, cb){ ls[''+key] = data; cb(null, 1) }
store.get = function(key, cb){ cb(null, ls[''+key]) }
return store;
}
if(Gun.window){
Gun.window.RlocalStorage = Store;
} else {
module.exports = Store;
}
}());

2
lib/rs3.js
View File

@ -8,7 +8,7 @@ Gun.on('create', function(root){
this.to.next(root);
var opt = root.opt;
if(!process.env.AWS_S3_BUCKET){ return }
opt.batch = opt.batch || (1000 * 1);
opt.batch = opt.batch || (1000 * 10);
opt.until = opt.until || (1000 * 15);
opt.chunk = opt.chunk || (1024 * 1024 * 10); // 10MB

3
lib/store.js
View File

@ -55,8 +55,7 @@ Gun.on('create', function(root){
if((tmp = get['%']) || opt.limit){
opt.limit = (tmp <= (opt.pack || (1000 * 100)))? tmp : 1;
}
//var start = (+new Date);
//console.log("GET!", id, JSON.stringify(key));
//var start = (+new Date); // console.log("GET!", id, JSON.stringify(key));
rad(key||'', function(err, data, o){
if(data){
if(typeof data !== 'string'){

2
package.json
View File

@ -1,6 +1,6 @@
{
"name": "gun",
"version": "0.20190222.0",
"version": "0.2019.323",
"description": "A realtime, decentralized, offline-first, graph data synchronization engine.",
"main": "index.js",
"browser": "gun.js",

3
test/normalize/normalize.html
View File

@ -7,9 +7,6 @@
<script async src="../../../gun/lib/meta.js"></script>
</head>
<body>
<div>
<a href="java&#x09;script:alert(1)">ATTACK ME</a>
</div>
<div id="edit" contenteditable='true'>the world is a beautiful place.</div>
<div id="out">The world is a beautiful place.</div>
<div id="test">

6
test/panic/radisk.js
View File

@ -3,9 +3,9 @@ var config = {
port: 8765,
servers: 2,
browsers: 3,
each: 1000000,
burst: 25,
wait: 25,
each: 100000, //1000000,
burst: 1,
wait: 1,
dir: __dirname,
chunk: 1024 * 1024 * 10,
notrad: false,

2
test/panic/scale.js
View File

@ -4,7 +4,7 @@ var config = {
servers: 1,
browsers: 2,
each: 2500,
burst: 10, // do not go below 1!
burst: 1, // do not go below 1!
wait: 1,
route: {
'/': __dirname + '/index.html',

102
test/rad/browser.html Normal file
View File

@ -0,0 +1,102 @@
<html>
<body>
<h1>RindexedDB</h1>
<script src="../../../gun/gun.js"></script>
<script src="../../../gun/lib/radix.js"></script>
<script src="../../../gun/lib/radisk.js"></script>
<script src="../../../gun/lib/store.js"></script>
<script src="../../../gun/lib/rindexed.js"></script>
<!-- script src="../../../gun/lib/rls.js"></script -->
<button onclick="spam()">spam</button>
<button onclick="read()">read</button>
<input id="wait" placeholder="wait" type="number">
<input id="burst" placeholder="burst" type="number">
<div id='debug'></div>
<div id='debugg'></div>
<script>
Gun.TESTING = true;
try{localStorage.clear()}catch(e){}
indexedDB.deleteDatabase('radatatest');
var opt = {localStorage: false};
//opt.store = RindexedDB(opt);
var gun = Gun(opt);
</script>
<script>
wait.onchange = function(){ spam.wait = this.value }
burst.onchange = function(){ spam.burst = this.value }
//setTimeout(spam, 1);
function spam(){
//spam.max = 100000; spam.left = spam.max; spam.wait = 1; spam.burst = 250; spam.c = 0; spam.s = (+new Date);
//spam.max = 100000; spam.left = spam.max; spam.wait = 1; spam.burst = 100; spam.c = 0; spam.s = (+new Date);
spam.max = 1000000; spam.left = spam.max; spam.wait = 50; spam.burst = 1; spam.c = 0; spam.s = (+new Date);
//spam.max = 100; spam.left = spam.max; spam.wait = 1; spam.burst = 1; spam.c = 0; spam.s = (+new Date);
var to = setInterval(function(){ var b = spam.burst;
if(spam.c >= spam.max){ clearTimeout(to); return; }
if(!b){ b = burst = 1 }
while(b--){ go(++spam.c) }
function go(i){ var d = 0;
//loc.put(i, {test: i}, function(err, ok){ var ack = {err: err, ok: ok};
//ind.put(i, {test: i}, function(err, ok){ var ack = {err: err, ok: ok};
var ref = gun.get(i).put({test: i}, function(ack){
if(ack.err){ console.log(ack); }
if(d++){ return }
if(--spam.left){ return }
spam.end = (+new Date) - spam.s;
console.log('DONE!\n', spam.max, 'in', spam.end/1000, 'seconds\n', Math.round(spam.max / (spam.end/1000)), 'per second.');
document.body.style.backgroundColor = 'lime';
});
}
},wait);
setInterval(function(){
if(spam.end === true){ return }
if(spam.end){ spam.end = true }
var t = (+new Date) - spam.s, tmp, sec;
var status = 'saved\n'+ (tmp = (spam.max - spam.left)) +' in '+ (sec = (t/1000)) +' seconds\n'+ Math.round(tmp / sec) +' per second';
debug.innerText = status;
}, 500);
}
</script>
<script>
;(function(){
var f = 'index';
indexedDB.deleteDatabase(f);
var o = indexedDB.open(f, 1), ind = {}, db;
o.onupgradeneeded = function(eve){ (eve.target.result).createObjectStore(f) }
o.onsuccess = function(){ db = o.result }
o.onerror = function(eve){ console.log(eve||1); }
ind.put = function(key, data, cb){
if(!db){ setTimeout(function(){ ind.put(key, data, cb) },9); return }
var tx = db.transaction([f], 'readwrite');
var obj = tx.objectStore(f);
var req = obj.put(data, ''+key);
req.onsuccess = obj.onsuccess = tx.onsuccess = function(){ cb(null, 1) }
req.onabort = obj.onabort = tx.onabort = function(eve){ cb(eve||2) }
req.onerror = obj.onerror = tx.onerror = function(eve){ cb(eve||3) }
}
ind.get = function(key, cb){
if(!db){ setTimeout(function(){ ind.get(key, cb) },9); return }
var tx = db.transaction([f], 'readwrite');
var obj = tx.objectStore(f);
var req = obj.get(''+key);
req.onsuccess = function(){ cb(null, req.result) }
req.onabort = function(eve){ cb(eve||4) }
req.onerror = function(eve){ cb(eve||5) }
}
window.ind = ind;
}());
</script>
<script>
;(function(){
localStorage.clear();
var ls = localStorage, loc = {};
loc.put = function(key, data, cb){ ls[''+key] = data; cb(null, 1) }
loc.get = function(key, cb){ cb(null, ls[''+key]) }
window.loc = loc;
}());
</script>
</body>
</html>

6
test/tmp/contacts.html
View File

@ -5,6 +5,9 @@
<script src="../../../gun/gun.js"></script>
<script src="../../../gun/examples/jquery.js"></script>
<script src="../../../gun/lib/radix.js"></script>
<script src="../../../gun/lib/radisk.js"></script>
<script src="../../../gun/lib/store.js"></script>
<script src="../../../gun/lib/rindexed.js"></script>
</head>
<body>
<button id="make">make contacts</button>
@ -13,7 +16,8 @@
</ul>
<script>
try{localStorage.clear()}catch(e){}
var gun = Gun('http://localhost:8765/gun');
//var gun = Gun('http://localhost:8765/gun');
var gun = Gun({localStorage: false});
$('#make').on('click', function(){
var all = {}, to, start, cbs = 0;