RAD rindexed radisk!

gun.js

@@ -1361,7 +1361,7 @@
 			// ~who#where.where=what>when@was
 			// TODO: BUG! Put probably cannot handle plural chains!
 			var gun = this, at = (gun._), root = at.root.$, ctx = root._, tmp;
-			if(tmp = ctx.puts){
+			/*if(tmp = ctx.puts){
 				if(tmp > 100){ // without this, when synchronous, writes to a 'not found' pile up, when 'not found' resolves it recursively calls `put` which incrementally resolves each write. Stack overflow limits can be as low as 10K, so this limit is hardcoded to 1% of 10K.
 					(ctx.stack || (ctx.stack = [])).push([gun, data, cb, as]);
 					clearTimeout(ctx.puto);
@@ -1375,7 +1375,7 @@
 					return gun;
 				}
 				++ctx.puts;
-			} else { ctx.puts = 1 }
+			} else { ctx.puts = 1 }*/
 			as = as || {};
 			as.data = data;
 			as.via = as.$ = as.via || as.$ || gun;
@@ -1470,7 +1470,9 @@
 					if(!ack.lack){ this.off() } // One response is good enough for us currently. Later we may want to adjust this.
 					if(!as.ack){ return }
 					as.ack(ack, this);
+					// --C;
 				}, as.opt);
+				//C++;
 				// NOW is a hack to get synchronous replies to correctly call.
 				// and STOP is a hack to get async behavior to correctly call.
 				// neither of these are ideal, need to be fixed without hacks,
@@ -1485,6 +1487,7 @@
 			}, as);
 			if(as.res){ as.res() }
 		} function no(v,k){ if(v){ return true } }
+		//console.debug(999,1); var C = 0; setInterval(function(){ try{ debugg.innerHTML = C }catch(e){console.log(e)} }, 500);
 
 		function map(v,k,n, at){ var as = this;
 			var is = Gun.is(v);
@@ -1673,6 +1676,7 @@
 		}
 
 		function val(msg, eve, to){
+			if(!msg.$){ eve.off(); return }
 			var opt = this.as, cat = opt.at, gun = msg.$, at = gun._, data = at.put || msg.put, link, tmp;
 			if(tmp = msg.$$){
 				link = tmp = (msg.$$._);

lib/radisk.js

@@ -9,8 +9,8 @@
 		if(has){ return has }
 
 		opt.pack = opt.pack || (opt.memory? (opt.memory * 1000 * 1000) : 1399000000) * 0.3; // max_old_space_size defaults to 1400 MB.
-		opt.until = opt.until || opt.wait || 9;
+		opt.until = opt.until || opt.wait || 250;
-		opt.batch = opt.batch || 10 * 1000;
+		opt.batch = opt.batch || (10 * 1000);
 		opt.chunk = opt.chunk || (1024 * 1024 * 10); // 10MB
 		opt.code = opt.code || {};
 		opt.code.from = opt.code.from || '!';
@@ -63,7 +63,8 @@
 			r.batch(key, val);
 			if(cb){ r.batch.acks.push(cb) }
 			if(++r.batch.ed >= opt.batch){ return r.thrash() } // (2)
-			clearTimeout(r.batch.to); // (1)
+			if(r.batch.to){ return }
+			//clearTimeout(r.batch.to); // (1) // THIS LINE IS EVIL! NEVER USE IT! ALSO NEVER DELETE THIS SO WE NEVER MAKE THE SAME MISTAKE AGAIN!
 			r.batch.to = setTimeout(r.thrash, opt.until || 1);
 		}
 
@@ -82,9 +83,11 @@
 			r.batch = Radix();
 			r.batch.acks = [];
 			r.batch.ed = 0;
+			//var id = Gun.text.random(2), S = (+new Date); console.log("<<<<<<<<<<<<", id);
 			r.save(batch, function(err, ok){
 				if(++i > 1){ opt.log('RAD ERR: Radisk has callbacked multiple times, please report this as a BUG at github.com/amark/gun/issues ! ' + i); return }
 				if(err){ opt.log('err', err) }
+				//console.log(">>>>>>>>>>>>", id, ((+new Date) - S), batch.acks.length);
 				map(batch.acks, function(cb){ cb(err, ok) });
 				thrash.at = null;
 				thrash.ing = false;
@@ -285,7 +288,7 @@
 							if(opt.pack <= data.length){
 								p.err = "Chunk too big!";
 							} else {
-								data = data.toString();
+								data = data.toString(); // If it crashes, it crashes here. How!?? We check size first!
 							}
 						}catch(e){ p.err = e }
 						if(p.err){ return map(q, p.ack) }

lib/rindexed.js

@@ -10,126 +10,46 @@
     opt = opt || {};
     opt.file = String(opt.file || 'radata');
     if(Gun.TESTING){ opt.file = 'radatatest' }
-    var db = null;
+    opt.chunk = opt.chunk || (1024 * 1024); // 1MB
+    var db = null, u;
 
     try{opt.indexedDB = opt.indexedDB || indexedDB}catch(e){}
     try{if(!opt.indexedDB || 'file:' == location.protocol){
-      var store = {}, s = {}, u;
+      var store = {}, s = {};
       store.put = function(f, d, cb){ s[f] = d; cb(null, 1) };
       store.get = function(f, cb){ cb(null, s[f] || u) };
+      console.log('Warning: No indexedDB exists to persist data to!');
       return store;
     }}catch(e){}
     
-    // Initialize indexedDB. Version 1.
+    var store = function Store(){};
-    var request = opt.indexedDB.open(opt.file, 1);
+    store.start = function(){
+      var o = indexedDB.open(opt.file, 1);
+      o.onupgradeneeded = function(eve){ (eve.target.result).createObjectStore(opt.file) }
+      o.onsuccess = function(){ db = o.result }
+      o.onerror = function(eve){ console.log(eve||1); }
+    }; store.start();
 
-    // Create schema. onupgradeneeded is called only when DB is first created or when the DB version increases.
+    store.put = function(key, data, cb){
-    request.onupgradeneeded = function(event){
+      if(!db){ setTimeout(function(){ store.put(key, data, cb) },1); return }
-      var db = event.target.result;
+      var tx = db.transaction([opt.file], 'readwrite');
-      db.createObjectStore(opt.file);
+      var obj = tx.objectStore(opt.file);
+      var req = obj.put(data, ''+key);
+      req.onsuccess = obj.onsuccess = tx.onsuccess = function(){ cb(null, 1) }
+      req.onabort = obj.onabort = tx.onabort = function(eve){ cb(eve||'put.tx.abort') }
+      req.onerror = obj.onerror = tx.onerror = function(eve){ cb(eve||'put.tx.error') }
     }
 
-    // onsuccess is called when the DB is ready.
+    store.get = function(key, cb){
-    request.onsuccess = function(){
+      if(!db){ setTimeout(function(){ store.get(key, cb) },9); return }
-      db = request.result;
+      var tx = db.transaction([opt.file], 'readonly');
+      var obj = tx.objectStore(opt.file);
+      var req = obj.get(''+key);
+      req.onsuccess = function(){ cb(null, req.result) }
+      req.onabort = function(eve){ cb(eve||4) }
+      req.onerror = function(eve){ cb(eve||5) }
     }
-
+    setInterval(function(){ db && db.close(); db = null; store.start() }, 1000 * 15); // reset webkit bug?
-    request.onerror = function(event){
-      console.log('ERROR: RAD IndexedDB generic error:', event);
-    };
-
-    var store = function Store(){}, u;
-
-    store.put = function(file, data, cb){
-      cb = cb || function(){};
-      var doPut = function(){
-        // Start a transaction. The transaction will be automaticallt closed when the last success/error handler took no new action.
-        var transaction = db.transaction([opt.file], 'readwrite');
-
-        // Add or update data.
-        var radStore = transaction.objectStore(opt.file);
-        var putRequest = radStore.put(data, file);
-        putRequest.onsuccess = radStore.onsuccess = transaction.onsuccess = function(){
-          //console.log('RAD IndexedDB put transaction was succesful.');
-          cb(null, 1);
-        };
-        putRequest.onabort = radStore.onabort = transaction.onabort = function(){
-          var es = 'ERROR: RAD IndexedDB put transaction was aborted.';
-          console.log(es);
-          cb(es, undefined);
-        };
-        putRequest.onerror = radStore.onerror = transaction.onerror = function(event){
-          var es = 'ERROR: RAD IndexedDB put transaction was in error: ' + JSON.stringify(event)
-          console.log(es);
-          cb(es, undefined);
-        };
-      }
-      if(!db){
-        waitDbReady(doPut, 100, function(){
-          var es = 'ERROR: Timeout: RAD IndexedDB not ready.';
-          console.log(es);
-          cb(es, undefined);
-        }, 10)
-      } else {
-        doPut();
-      }
-    };
-
-    store.get = function(file, cb){
-      cb = cb || function(){};
-      var doGet = function(){
-        // Start a transaction. The transaction will be automaticallt closed when the last success/error handler took no new action.
-        var transaction = db.transaction([opt.file], 'readwrite');
-
-        // Read data.
-        var radStore = transaction.objectStore(opt.file);
-        var getRequest = radStore.get(file);
-        getRequest.onsuccess = function(){
-          //console.log('RAD IndexedDB get transaction was succesful.');
-          cb(null, getRequest.result);
-        };
-        getRequest.onabort = function(){
-          var es = 'ERROR: RAD IndexedDB get transaction was aborted.';
-          console.log(es);
-          cb(es, undefined);
-        };
-        getRequest.onerror = function(event){
-          var es = 'ERROR: RAD IndexedDB get transaction was in error: ' + JSON.stringify(event)
-          console.log(es);
-          cb(es, undefined);
-        };
-      }
-      if(!db){
-        waitDbReady(doGet, 100, function(){
-          var es = 'ERROR: Timeout: RAD IndexedDB not ready.';
-          console.log(es);
-          cb(es, undefined);
-        }, 10)
-      } else {
-        doGet();
-      }
-    };
-
-    var waitDbReady = function(readyFunc, checkInterval, timeoutFunc, timeoutSecs){
-      var startTime = new Date();
-      var checkFunc = function(){
-        if(db){
-          readyFunc(); 
-        } else {
-          if((new Date() - startTime) / 1000 >= timeoutSecs){
-            timeoutFunc();
-          } else {
-            setTimeout(checkFunc, checkInterval);
-          }
-        }
-      };
-      checkFunc();
-      try{
-        //if(/^((?!chrome|android).)*safari/i.test(navigator.userAgent)){ return }
-        setTimeout(function(){ db && db.close(); db = null }, 1000 * 15); // reset webkit bug
-      }catch(e){}
-    };
-
     return store;
   }
 

lib/rls.js

@@ -0,0 +1,28 @@
+;(function(){
+  var Gun = (typeof window !== "undefined")? window.Gun : require('../gun');
+
+  Gun.on('create', function(root){
+    this.to.next(root);
+    root.opt.store = root.opt.store || Store(root.opt);
+  });
+
+  function Store(opt){
+    opt = opt || {};
+    opt.file = String(opt.file || 'radata');
+    if(Gun.TESTING){ opt.file = 'radatatest' }
+    var ls = localStorage;
+
+    var store = function Store(){};
+
+    store.put = function(key, data, cb){ ls[''+key] = data; cb(null, 1) }
+    store.get = function(key, cb){ cb(null, ls[''+key]) }
+
+    return store;
+  }
+
+  if(Gun.window){
+    Gun.window.RlocalStorage = Store;
+  } else {
+    module.exports = Store;
+  }
+}());

lib/rs3.js

@@ -8,7 +8,7 @@ Gun.on('create', function(root){
 	this.to.next(root);
 	var opt = root.opt;
 	if(!process.env.AWS_S3_BUCKET){ return }
-	opt.batch = opt.batch || (1000 * 1);
+	opt.batch = opt.batch || (1000 * 10);
 	opt.until = opt.until || (1000 * 15);
 	opt.chunk = opt.chunk || (1024 * 1024 * 10); // 10MB
 

lib/store.js

@@ -55,8 +55,7 @@ Gun.on('create', function(root){
         if((tmp = get['%']) || opt.limit){
             opt.limit = (tmp <= (opt.pack || (1000 * 100)))? tmp : 1;
         }
-        //var start = (+new Date);
+        //var start = (+new Date); // console.log("GET!", id, JSON.stringify(key));
-        //console.log("GET!", id, JSON.stringify(key));
         rad(key||'', function(err, data, o){
             if(data){
                 if(typeof data !== 'string'){

package.json

@@ -1,6 +1,6 @@
 {
   "name": "gun",
-  "version": "0.20190222.0",
+  "version": "0.2019.323",
   "description": "A realtime, decentralized, offline-first, graph data synchronization engine.",
   "main": "index.js",
   "browser": "gun.js",

test/normalize/normalize.html

@@ -7,9 +7,6 @@
 		<script async src="../../../gun/lib/meta.js"></script>
 	</head>
 	<body>
-		<div>
-			<a href="java&#x09;script:alert(1)">ATTACK ME</a>
-		</div>
 		<div id="edit" contenteditable='true'>the world is a beautiful place.</div>
 		<div id="out">The world is a beautiful place.</div>
 		<div id="test">

test/panic/radisk.js

@@ -3,9 +3,9 @@ var config = {
 	port: 8765,
 	servers: 2,
 	browsers: 3,
-	each: 1000000,
+	each: 100000, //1000000,
-	burst: 25,
+	burst: 1,
-	wait: 25,
+	wait: 1,
 	dir: __dirname,
 	chunk: 1024 * 1024 * 10,
 	notrad: false,

test/panic/scale.js

@@ -4,7 +4,7 @@ var config = {
 	servers: 1,
 	browsers: 2,
 	each: 2500,
-	burst: 10, // do not go below 1!
+	burst: 1, // do not go below 1!
 	wait: 1,
 	route: {
 		'/': __dirname + '/index.html',

test/rad/browser.html

@@ -0,0 +1,102 @@
+<html>
+<body>
+<h1>RindexedDB</h1>
+
+<script src="../../../gun/gun.js"></script>
+<script src="../../../gun/lib/radix.js"></script>
+<script src="../../../gun/lib/radisk.js"></script>
+<script src="../../../gun/lib/store.js"></script>
+<script src="../../../gun/lib/rindexed.js"></script>
+<!-- script src="../../../gun/lib/rls.js"></script -->
+
+<button onclick="spam()">spam</button>
+<button onclick="read()">read</button>
+<input id="wait" placeholder="wait" type="number">
+<input id="burst" placeholder="burst" type="number">
+<div id='debug'></div>
+<div id='debugg'></div>
+
+<script>
+Gun.TESTING = true;
+try{localStorage.clear()}catch(e){}
+indexedDB.deleteDatabase('radatatest');
+
+var opt = {localStorage: false};
+//opt.store = RindexedDB(opt);
+var gun = Gun(opt);
+</script>
+<script>
+wait.onchange = function(){ spam.wait = this.value }
+burst.onchange = function(){ spam.burst = this.value }
+//setTimeout(spam, 1);
+function spam(){
+	//spam.max = 100000; spam.left = spam.max; spam.wait = 1; spam.burst = 250; spam.c = 0; spam.s = (+new Date);
+	//spam.max = 100000; spam.left = spam.max; spam.wait = 1; spam.burst = 100; spam.c = 0; spam.s = (+new Date);
+	spam.max = 1000000; spam.left = spam.max; spam.wait = 50; spam.burst = 1; spam.c = 0; spam.s = (+new Date);
+	//spam.max = 100; spam.left = spam.max; spam.wait = 1; spam.burst = 1; spam.c = 0; spam.s = (+new Date);
+	var to = setInterval(function(){ var b = spam.burst;
+		if(spam.c >= spam.max){ clearTimeout(to); return; }
+		if(!b){ b = burst = 1 }
+		while(b--){ go(++spam.c) }
+		function go(i){ var d = 0;
+			//loc.put(i, {test: i}, function(err, ok){ var ack = {err: err, ok: ok};
+			//ind.put(i, {test: i}, function(err, ok){ var ack = {err: err, ok: ok};
+			var ref = gun.get(i).put({test: i}, function(ack){
+				if(ack.err){ console.log(ack); }
+				if(d++){ return }
+				if(--spam.left){ return }
+				spam.end = (+new Date) - spam.s;
+				console.log('DONE!\n', spam.max, 'in', spam.end/1000, 'seconds\n', Math.round(spam.max / (spam.end/1000)), 'per second.');
+				document.body.style.backgroundColor = 'lime';
+			});
+		}
+	},wait);
+	setInterval(function(){
+		if(spam.end === true){ return }
+		if(spam.end){ spam.end = true }
+		var t = (+new Date) - spam.s, tmp, sec;
+		var status = 'saved\n'+ (tmp = (spam.max - spam.left)) +' in '+ (sec = (t/1000)) +' seconds\n'+ Math.round(tmp / sec) +' per second';
+		debug.innerText = status; 
+	}, 500);
+}
+</script>
+<script>
+;(function(){
+var f = 'index';
+indexedDB.deleteDatabase(f);
+var o = indexedDB.open(f, 1), ind = {}, db;
+o.onupgradeneeded = function(eve){ (eve.target.result).createObjectStore(f) }
+o.onsuccess = function(){ db = o.result }
+o.onerror = function(eve){ console.log(eve||1); }
+ind.put = function(key, data, cb){
+	if(!db){ setTimeout(function(){ ind.put(key, data, cb) },9); return }
+	var tx = db.transaction([f], 'readwrite');
+	var obj = tx.objectStore(f);
+	var req = obj.put(data, ''+key);
+	req.onsuccess = obj.onsuccess = tx.onsuccess = function(){ cb(null, 1) }
+	req.onabort = obj.onabort = tx.onabort = function(eve){ cb(eve||2) }
+	req.onerror = obj.onerror = tx.onerror = function(eve){ cb(eve||3) }
+}
+ind.get = function(key, cb){
+	if(!db){ setTimeout(function(){ ind.get(key, cb) },9); return }
+	var tx = db.transaction([f], 'readwrite');
+	var obj = tx.objectStore(f);
+	var req = obj.get(''+key);
+	req.onsuccess = function(){ cb(null, req.result) }
+	req.onabort = function(eve){ cb(eve||4) }
+	req.onerror = function(eve){ cb(eve||5) }
+}
+window.ind = ind;
+}());
+</script>
+<script>
+;(function(){
+localStorage.clear();
+var ls = localStorage, loc = {};
+loc.put = function(key, data, cb){ ls[''+key] = data; cb(null, 1) }
+loc.get = function(key, cb){ cb(null, ls[''+key]) }
+window.loc = loc;
+}());
+</script>
+</body>
+</html>

test/tmp/contacts.html

@@ -5,6 +5,9 @@
 		<script src="../../../gun/gun.js"></script>
 		<script src="../../../gun/examples/jquery.js"></script>
 		<script src="../../../gun/lib/radix.js"></script>
+		<script src="../../../gun/lib/radisk.js"></script>
+		<script src="../../../gun/lib/store.js"></script>
+		<script src="../../../gun/lib/rindexed.js"></script>
 	</head>
 	<body>
 		<button id="make">make contacts</button>
@@ -13,7 +16,8 @@
 		</ul>
 		<script>
 			try{localStorage.clear()}catch(e){}
-			var gun = Gun('http://localhost:8765/gun');
+			//var gun = Gun('http://localhost:8765/gun');
+			var gun = Gun({localStorage: false});
 
 			$('#make').on('click', function(){
 				var all = {}, to, start, cbs = 0;