s3 fix & begin adding load tests

e2e/distributed.js

@@ -3,7 +3,7 @@ require('./holy/grail');
 describe('PANIC!', function(){
 	this.timeout(1000 * 100);
 
-	var Gun = require('..');
+	var Gun = require('../');
 	var gun = Gun();
 
 	var panic = require('panic-server');

gun.js

@@ -1,4 +1,3 @@
-//console.log("!!!!!!!!!!!!!!!! WARNING THIS IS GUN 0.5 !!!!!!!!!!!!!!!!!!!!!!");
 ;(function(){
 
 	/* UNBUILD */

lib/s3.js

@@ -72,15 +72,17 @@
 			if(!opt.throttle){
 				return now();
 			}
-			clearTimeout(s3.wait);
 			s3.wait = s3.wait || setTimeout(now, opt.throttle * 1000); // in seconds
 		});
 		function now(){
 			clearTimeout(s3.wait);
+			var keep = batch;
+			batch = {};
 			s3.batching = 0;
 			var now = s3.next;
 			s3.next = Gun.time.is();
-			Gun.obj.map(batch, function put(exists, soul, count){
+			s3.wait = null;
+			Gun.obj.map(keep, function put(exists, soul, count){
 				//console.log("s3ving...", soul);
 				var node = root._.graph[soul] || async[soul]; // the batch does not actually have the nodes, but what happens when we do cold data? Could this be gone?
 				s3.PUT(opt.prefix+soul, node, function(err, reply){

lib/uws.js

@@ -0,0 +1,92 @@
+var Gun = require('gun/gun');
+
+var WebSocket = require('uws');
+
+var url = require('url');
+
+var con;
+
+Gun.on('opt', function(at){
+	this.to.next(at);
+	if(at.once){ return }
+	var opt = at.opt.uws || at.opt.ws || (at.opt.uws = {});
+	var cat = at.gun.back(-1)._;
+
+	opt.server = new WebSocket.Server(opt || {port: 8080});
+	var peers = cat.opt.peers;
+
+	console.log("????", opt.server);
+	opt.server.on('connection', function(ws){
+		ws.upgradeReq = ws.upgradeReq || {};
+		ws.url = url.parse(ws.upgradeReq.url||'', true);
+		ws.id = ws.id || Gun.text.random(6);
+		peers[ws.id] = {wire: ws};
+		ws.on('message', function(msg){
+			console.log("MESSAGE", msg);
+			receive(msg, ws, cat);
+		});
+		ws.on('close', function(){
+			Gun.obj.del(peers, ws.id);
+		});
+	});
+});
+
+function receive(msg, wire, cat){
+	if(!cat){ return }
+	try{msg = JSON.parse(msg);
+		msg.url = wire.url;
+	}catch(e){}
+	cat.gun.on('in', msg.body || msg);
+}
+
+var message;
+Gun.on('out', function(at){
+	this.to.next(at);
+	var cat = at.gun._.root._;
+	message = JSON.stringify({body: at, headers: {}});
+	Gun.obj.map(cat.opt.peers, send, cat);
+});
+
+function send(peer){
+	var msg = message, cat = this;
+	var wire = peer.wire || open(peer, cat);
+	if(!wire){ return }
+	if(wire.readyState === wire.OPEN){
+		wire.send(msg);
+		return;
+	}
+	(peer.queue = peer.queue || []).push(msg);
+}
+
+function open(peer, as){
+	if(!peer || !peer.url){ return }
+	var url = peer.url.replace('http', 'ws');
+	var wire = peer.wire = new WebSocket(url);
+	wire.on('close', function(){
+		reconnect(peer, as);
+	});
+	wire.on('error', function(error){
+		if(error.code === 'ECONNREFUSED'){
+			reconnect(peer, as);
+		}
+	});
+	wire.on('open', function(){
+		var queue = peer.queue;
+		peer.queue = [];
+		Gun.obj.map(queue, function(msg){
+			message = msg;
+			send.call(as, peer);
+		});
+	});
+	wire.on('message', function(msg){
+		receive(msg, wire, as);
+	});
+	return wire;
+}
+
+function reconnect(peer, as){
+	clearTimeout(peer.defer);
+	peer.defer = setTimeout(function(){
+		open(peer, as);
+	}, 2 * 1000);
+}

package.json

@@ -1,6 +1,6 @@
 {
   "name": "gun",
-  "version": "0.6.1",
+  "version": "0.6.2",
   "description": "Graph engine",
   "main": "index.js",
   "browser": "gun.min.js",
@@ -55,6 +55,9 @@
   "devDependencies": {
     "express": "~>4.13.4",
     "mocha": "~>1.9.0",
-    "uglify-js": "~>2.2.0"
+    "uglify-js": "~>2.2.0",
+    "panic-manager": "^1.2.0",
+    "panic-server": "^1.1.0",
+    "ip": "^1.1.4"
   }
 }

test/common.js

@@ -3438,6 +3438,52 @@ describe('Gun', function(){
 			},300);
 		});
 		return;
+		it.only('Custom extensions are chainable', function(done){
+			Gun.chain.filter = function(filter){
+			  var chain = this.chain();
+			  var context = this;
+			  var _tags;
+			  context.val(function(obj, key){
+			    if(!obj.tags){
+			      console.warn('Not tagged to anything!');
+			      context._.valid = false;
+			      chain._.on('in', {get: key, gun: this});
+			      return false;
+			    } else { 
+			     _tags = Gun.obj.ify(obj.tags);
+			      if(Array.isArray(filter)){
+			        context._.valid = filter.every(function(f){ return ( _tags[f] && _tags[f]==1) });
+			        if(context._.valid){
+			          chain._.on('in', {get: key, put: obj, gun: this});
+			          return context;
+			        } else {
+			          console.log("that was wrong");
+			          chain._.on('in', {get: key, put: undefined, gun: this});
+			        }
+			        return false;
+			      } else {
+			        console.warn('filter should be an Array');
+			        return false;
+			      }
+			    }
+			  });
+			  return chain;
+			}
+
+			var gun = Gun();
+
+			var fake1 = gun.get('fake1').put({name:'faker1',tags:JSON.stringify({a:1,b:0,c:1})});
+			var fake2 = gun.get('fake2').put({name:'faker2',tags:JSON.stringify({a:1,b:1,c:1})});
+			var list = gun.get('list');
+			list.set(fake1);
+			list.set(fake2);
+
+			gun.get('fake1')//.map()
+			      .filter(['a','b'])  // Gun.chain.filter = function(tags){ .... }
+			      .get(function(no){console.log("NO!", no)})
+			      .val(function(yes){console.log("YES!", yes)})
+		});
+
 		it.only('Make sure circular contexts are not copied', function(done){
 			/* let's define an appropriate deep default database... */
 			var dfltSansUsers = { 1: { name : "org1", sites : { 1: {name : "site1"} } } };

test/panic/index.html

@@ -0,0 +1,7 @@
+<script src='panic.js'></script>
+<script>panic.server(location.origin)</script>
+<script src='gun.js'></script>
+<script src='jquery.js'></script>
+<h1>Running Tests.</h1>
+<div id="log"></div>
+<!-- <textarea id="print" style="width: 100%; height: 90%; border: 0;"></textarea> -->

test/panic/load.js

@@ -0,0 +1,271 @@
+var config = {
+	IP: require('ip').address(),
+	port: 8080,
+	servers: 1,
+	browsers: 2,
+	each: 500,
+	wait: 1,
+	route: {
+		'/': __dirname + '/index.html',
+		'/gun.js': __dirname + '/../../gun.js',
+		'/jquery.js': __dirname + '/../../examples/jquery.js'
+	}
+}
+
+/*
+	Welcome, person!
+	You have found the test that causes gun to PANIC with load!
+	Above are options to configure, the only ones useful are:
+	 - browsers // number of browsers you want to load test across.
+	 - each // the number of messages each browser should sync.
+	This test is less than 200 lines of code (without comments)!
+	However, if you aren't familiar with PANIC - you are in for a surprise!
+	I'm Plublious, and I shall be your guide!
+*/
+
+// First we need to create a PANIC server.
+// Each device/browser in the distributed system we are testing connects to it.
+// It then coordinates these clients to cause chaos in the distributed system.
+// Cool huh?
+var panic = require('panic-server');
+panic.server().on('request', function(req, res){ // Static server
+	config.route[req.url] && require('fs').createReadStream(config.route[req.url]).pipe(res);
+}).listen(config.port); // Start panic server.
+
+// In order to tell the clients what to do,
+// We need a way to reference all of them.
+var clients = panic.clients;
+
+// Some of the clients may be NodeJS servers on different machines.
+// PANIC manager is a nifty tool that lets us remotely spawn them.
+var manager = require('panic-manager')();
+manager.start({
+    clients: Array(config.servers).fill().map(function(u, i){ // Create a bunch of servers.
+			return {
+				type: 'node',
+				port: config.port + (i + 1) // They'll need unique ports to start their servers on, if we run the test on 1 machine.
+			}
+    }),
+    panic: 'http://' + config.IP + ':' + config.port // Auto-connect to our panic server.
+});
+
+// Now lets divide our clients into "servers" and "browsers".
+var servers = clients.filter('Node.js');
+var browsers = clients.excluding(servers);
+
+// Sweet! Now we can start the tests.
+// PANIC works with Mocha and other testing libraries!
+// So it is easy to use PANIC.
+
+describe("Load test "+ config.browsers +" browser(s) across "+ config.servers +" server(s)!", function(){
+
+	// We'll have to manually launch the browsers,
+	// So lets up the timeout so we have time to do that.
+	this.timeout(5 * 60 * 1000);
+
+	it("Servers have joined!", function(){
+		// Alright, lets wait until enough gun server peers are connected.
+		return servers.atLeast(config.servers);
+	});
+
+	it("GUN has spawned!", function(){
+		// Once they are, we need to actually spin up the gun server.
+		var tests = [], i = 0;
+		servers.each(function(client){
+			// for each server peer, tell it to run this code:
+			tests.push(client.run(function(test){
+				// NOTE: Despite the fact this LOOKS like we're in a closure...
+				// it is not! This code is actually getting run
+				// in a DIFFERENT machine or process!
+				var env = test.props;
+				// As a result, we have to manually pass it scope.
+				test.async();
+				// Clean up from previous test.
+				try{ require('fs').unlinkSync(env.i+'data.json') }catch(e){}
+				var server = require('http').createServer(function(req, res){
+					res.end("I am "+ env.i +"!");
+				});
+				// Launch the server and start gun!
+				var Gun = require('gun');
+				var gun = Gun({file: env.i+'data.json'});
+				// Attach the server to gun.
+				//gun.wsp(server);
+				server.listen(env.config.port + env.i, function(){
+					// This server peer is now done with the test!
+					// It has successfully launched.
+					test.done();
+				});
+			}, {i: i += 1, config: config})); 
+		});
+		// NOW, this is very important:
+		// Do not proceed to the next test until
+		// every single server (in different machines/processes)
+		// have ALL successfully launched.
+		return Promise.all(tests);
+	});
+
+	it(config.browsers +" browser(s) have joined!", function(){
+		// Okay! Cool. Now we can move on to the next step...
+		console.log("PLEASE OPEN http://"+ config.IP +":"+ config.port +" IN "+ config.browsers +" BROWSER(S)!");
+		// Which is to manually open up a bunch of browser tabs
+		// and connect to the PANIC server in the same way
+		// the NodeJS servers did.
+
+		// However! We're gonna cheat...
+		browsers.atLeast(1).then(function(){
+			// When there is at least one browser opened, tell it to run this code:
+			browsers.run(function(test){
+				// NOTE: This closure is now being run IN THE BROWSER.
+				// This code is not server side code, despite the fact
+				// that we've written it on the server. It is not.
+				// Mind blowing, right?
+				var env = test.props;
+				// Like before, we had to manually pass it some scope.
+				$('body').prepend("<button onclick='allopen()'>Open All Browsers</button>");
+				// All right, lets cheat by making a button
+				// that will automatically open all the
+				// remaining browser tabs for us
+				// so we don't have to do it manually.
+				window.allopen = function(i){
+					if(env.config.browsers <= i){ return }
+					i = i || 1;
+					var win = window.open(location, '_blank');
+					win.focus();
+					setTimeout(function(){allopen(i+1)},0);
+				}
+			}, {config: config});
+		});
+		// Cool! Once that is done...
+		// WAIT until all those browser tabs
+		// have connected to the PANIC server
+		// THEN move onto the next step
+		// where we will cause chaos!
+		return browsers.atLeast(config.browsers);
+	});
+
+	it("Data was saved and synced across all browsers!", function(){
+		// This is where it gets good!
+		var tests = [], ids = {}, i = 0;
+		// Let us create a list of all the browsers IDs connected.
+		// This will later let each browser check against every other browser.
+		browsers.each(function(client, id){
+			ids[id] = 1;
+		});
+		browsers.each(function(client, id){
+			// for every browser, run the following code:
+			tests.push(client.run(function(test){
+				localStorage.clear(); // Clean up anything from before.
+				var env = test.props;
+				// Get access to the "outer scope" which has the browser IDs
+				// as well as other configuration information.
+				test.async();
+				// Now we want to connect to every gun server peer...
+				var peers = [], i = env.config.servers;
+				while(i--){
+					// For the total number of servers listed in the configuration
+					// Add their URL into an array.
+					peers.push('http://'+ env.config.IP + ':' + (env.config.port + (i + 1)));
+				}
+				// Pass all the servers we want to connect to into gun.
+				var gun = Gun(peers);
+				// Now we want to create a list
+				// of all the messages that WILL be sent
+				// according to the expected configuration.
+				// This is equal to...
+				var num = 0, total = 0, check = Gun.obj.map(env.ids, function(v,id,t){
+					// for each browser ID
+					// they will be saving X number of messages each.
+					var i = env.config.each;
+					while(i--){
+						// So add a deterministic key we can check against.
+						t(id + (i + 1), 1);
+						// And count up the total number of messages we expect for all.
+						total += 1;
+					}
+				});
+				// Note, this `check` hash table now looks something like this:
+				// {alice1: 1, alice2: 1, alice3: 1, bob1: 1, bob2: 1, bob3: 1}
+				var report = $("<div>").css({position: 'fixed', top: 0, right: 0, background: 'white', padding: 10}).text(num +" / "+ total +" Verified").prependTo('body');
+				// Add a nifty UI that tells us how many messages have been verified.
+				// FINALLY, tell gun to subscribe to every record
+				// that is is/will be saved to this table.
+				gun.get('test').map().on(function(data, key){
+					// When we get realtime updates to the data,
+					// create or reuse a DIV that we
+					//var el = $('#'+key).length ? $('#'+key) : $('<div>');
+					// log the data out to, so we can visually see our test.
+					//$(log).append(el.attr('id', key).text(key +": "+ data));
+					// Now, make sure the received data
+					// matches exactly the data we EXPECT
+					if(("Hello world, "+key+"!") === data){
+						// if it does, we can "check off" record
+						// from our verify todo list.
+						check[key] = 0;
+						// Bump the total number of verified items and update the UI.
+						num += 1;
+						report.text(num +" / "+ total +" Verified");
+					}
+					// Scroll down with the logging.
+					//$('body').stop(true).animate({scrollTop: $(log).height()});
+					// This next part is important:
+					if(Gun.obj.map(check, function(still){
+						// IF THERE ARE ANY RECORDS STILL LEFT TO BE VERIFIED
+						if(still){ return true }
+					})){ return } // return, PREVENTING the test from being finished.
+					// IF HOWEVER, every single message
+					// that we EXPECTED to see
+					// has now been seen
+					// then THIS ONE BROWSER PEER (of many peers)
+					// is finally done.
+					test.done();
+				});
+				// But we have to actually tell the browser to save data!
+				var i = 0, to = setTimeout(function go(){
+					// Cool, make a recursive function
+					// that keeps going until we've saved each message.
+					if(env.config.each <= i){
+						clearTimeout(to);
+						return;
+					}
+					to = setTimeout(go, env.config.wait * Math.random()); // add a little jitter.
+					i += 1;
+					var p = env.id + i;
+					// And actually save the data with gun,
+					// as a record added to one big 'test' table.
+					gun.get('test').get(p).put('Hello world, '+ p +'!');
+				}, env.config.wait);
+			}, {i: i += 1, id: id, ids: ids, config: config})); 
+		});
+		// YAY! We're finally done.
+		// IF AND ONLY IF
+		// EVERY SINGLE BROWSER
+		// HAS VERIFIED
+		// EVERY OTHER BROWSERS' data.
+		// If they are ALL done, go to the next step.
+		return Promise.all(tests);
+	});
+
+	after("Everything shut down.", function(){
+		// which is to shut down all the browsers.
+		browsers.run(function(){
+			setTimeout(function(){
+				location.reload();
+			}, 15 * 1000);
+		});
+		// And shut down all the servers.
+		return servers.run(function(){
+			process.exit();
+		});
+	});
+})
+// THE END!
+// Congrats, wasn't that epic?
+// Or still confused how a single 200 LOC test file
+// Is running correctness verification tests
+// across an entire distributed system of devices/browsers?
+// Well, jump on https://gitter.im/amark/gun !
+
+// Think adding tests like this to your work place would be bomb awesome?
+// We totally sell PANIC training, licenses, and support!
+// Please reach out to hi@gunDB.io if you are interested
+// in purchasing consulting or services for PANIC.