Mirroristas/gun
2
0
Fork 0
mirror of https://github.com/amark/gun.git synced 2025-06-03 20:56:43 +00:00
Code Issues Packages Projects Releases Wiki Activity

load testing!

Browse Source
This commit is contained in:
Mark Nadal 2017-03-11 07:03:55 -08:00
parent a700afba73
commit ec3288e583
13 changed files with 760 additions and 535 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

1
.gitignore vendored
View File

@ -2,6 +2,7 @@ node_modules
npm-debug.log
yarn.lock
*data.json
*data*
*.db
.idea/
*.bak

23
examples/http.js
View File

@ -1,8 +1,21 @@
var port = process.env.OPENSHIFT_NODEJS_PORT || process.env.VCAP_APP_PORT || process.env.PORT || process.argv[2] || 8080;
var Gun = require('../');
var server = require('http').createServer(function(req, res){
if(Gun.serve(req, res)){ return } // filters gun requests!
require('fs').createReadStream(require('path').join(__dirname, req.url)).on('error',function(){ // static files!
res.writeHead(200, {'Content-Type': 'text/html'});
res.end(require('fs')
.readFileSync(require('path')
.join(__dirname, 'index.html') // or default to index
));
}).pipe(res); // stream
});
var gun = Gun({
file: 'data.json',
web: server,
s3: {
key: '', // AWS Access Key
secret: '', // AWS Secret Token
@ -10,16 +23,6 @@ var gun = Gun({
}
});
var server = require('http').createServer(function(req, res){
if(gun.wsp.server(req, res)){
return; // filters gun requests!
}
require('fs').createReadStream(require('path').join(__dirname, req.url)).on('error',function(){ // static files!
res.writeHead(200, {'Content-Type': 'text/html'});
res.end(require('fs').readFileSync(require('path').join(__dirname, 'index.html'))); // or default to index
}).pipe(res); // stream
});
gun.wsp(server);
server.listen(port);
console.log('Server started on port ' + port + ' with /gun');

852
gun.js
View File

File diff suppressed because it is too large Load Diff

57
lib/file.js
View File

@ -5,30 +5,54 @@
var Gun = require('../gun'),
fs = require('fs');
var files = {};
Gun.on('put', function(at){
//console.log("file write", Gun.obj.copy(at), at.gun.back(-1)._.opt.file);
this.to.next(at);
var fileOpt = at.gun.back('opt._file')
if(!fileOpt.use){ return }
var root = at.gun.back(-1);
var f = at.gun.back('opt._file')
if(!f.use){ return }
var graph = at.put, opt = at.opt || {};
var Graph = fileOpt.gun._.graph
var Graph = f.gun._.graph
Gun.obj.map(graph, function(node, soul){
fileOpt.disk.graph[soul] = Graph[soul] || graph[soul];
f.disk.graph[soul] = Graph[soul] || graph[soul];
});
graph = JSON.stringify(fileOpt.disk, null, 2);
// TODO: Allow for a `fs.writeFile` compatible module, that is more reliable/safe, to be passed in through the options.
fs.writeFile(opt.file || fileOpt.file, graph, function(err){
fileOpt.gun.on('in', {
'@': at['#'],
ok: err? undefined : 1,
err: err
f.count = (f.count || 0) + 1;
if(!at['@']){ // don't ack other acks!
(f.check || (f.check = {}))[at['#']] = root;
}
function save(){
clearTimeout(f.wait);
var ack = f.check;
f.count = 0;
f.wait = false;
f.check = {};
graph = JSON.stringify(f.disk, null, 2);
// TODO: Allow for a `fs.writeFile` compatible module, that is more reliable/safe, to be passed in through the options.
fs.writeFile(opt.file || f.file, graph, function(err){
Gun.obj.map(ack, function(root, id){
root.on('in', {
'@': id,
ok: err? undefined : 1,
err: err || undefined
});
});
});
});
}
if(f.count >= 10000){ // goal is to do 10K inserts/second.
return save();
}
if(f.wait){ return }
clearTimeout(f.wait);
f.wait = setTimeout(save, 1000);
});
Gun.on('get', function(at){
var fileOpt = at.gun.back('opt._file');
//if(at.cap && fileOpt.use){ at.cap-- }
this.to.next(at);
var fileOpt = at.gun.back('opt._file')
if(!fileOpt.use){ return }
var opt = at.opt || {};
var soul = at.get['#'];
@ -39,7 +63,8 @@ Gun.on('get', function(at){
}
fileOpt.gun.on('in', {
put: Gun.graph.node(node),
'@': at['#']
'@': at['#'],
how: 'file'
})
});
@ -75,7 +100,7 @@ Gun.on('opt', function(at){
opt._file.use = true;
opt._file.file = String(opt.file || opt._file.file || 'data.json');
opt._file.raw = opt._file.raw || ((fs.existsSync || require('path').existsSync)(opt._file.file) ? fs.readFileSync(opt._file.file).toString() : null);
opt._file.disk = opt._file.disk || Gun.obj.ify(opt._file.raw || {graph: {}});
opt._file.disk = files[opt._file.file] = files[opt._file.file] || opt._file.disk || Gun.obj.ify(opt._file.raw || {graph: {}});
opt._file.disk.graph = opt._file.disk.graph || {};
opt._file.gun = gun;
});

3
lib/s3.js
View File

@ -1,4 +1,7 @@
;(function(){
if(!process.env.AWS_S3_BUCKET){ return }
var Gun = require('../gun');
var S3 = require('./aws');

11
lib/serve.js Normal file
View File

@ -0,0 +1,11 @@
module.exports = function serve(req, res, next){
if(!req || !res){ return false }
next = next || serve;
if(!req.url){ return next() }
if(0 <= req.url.indexOf('gun.js')){
res.writeHead(200, {'Content-Type': 'text/javascript'});
res.end(serve.js = serve.js || require('fs').readFileSync(__dirname + '/../gun.js'));
return true;
}
return next();
}

4
lib/server.js
View File

@ -1,7 +1,9 @@
;(function(){
var Gun = require('../gun');
Gun.serve = require('./serve');
require('./s3');
require('./wsp/server');
require('./uws');
//require('./wsp/server');
require('./file');
module.exports = Gun;
}());

60
lib/uws.js
View File

@ -4,25 +4,29 @@ var WebSocket = require('uws');
var url = require('url');
var con;
console.log("Experimental high performance uWS server is being used.");
Gun.on('opt', function(at){
this.to.next(at);
if(at.once){ return }
var opt = at.opt.uws || at.opt.ws || (at.opt.uws = {});
var cat = at.gun.back(-1)._;
var cat = (at.gun.back(-1)._);
opt.server = new WebSocket.Server(opt || {port: 8080});
opt.server = opt.server || at.opt.web;
if(opt.server && opt.server.use){ // if ExpressJS
opt.server.use(Gun.serve);
}
opt.web = new WebSocket.Server(opt);
var peers = cat.opt.peers;
console.log("????", opt.server);
opt.server.on('connection', function(ws){
opt.web.on('connection', function(ws){
ws.upgradeReq = ws.upgradeReq || {};
ws.url = url.parse(ws.upgradeReq.url||'', true);
ws.id = ws.id || Gun.text.random(6);
peers[ws.id] = {wire: ws};
ws.on('message', function(msg){
console.log("MESSAGE", msg);
//console.log("MESSAGE", msg);
receive(msg, ws, cat);
});
ws.on('close', function(){
@ -31,22 +35,47 @@ Gun.on('opt', function(at){
});
});
function receive(msg, wire, cat){
if(!cat){ return }
try{msg = JSON.parse(msg);
msg.url = wire.url;
}catch(e){}
cat.gun.on('in', msg.body || msg);
}
var message;
Gun.on('out', function(at){
this.to.next(at);
var cat = at.gun._.root._;
message = JSON.stringify({body: at, headers: {}});
message = JSON.stringify(at);
if(cat.udrain){
cat.udrain.push(message);
return;
}
cat.udrain = [];
setTimeout(function(){
if(!cat.udrain){ return }
//if(count += cat.udrain.length){ console.log("msg out:", count) }
var tmp = cat.udrain;
cat.udrain = null;
message = JSON.stringify(tmp);
Gun.obj.map(cat.opt.peers, send, cat);
},1);
Gun.obj.map(cat.opt.peers, send, cat);
});
var count = 0;
function receive(msg, wire, cat){
if(!cat){ return }
try{msg = JSON.parse(msg);
}catch(e){}
if(msg instanceof Array){
var i = 0, m; while(m = msg[i++]){
receive(m, wire, cat);
}
return;
}
//if(++count){ console.log("msg in:", count) }
//msg.url = wire.url;
cat.gun.on('in', msg.body || msg);
}
// EVERY message taken care of. The "extra" ones are from in-memory not having "asked" for it yet - which we won't want it to do for foreign requests. Likewise, lots of chattyness because the put/ack replies happen before the `get` syncs so everybody now has it in-memory already to reply with.
function send(peer){
var msg = message, cat = this;
var wire = peer.wire || open(peer, cat);
@ -66,6 +95,7 @@ function open(peer, as){
reconnect(peer, as);
});
wire.on('error', function(error){
if(!error){ return }
if(error.code === 'ECONNREFUSED'){
reconnect(peer, as);
}

13
lib/wsp/server.js
View File

@ -16,18 +16,26 @@ Gun.on('opt', function (at) {
gun.__ = at.root._;
gun.__.opt.ws = opt.ws = gun.__.opt.ws || opt.ws || {};
if(gun.__.opt.web){
setTimeout(function(){
if(gun.__.opt.web.use){
gun.__.opt.web.use(Gun.serve);
}
start(gun.__.opt.web);
},1);
}
function start (server, port, app) {
if (app && app.use) {
app.use(gun.wsp.server);
}
server = gun.__.opt.ws.server = gun.__.opt.ws.server || opt.ws.server || server;
server = gun.__.opt.ws.server = gun.__.opt.ws.server || gun.__.opt.web || opt.ws.server || server;
if (!gun.wsp.ws) {
//console.log("????????", gun.__.opt.ws);
gun.wsp.ws = new WSS(gun.__.opt.ws);
attach(gun, gun.wsp.ws);
}
gun.wsp.ws = gun.wsp.ws || new WSS(gun.__.opt.ws);
require('./ws')(gun.wsp.ws, function (req, res) {
var ws = this;
req.headers['gun-sid'] = ws.sid = ws.sid ? ws.sid : req.headers['gun-sid'];
@ -45,6 +53,7 @@ Gun.on('opt', function (at) {
gun.__.opt.ws.port = gun.__.opt.ws.port || opt.ws.port || port || 80;
}
var wsp = gun.wsp = gun.wsp || function (server) {
console.log("WARNING: gun.wsp(server) should be switched to Gun({web: server}) by v0.7!")
if (!server) { return gun; }
if (Gun.fns.is(server.address)) {
if (server.address()) {

4
package.json
View File

@ -1,6 +1,6 @@
{
"name": "gun",
"version": "0.6.3",
"version": "0.6.4",
"description": "Graph engine",
"main": "index.js",
"browser": "gun.min.js",
@ -50,7 +50,7 @@
"dependencies": {
"aws-sdk": "~>2.15.0",
"formidable": "~>1.0.15",
"ws": "~>1.0.1"
"uws": "~>0.13.0"
},
"devDependencies": {
"express": "~>4.13.4",

67
test/common.js
View File

@ -1755,6 +1755,7 @@ describe('Gun', function(){
//expect(count['Alice']).to.be(1);
//expect(count['Bob']).to.be(1);
//expect(count['Alice Zzxyz']).to.be(1);
if(done.c){ return } done.c = 1;
done();
},200);
}
@ -3132,6 +3133,7 @@ describe('Gun', function(){
gun.path('alias').get(function(at){
done.alias = done.alias || at.put.mark;
}).path('mark').get(function(at){
//console.log("************", at.put);
setTimeout(function(){
done.mark = done.mark || at.put.pub;
expect(Gun.val.rel.is(done.mark)).to.be('pub');
@ -3159,7 +3161,6 @@ describe('Gun', function(){
setTimeout(function(){
var gun2 = Gun();
gun2.get('stef').path('address').val(function(data){ // Object {_: Object, country: "Netherlands", zip: "1766KP"} "adress"
//console.log("******", data);
done.a = true;
expect(data.country).to.be('Netherlands');
expect(data.zip).to.be('999999');
@ -3168,7 +3169,6 @@ describe('Gun', function(){
done();
});
gun2.get('stef').val(function(data){ //Object {_: Object, address: Object} "stef"
//console.log("*****************", data);
done.s = true;
expect(data.name).to.be('Stef');
expect(data.address).to.be.ok();
@ -3328,8 +3328,8 @@ describe('Gun', function(){
var app = gun.get('mult/times');
app.path('alias').path('mark').set(gun.get('asdf').put({
pub: 'asdf',
app.path('alias').path('mark').set(gun.get('ASDF').put({
pub: 'ASDF',
alias: 'mark',
born: 1
}));
@ -3343,7 +3343,7 @@ describe('Gun', function(){
app.path('alias').map().map().path('alias').on(function(data){
done.two = data;
//console.log("alias 2!", data);
expect(done.one).to.be("asdf");
expect(done.one).to.be("ASDF");
expect(done.two).to.be("mark");
if(done.c){ return } done.c = 1;
done();
@ -3358,7 +3358,7 @@ describe('Gun', function(){
Gun.on('put', {gun: gun, put: Gun.graph.ify({
alias: {
mark: {
pub: {_:{'#':'pub'},
pub: {_:{'#':'PUB'},
pub: 'asdf',
alias: 'mark',
born: 1
@ -3386,6 +3386,7 @@ describe('Gun', function(){
});
it('map with map function', function(done){
console.debug.i=0;
var gun = Gun(), s = 'map/mapfunc', u;
var app = gun.get(s);
var list = app.get('list');
@ -3399,8 +3400,7 @@ describe('Gun', function(){
done();
}
});
list.set({name: 'alice', age: 27});
list.set({name: 'alice', age: 27}); // on put, table-scan flag doesn't get set, but is needed for initial!??
list.set({name: 'bob', age: 27});
list.set({name: 'carl', age: 29});
list.set({name: 'dave', age: 25});
@ -3455,7 +3455,42 @@ describe('Gun', function(){
list.path('message').put('hello world'); // outputs "message: hello world"
list.path('message').put(null); // throws Uncaught TypeError: Cannot read property '#' of null
});
it('Check multi instance message passing', function(done){
try{ require('fs').unlinkSync('bdata') }catch(e){}
try{ require('fs').unlinkSync('ddata') }catch(e){}
Gun.on('out', function(msg){
//console.log("oye", msg);
this.to.next(msg);
var onGun = msg.gun.back(-1);
if(onGun === b) {
if(d){
//console.log("b can send to d....", Gun.obj.copy(msg));
d.on("in", msg);
}
} else if(onGun === d){
//console.log("d sends to b....", Gun.obj.copy(msg));
b.on("in", msg);
}
})
var b = Gun({file: "bdata"});
var d = null;
var bb = b.get("key");
bb.put({msg: "hello"});
d = Gun({file: "ddata"});
var db = d.get("key");
db.map().on(function(val,field){
//console.log("d key got val:", field, val)
expect(val).to.be('hello');
if(done.c){ return } done.c = 1;
setTimeout(function(){
done();
},1700);
});
});
return;
it.only('Custom extensions are chainable', function(done){
Gun.chain.filter = function(filter){
@ -3503,6 +3538,20 @@ describe('Gun', function(){
.val(function(yes){console.log("YES!", yes)})
});
it.only('Check that events are called with multiple instances', function(done){
var gunA = Gun( { file : "A.json" } );
var gunB = Gun( { file : "B.json" });
var gunC = Gun( { file : "C.json" });
gunA.get( "some path A" ).map( (v,f)=>{ console.log( "event on A: ", f, v ) } );
gunB.get( "some path B" ).map( (v,f)=>{ console.log( "event on B: ", f, v ) } );
gunC.get( "some path C" ).map( (v,f)=>{ console.log( "event on C: ", f, v ) } );
gunA.get( "some path A" ).put( { simple:"message" } );
gunB.get( "some path B" ).put( { simple:"message" } );
gunC.get( "some path C" ).put( { simple:"message" } );
});
it.only('Make sure circular contexts are not copied', function(done){
/* let's define an appropriate deep default database... */
var dfltSansUsers = { 1: { name : "org1", sites : { 1: {name : "site1"} } } };

176
test/panic/b2s2s2b.js Normal file
View File

@ -0,0 +1,176 @@
var config = {
IP: require('ip').address(),
port: 8080,
servers: 2,
browsers: 2,
each: 12000,
burst: 1000,
wait: 1,
route: {
'/': __dirname + '/index.html',
'/gun.js': __dirname + '/../../gun.js',
'/jquery.js': __dirname + '/../../examples/jquery.js'
}
}
var panic = require('panic-server');
panic.server().on('request', function(req, res){
config.route[req.url] && require('fs').createReadStream(config.route[req.url]).pipe(res);
}).listen(config.port);
var clients = panic.clients;
var manager = require('panic-manager')();
manager.start({
clients: Array(config.servers).fill().map(function(u, i){
return {
type: 'node',
port: config.port + (i + 1)
}
}),
panic: 'http://' + config.IP + ':' + config.port
});
var servers = clients.filter('Node.js');
var browsers = clients.excluding(servers);
describe("Load test "+ config.browsers +" browser(s) across "+ config.servers +" server(s)!", function(){
//this.timeout(5 * 60 * 1000);
this.timeout(10 * 60 * 1000);
it("Servers have joined!", function(){
return servers.atLeast(config.servers);
});
it("GUN has spawned!", function(){
var tests = [], i = 0;
servers.each(function(client){
tests.push(client.run(function(test){
var env = test.props;
test.async();
try{ require('fs').unlinkSync(env.i+'data') }catch(e){}
var server = require('http').createServer(function(req, res){
res.end("I am "+ env.i +"!");
});
var port = env.config.port + env.i;
var Gun = require('gun');
var peers = [], i = env.config.servers;
while(i--){
var tmp = (env.config.port + (i + 1));
if(port != tmp){ // ignore ourselves
peers.push('http://'+ env.config.IP + ':' + tmp + '/gun');
}
}
console.log(port, " connect to ", peers);
var gun = Gun({file: env.i+'data', peers: peers, web: server});
server.listen(port, function(){
test.done();
});
}, {i: i += 1, config: config}));
});
return Promise.all(tests);
});
it(config.browsers +" browser(s) have joined!", function(){
console.log("PLEASE OPEN http://"+ config.IP +":"+ config.port +" IN "+ config.browsers +" BROWSER(S)!");
browsers.atLeast(1).then(function(){
browsers.run(function(test){
var env = test.props;
$('body').prepend("<button onclick='allopen()'>Open All Browsers</button>");
window.allopen = function(i){
if(env.config.browsers <= i){ return }
i = i || 1;
var win = window.open(location, '_blank');
win.focus();
setTimeout(function(){allopen(i+1)},0);
}
}, {config: config});
});
return browsers.atLeast(config.browsers);
});
it("Data was saved and synced across all browsers!", function(){
var tests = [], ids = {}, i = 0;
browsers.each(function(client, id){
ids[id] = 1;
});
browsers.each(function(client, id){
tests.push(client.run(function(test){
localStorage.clear();
var env = test.props;
test.async();
var peers = [], i = env.i;
//while(i--){
peers.push('http://'+ env.config.IP + ':' + (env.config.port + (i)) + '/gun');
//}
console.log("Connect to", peers);
var gun = Gun(peers);
var num = 0, total = 0, check = Gun.obj.map(env.ids, function(v,id,t){
var i = env.config.each;
while(i--){
t(id + (i + 1), 1);
total += 1;
}
});
var report = $("<div>").css({position: 'fixed', top: 0, right: 0, background: 'white', padding: 10}).text(num +" / "+ total +" Verified").prependTo('body');
var wait;
gun.get('test').map().on(function(data, key){
//$(log).text(key +": "+ data);
if(("Hello world, "+key+"!") === data){
if(check[key]){ num += 1 }
check[key] = 0;
report.text(num +" / "+ total +" Verified");
}
if(wait){ return }
wait = setTimeout(function(){
wait = false;
if(Gun.obj.map(check, function(still){
if(still){ return true }
})){ return }
console.log("SUCCESS");
test.done();
},10);
});
setTimeout(function(){
console.log("<<<<< START >>>>>");
var i = 0, burst = false, to = setInterval(function go(){
if(!burst){
burst = env.config.burst;
while(--burst){
go();
}
burst = false;
return;
}
if(env.config.each <= i){
clearTimeout(to);
return;
}
i += 1;
var p = env.id + i;
gun.get('test').get(p).put('Hello world, '+ p +'!');
}, env.config.wait);
},500);
}, {i: i += 1, id: id, ids: ids, config: config}));
});
return Promise.all(tests);
});
it("All finished!", function(done){
console.log("Done! Cleaning things up...");
setTimeout(function(){
done();
},1000);
});
after("Everything shut down.", function(){
browsers.run(function(){
//location.reload();
//setTimeout(function(){
//}, 15 * 1000);
});
return servers.run(function(){
process.exit();
});
});
});

24
test/panic/load.js
View File

@ -2,8 +2,8 @@ var config = {
IP: require('ip').address(),
port: 8080,
servers: 1,
browsers: 2,
each: 500,
browsers: 4,
each: 1500,
wait: 1,
route: {
'/': __dirname + '/index.html',
@ -87,9 +87,8 @@ describe("Load test "+ config.browsers +" browser(s) across "+ config.servers +"
});
// Launch the server and start gun!
var Gun = require('gun');
var gun = Gun({file: env.i+'data.json'});
// Attach the server to gun.
//gun.wsp(server);
var gun = Gun({file: env.i+'data', web: server});
server.listen(env.config.port + env.i, function(){
// This server peer is now done with the test!
// It has successfully launched.
@ -154,6 +153,7 @@ describe("Load test "+ config.browsers +" browser(s) across "+ config.servers +"
browsers.each(function(client, id){
// for every browser, run the following code:
tests.push(client.run(function(test){
//var audio = new Audio('https://www.nasa.gov/mp3/640170main_Roger%20Roll.mp3');audio.addEventListener('ended', function() {this.currentTime = 0;this.play();}, false);audio.play(); // testing if audio prevents Chrome throttle?
localStorage.clear(); // Clean up anything from before.
var env = test.props;
// Get access to the "outer scope" which has the browser IDs
@ -164,9 +164,10 @@ describe("Load test "+ config.browsers +" browser(s) across "+ config.servers +"
while(i--){
// For the total number of servers listed in the configuration
// Add their URL into an array.
peers.push('http://'+ env.config.IP + ':' + (env.config.port + (i + 1)));
peers.push('http://'+ env.config.IP + ':' + (env.config.port + (i + 1)) + '/gun');
}
// Pass all the servers we want to connect to into gun.
//var gun = Gun();
var gun = Gun(peers);
// Now we want to create a list
// of all the messages that WILL be sent
@ -195,18 +196,19 @@ describe("Load test "+ config.browsers +" browser(s) across "+ config.servers +"
//var el = $('#'+key).length ? $('#'+key) : $('<div>');
// log the data out to, so we can visually see our test.
//$(log).append(el.attr('id', key).text(key +": "+ data));
$(log).text(key +": "+ data); // DOM updates thrash performance, try this.
// Scroll down with the logging.
//$('body').stop(true).animate({scrollTop: $(log).height()});
// Now, make sure the received data
// matches exactly the data we EXPECT
if(("Hello world, "+key+"!") === data){
// if it does, we can "check off" record
// Bump the total number of verified items and update the UI.
if(check[key]){ num += 1 }
// from our verify todo list.
check[key] = 0;
// Bump the total number of verified items and update the UI.
num += 1;
report.text(num +" / "+ total +" Verified");
}
// Scroll down with the logging.
//$('body').stop(true).animate({scrollTop: $(log).height()});
// This next part is important:
if(Gun.obj.map(check, function(still){
// IF THERE ARE ANY RECORDS STILL LEFT TO BE VERIFIED
@ -220,14 +222,14 @@ describe("Load test "+ config.browsers +" browser(s) across "+ config.servers +"
test.done();
});
// But we have to actually tell the browser to save data!
var i = 0, to = setTimeout(function go(){
var i = 0, to = setInterval(function go(){
// Cool, make a recursive function
// that keeps going until we've saved each message.
if(env.config.each <= i){
clearTimeout(to);
return;
}
to = setTimeout(go, env.config.wait * Math.random()); // add a little jitter.
//to = setTimeout(go, env.config.wait * Math.random()); // add a little jitter.
i += 1;
var p = env.id + i;
// And actually save the data with gun,