From 13f6e287fa19f57d52bd184f19836297be0a8779 Mon Sep 17 00:00:00 2001 From: Mark Nadal Date: Tue, 20 Jan 2015 06:37:15 -0700 Subject: [PATCH] merged stash --- gun.js | 5 +- index.html | 923 +++++++++++++++++++++--------------------- lib/file.js | 7 +- test/common.js | 56 ++- web/forrest-notes.txt | 4 + 5 files changed, 506 insertions(+), 489 deletions(-) diff --git a/gun.js b/gun.js index dc1c8b61..8c1a6f6a 100644 --- a/gun.js +++ b/gun.js @@ -418,9 +418,7 @@ if(err){ return cb(err) } return cb(null); }); - } else { - Gun.log.call(gun, "Warning! You have no persistence layer to save to!"); - } + }); }); if(!gun.back){ gun.shot('then').fire(set); @@ -458,6 +456,7 @@ }); return gun; } + */ // Union is different than set. Set casts non-gun style of data into a gun compatible data. // Union takes already gun compatible data and validates it for a merge. // Meaning it is more low level, such that even set uses union internally. diff --git a/index.html b/index.html index fb55bab1..64f394e0 100644 --- a/index.html +++ b/index.html @@ -1,491 +1,492 @@ - - - - - - - + + + + + + + -
+
-
+
-
+
- + -
-
-
-
-
-
-
-
-
- -
-
-
-
-
-
-
-
- -
-
-
-
-
-
- -
-
-
-
-
-
- -
-
-
-
-
-
- -
-
-
-
-
-
- - Distributed embedded graph database engine. The no pain, no cost, no backend, NoDB database, gun. +
+
+
-
- -
-
-
-
-
-
- - - -
-
-
-
-
-
-
-
-
- -
-
-
-
-
-
-
-
- -
-
-
-
-
-
- -
-
-
-
-
-
- - -
-
-
-
-
-
- -
-
-
-
-
-
- - Distributed embedded graph database engine. The no pain, no cost, no backend, NoDB database, gun.
- - - -
-
Easiest Database Ever
-
Sync state in a cinch at a distributed system scale.
-
- -
- +
- Fork me on GitHub - +
+
+ +
+
+
+
+
+
+
+
+ +
+
+
+
+
+
+ +
+
+
+
+
+
+ +
+
+
+
+
+
+ +
+
+
+
+
+
+ +Distributed embedded graph database engine. The no pain, no cost, no backend, NoDB database, gun. +
+
+ +
+
+
+
+
+
+ + + +
+
+
+
+
+
+
+
+
+ +
+
+
+
+
+
+
+
+ +
+
+
+
+
+
+ +
+
+
+
+
+
+ + +
+
+
+
+
+
+ +
+
+
+
+
+
+ +Distributed embedded graph database engine. The no pain, no cost, no backend, NoDB database, gun. +
+ + + +
+
Easiest Database Ever
+
Sync state in a cinch at a distributed system scale.
+
+ +
+ +
+Fork me on GitHub +
-

The blog that started it all...

-

- Gun is a persisted distributed cache, part of a NoDB movement. - It requires zero maintenance and runs on your own infrastructure. - Think of it as "Dropbox for Databases" or a "Self-hosted Firebase". - This is an early preview, so check out the github and read on. -

-

- Everything gets cached, so your users experience lightning fast response times. - Since gun can be embedded anywhere javascript can run, - that cache can optionally be right inside your user's browser using localstorage fallbacks. - Updates are then pushed up to the servers when the network is available. -

-

- All conflict resolution happens locally in each peer using a deterministic algorithm. - Such that eventual consistency is guaranteed across all writes within the mesh, - with fault tolerant retries built in at each step. Data integrity is now a breeze. -

-

- Gun also establishes and repairs server to server communication across geographically separated machines, - with just the help of an initial IP from you. - It bridges the distance with a realtime connection, - so updates propagate at the speed of the raw pipes linking them. - However each server is intelligent enough to only subscribe to the necessary subsection of your data set that is in its working memory, - keeping things nimble for its connected users. -

-

- Data is then persisted to any S3 like service, - allowing you to save a truly webscale amount of "big data" without breaking your wallet. - Consistency across concurrency is achieved at this layer - by each parallel snapshot going through an idempotent transformation that is agreed upon. - The granularity and frequency of these snapshots can be tweaked by you, - easily tailor fitting it to your data needs and budget concerns. -

-

- In summary, this marks an important progression in web technologies. - Memory is getting cheap enough that we can now front load each connected user's active data - right into the application layer and manipulate it directly. - Networks are fast enough that if we get too many connected users we can just horizontally - redistribute them across different machines. - Conflict resolution algorithms are sophisticated enough to handle things immediately - in the majority of data cases, with the remaining few as transactions performed ontop. +

The blog that started it all...

- This is will be a win for both developers and users, - because it makes things easier to build and faster to respond. - We are excited about this and working hard to finish the first release. + Gun is a persisted distributed cache, part of a NoDB movement. + It requires zero maintenance and runs on your own infrastructure. + Think of it as "Dropbox for Databases" or a "Self-hosted Firebase". + This is an early preview, so check out the github and read on.

- If you would like to learn more, email hi@gunDB.io - - or join the Google Group. - Plus we're hiring, so contact us! -

-

FAQ

-

Why did you build this thing?

-

- 1. I love databases, especially new ones that keep the working set in memory. - But I was horrified to realize that if I use a Database as a Service (DaaS) - then I would have to rely on a network call to get that data, which is horribly slow. - I wanted something faster, and if possible, cheaper - because they start charging you - outrageous prices if you get past their incredibly tiny free tier. + Everything gets cached, so your users experience lightning fast response times. + Since gun can be embedded anywhere javascript can run, + that cache can optionally be right inside your user's browser using localstorage fallbacks. + Updates are then pushed up to the servers when the network is available.

- 2. Hosting your own database is a pain because you have to maintain the hard drives. - Even the basic setup and configuration is nasty... - having to create a bunch of EBS volumes, attaching them, then mounting, formatting, - MDADM and LVM, striping, mirroring, and keeping fstab from locking on boot. - This is ignoring having to figure out how to resize them. - Even with SSDs you have problems that they are bound to one instance and - you get charged for the total volume size, not the amount used. + All conflict resolution happens locally in each peer using a deterministic algorithm. + Such that eventual consistency is guaranteed across all writes within the mesh, + with fault tolerant retries built in at each step. Data integrity is now a breeze.

- I wanted something easy, needing no maintenance and be extremely portable - - allowing me to spin up an ephemeral server anywhere, on any cloud, - and my data "just work" there. -

-

How are you different from every other database that is trying to reinvent the wheel?

-

- 1. Because gun is not a database (NoDB), it is a persisted distributed cache. - The fatal flaw with databases is that they assume some centralized authority. - While this may be the case initially when you are small, - it always ceases to be true when you become large enough that concurrency is unavoidable. - No amount of leader election and consensus algorithms can patch this - without facing an unjustified amount of complexity. - Gun resolves all this by biting the bullet - - it solves the hard problems first, not last. - It gets data synchronization and conflict resolution right from the beginning, - so it never has to rely on vulnerable leader election or consensus locking. + Gun also establishes and repairs server to server communication across geographically separated machines, + with just the help of an initial IP from you. + It bridges the distance with a realtime connection, + so updates propagate at the speed of the raw pipes linking them. + However each server is intelligent enough to only subscribe to the necessary subsection of your data set that is in its working memory, + keeping things nimble for its connected users.

- 2. Plus it embeds directly into your application, - so you can do your own custom queries with just pure programming logic. - Meaning you never have to learn some silly separate query language again. - A query language which just attempts to be some DSL to RPC another machine - into doing the same query you could have already written in half the time - it took to learn the query language. Because face it, any sufficiently capable - query language has to be Turing complete, and at that point - - why aren't you just writing your entire application logic in it? - Your app is nothing without your data. + Data is then persisted to any S3 like service, + allowing you to save a truly webscale amount of "big data" without breaking your wallet. + Consistency across concurrency is achieved at this layer + by each parallel snapshot going through an idempotent transformation that is agreed upon. + The granularity and frequency of these snapshots can be tweaked by you, + easily tailor fitting it to your data needs and budget concerns.

-
+

+ In summary, this marks an important progression in web technologies. + Memory is getting cheap enough that we can now front load each connected user's active data + right into the application layer and manipulate it directly. + Networks are fast enough that if we get too many connected users we can just horizontally + redistribute them across different machines. + Conflict resolution algorithms are sophisticated enough to handle things immediately + in the majority of data cases, with the remaining few as transactions performed ontop. +

+ This is will be a win for both developers and users, + because it makes things easier to build and faster to respond. + We are excited about this and working hard to finish the first release. +

+

+ If you would like to learn more, email hi@gunDB.io - + or join the Google Group. + Plus we're hiring, so contact us! +

+

FAQ

+

Why did you build this thing?

+

+ 1. I love databases, especially new ones that keep the working set in memory. + But I was horrified to realize that if I use a Database as a Service (DaaS) + then I would have to rely on a network call to get that data, which is horribly slow. + I wanted something faster, and if possible, cheaper - because they start charging you + outrageous prices if you get past their incredibly tiny free tier. +

+

+ 2. Hosting your own database is a pain because you have to maintain the hard drives. + Even the basic setup and configuration is nasty... + having to create a bunch of EBS volumes, attaching them, then mounting, formatting, + MDADM and LVM, striping, mirroring, and keeping fstab from locking on boot. + This is ignoring having to figure out how to resize them. + Even with SSDs you have problems that they are bound to one instance and + you get charged for the total volume size, not the amount used. +

+

+ I wanted something easy, needing no maintenance and be extremely portable + - allowing me to spin up an ephemeral server anywhere, on any cloud, + and my data "just work" there. +

+

How are you different from every other database that is trying to reinvent the wheel?

+

+ 1. Because gun is not a database (NoDB), it is a persisted distributed cache. + The fatal flaw with databases is that they assume some centralized authority. + While this may be the case initially when you are small, + it always ceases to be true when you become large enough that concurrency is unavoidable. + No amount of leader election and consensus algorithms can patch this + without facing an unjustified amount of complexity. + Gun resolves all this by biting the bullet - + it solves the hard problems first, not last. + It gets data synchronization and conflict resolution right from the beginning, + so it never has to rely on vulnerable leader election or consensus locking. +

+

+ 2. Plus it embeds directly into your application, + so you can do your own custom queries with just pure programming logic. + Meaning you never have to learn some silly separate query language again. + A query language which just attempts to be some DSL to RPC another machine + into doing the same query you could have already written in half the time + it took to learn the query language. Because face it, any sufficiently capable + query language has to be Turing complete, and at that point - + why aren't you just writing your entire application logic in it? + Your app is nothing without your data. +

+
- + - + diff --git a/lib/file.js b/lib/file.js index 99392b51..e5fab8d8 100644 --- a/lib/file.js +++ b/lib/file.js @@ -1,4 +1,5 @@ // This was written by the wonderful Forrest Tait +// modified by Mark to be part of core for convenience // twas not designed for production use // only simple local development. @@ -9,7 +10,7 @@ Gun.on('opt').event(function(gun, opts){ var fs = file.fs = file.fs || require('fs'); var raw = file.raw = file.raw || fs.existsSync(opts.file || data.txt)? - fs.readFileSync(opts.file || 'data.txt').toString() + fs.readFileSync(opts.file || 'data.json').toString() : null; var all = file.all = file.all || Gun.obj.ify(raw || {nodes: {}, keys: {}}); @@ -29,11 +30,11 @@ Gun.on('opt').event(function(gun, opts){ } } } - fs.writeFile(opts.file || 'data.txt', Gun.text.ify(all), cb); + fs.writeFile(opts.file || 'data.json', Gun.text.ify(all), cb); } ,key: function(key, soul, cb){ all.keys[key] = soul; - fs.writeFile(opts.file || 'data.txt', Gun.text.ify(all), cb); + fs.writeFile(opts.file || 'data.json', Gun.text.ify(all), cb); } }}, true); diff --git a/test/common.js b/test/common.js index 72185786..72ce13f2 100644 --- a/test/common.js +++ b/test/common.js @@ -1,6 +1,7 @@ describe('Gun', function(){ var Gun = require('../gun') , t = {}; + require(__dirname + '/../lib/file'); describe('Utility', function(){ describe('Type Check', function(){ it('binary', function(){ @@ -43,7 +44,7 @@ describe('Gun', function(){ expect(Gun.text.is({})).to.be(false); expect(Gun.text.is({a:1})).to.be(false); expect(Gun.text.is(function(){})).to.be(false); - }); + }); it('list',function(){ expect(Gun.list.is([])).to.be(true); expect(Gun.list.is([1])).to.be(true); @@ -140,7 +141,7 @@ describe('Gun', function(){ it('ify',function(){ expect(Gun.obj.ify('[0,1]')).to.eql([0,1]); expect(Gun.obj.ify('{"a":false,"b":1,"c":"d","e":[0,1],"f":{"g":"h"}}')).to.eql({"a":false,"b":1,"c":"d","e":[0,1],"f":{"g":"h"}}); - }); + }); it('map',function(){ expect(Gun.obj.map({a:'z',b:'y',c:'x'},function(v,i,t){ t(v,i) })).to.eql({x:'c',y:'b',z:'a'}); expect(Gun.obj.map({a:'z',b:false,c:'x'},function(v,i,t){ if(!v){ return } t(i,v) })).to.eql({a:'z',c:'x'}); @@ -163,7 +164,7 @@ describe('Gun', function(){ })); }); }); - + describe('Gun Safety', function(){ var gun = Gun(); it('is',function(){ @@ -256,14 +257,14 @@ describe('Gun', function(){ }); }); }); - + it('ify', function(){ var data, test; - + data = {a: false, b: true, c: 0, d: 1, e: '', f: 'g', h: null}; test = Gun.ify(data); expect(test.err).to.not.be.ok(); - + data = {}; data.a = {x: 1, y: 2, z: 3} data.b = {m: 'n', o: 'p', q: 'r', s: 't'}; @@ -272,54 +273,54 @@ describe('Gun', function(){ data.loop = [data.b, data.a.kid, data]; test = Gun.ify(data); expect(test.err).to.not.be.ok(); - + data = {_: {'#': 'shhh', meta: {yay: 1}}, sneak: true}; test = Gun.ify(data); expect(test.err).to.not.be.ok(); // metadata needs to be stored, but it can't be used for data. - + data = {}; data.sneak = false; data.both = {inside: 'meta data'}; data._ = {'#': 'shhh', data: {yay: 1}, spin: data.both}; test = Gun.ify(data); expect(test.err.meta).to.be.ok(); // TODO: Fail: this passes, somehow? Fix ify code! - + data = {one: {two: [9, 8, 7, 6, 5]}}; test = Gun.ify(data); expect(test.err.array).to.be.ok(); - + data = {z: undefined, x: 'bye'}; test = Gun.ify(data); expect(test.err.invalid).to.be.ok(); - + data = {a: NaN, b: 2}; test = Gun.ify(data); expect(test.err.invalid).to.be.ok(); - + data = {a: 1, b: Infinity}; test = Gun.ify(data); expect(test.err.invalid).to.be.ok(); - + data = {c: function(){}, d: 'hi'}; test = Gun.ify(data); expect(test.err.invalid).to.be.ok(); - + console.log(test.nodes); }); - + it('union', function(){ var graph, prime; - + graph = Gun.ify({a: false, b: true, c: 0, d: 1, e: '', f: 'g', h: null}).nodes; prime = Gun.ify({h: 9, i: 'foo', j: 'k', l: 'bar', m: 'Mark', n: 'Nadal'}).nodes; - - Gun.union(graph, prime); + + Gun.union(graph, prime); // TODO: BUG! Where is the expect??? }); - + it('path', function(done){ console.log("fix path!"); return done(); // TODO: FIX! This is broken because of API changes, fix it! - + this.timeout(9000); var gun = require('gun')({ s3: require('./shotgun') // replace this with your own keys! @@ -332,5 +333,16 @@ describe('Gun', function(){ }); console.log("________________________"); }); - -}); \ No newline at end of file + + it('set key get', function(done){ + var gun = require('gun/gun')(); + + gun.set({hello: "world"}).key('hello/world').get(function(val){ + console.log("?", val); + expect(val.hello).to.be('world'); + done(); + }).blank(function(){ console.log("nothing"); }) + .err(function(){ console.log("err"); }) + }); + +}); diff --git a/web/forrest-notes.txt b/web/forrest-notes.txt index 78d1acca..3f565cd1 100644 --- a/web/forrest-notes.txt +++ b/web/forrest-notes.txt @@ -1,5 +1,9 @@ NOTES FROM FORREST: +-3. `gun.path('players').path(id).set({x:0, y:0});` doesn't trigger `gun.path('players').on()` +-2. gun.on() needs to be chainable with map and stuff. +-1. Maybe make the map module core to gun? + 1 make raw HTTP work and document it, document that attach needs to be BEFORE listen. (talk to AJ) 2 make sure you SUPPRESS your comments, Mark. 3 .get doesn't work on subsequent closures. WHYYY??