diff --git a/docs/architecture.md b/docs/architecture.md new file mode 100644 index 0000000..3d405b9 --- /dev/null +++ b/docs/architecture.md @@ -0,0 +1,107 @@ +***WIP*** + +### Structure +The database has one log (OrbitList) for each *channel*. *Channel* is comparable to a *"topic"* or *"table"*. + +``` +DB +|-- channel 1 +| |-- list 1 +| | |-- node 1 +| | |-- operation +| | |-- value +| |-- list 2 +|-- channel 2 + |-- list 3 +... +``` + +### Operation Logs +- Each *channel* is saved as a log of operations: add, put, del +- Operations are stored in an append-only log linked list +- Each node in the linked list points to the previous node +- Event log: take latest add or del operation for +- JV-store: take last put or del operation for + +### CRDTs +- orbit-db is a CmRDT and implements an LWW-element-set +- operation-based CRDT +- (locally biased) vector clocks for partial ordering + +### add/put IO: +==> Not expensive + +1. Create POST for the content +`ipfs object get QmZzic86oN7B5GMMYnTFvZMyDMjkKEBXe5SYsryXPeoGdb?` +```json +{ + "content": "hello 1", + "ts": 1457703735759, + "meta": { + "type": "text", + "size": 7, + "ts": 1457703735759 + } +} +``` + +2. Create POST for the DB operation +`ipfs object get QmZzBNrUiYATJ4aicPTbupnEUWH3AHDmfBbDTQK3fhhYDE` +```json +{ + "op": "ADD", + "key": "QmZzic86oN7B5GMMYnTFvZMyDMjkKEBXe5SYsryXPeoGdb", + "value": "QmZzic86oN7B5GMMYnTFvZMyDMjkKEBXe5SYsryXPeoGdb", + "meta": { + "type": "orbit-db-op", + "size": 15, + "ts": 1457703735779 + }, + "by": "userA" +} +``` + +3. Create ipfs object for the nodet +`ipfs object get QmX2Jq1JHmgjgM3YVuHyGSRpUWMDbv4PuRhqBhsZqDmagD` +```json +{ + "id": "userA", + "seq": 0, + "ver": 1, + "data": "QmZzBNrUiYATJ4aicPTbupnEUWH3AHDmfBbDTQK3fhhYDE", + "next": { + "userA.0.0": "QmXUTgYPG4cSaHW8dKggJRfLvPWjaDktWyRAgn5NPwTqtz" + } +} +``` + +4. Create ipfs object for the current list +`ipfs object get Qmb2rpex9TdmoXvwLKLL24atWa2HfPbArobN7XiBAFvmZ9` +```json +{ + "id": "userA", + "seq": 1, + "ver": 0, + "items": { + "userA.0.0": "QmXUTgYPG4cSaHW8dKggJRfLvPWjaDktWyRAgn5NPwTqtz", + "userA.0.1": "QmX2Jq1JHmgjgM3YVuHyGSRpUWMDbv4PuRhqBhsZqDmagD" + } +} +``` + +5. Pubsub.publish (send to socket, network IO) +```json +{ + channel: '', + hash: 'Qmb2rpex9TdmoXvwLKLL24atWa2HfPbArobN7XiBAFvmZ9' +} +``` + +### get IO: +==> A little expensive + +1. Payload (get from ipfs-hash, network IO) + +### sync/merge IO: +==> Expensive! +TODO diff --git a/docs/ordering.md b/docs/ordering.md new file mode 100644 index 0000000..0dba2f8 --- /dev/null +++ b/docs/ordering.md @@ -0,0 +1,131 @@ +# Odering in OrbitList + +``` + A B C + 0.0 + | +0.0 0.1 + | | +0.1 0.2 + \ / <--- Sync B with A and C + 1.0 + | + 1.1 + / <--- Sync A with B +2.0 + \ + \ <--- Sync C with A + \ + 3.0 + <--- Sync ALL +``` + +Initial state: +``` +A = [] +B = [] +C = [] +``` + +Two nodes add events concurrently: + +Add items to A: +``` +listA.add("mango") +listA.add("banana") +// "A": [ +// { "id": "A", "seq": 0, "ver": 0, "prev": null} +// { "id": "A", "seq": 0, "ver": 1, "prev": ["A.0.0"]} +// ] +``` + +Add items to C: +``` +listC.add("apple") +listC.add("strawberry") +listC.add("orange") +// "C": [ +// { "id": "C", "seq": 0, "ver": 0, "prev": null} +// { "id": "C", "seq": 0, "ver": 1, "prev": ["C.0.0"]} +// { "id": "C", "seq": 0, "ver": 2, "prev": ["C.0.1"]} +// ] +``` + +B receives a 'sync' from A and C: +``` +Sync: B <--> A +Sync: B <--> C +``` + +Add items to B: +``` +listB.add("pineapple") +listB.add("papaya") +// "B": [ +// { "id": "A", "seq": 0, "ver": 0, "prev": null} +// { "id": "A", "seq": 0, "ver": 1, "prev": ["A.0.0"]} +// { "id": "C", "seq": 0, "ver": 0, "prev": null} +// { "id": "C", "seq": 0, "ver": 1, "prev": ["C.0.0"]} +// { "id": "C", "seq": 0, "ver": 2, "prev": ["C.0.1"]} +// { "id": "B", "seq": 1, "ver": 0, "prev": ["A.0.1", "C.0.2"]} +// { "id": "B", "seq": 1, "ver": 1, "prev": ["B.1.0"]} +// ] +``` + +A receives a 'sync' from B: +``` +Sync: A <--> B +``` + +``` +listA.add("kiwi") +// "B": [ +// { "id": "A", "seq": 0, "ver": 0, "prev": null} +// { "id": "A", "seq": 0, "ver": 1, "prev": ["A.0.0"]} +// { "id": "C", "seq": 0, "ver": 0, "prev": null} +// { "id": "C", "seq": 0, "ver": 1, "prev": ["C.0.0"]} +// { "id": "C", "seq": 0, "ver": 2, "prev": ["C.0.1"]} +// { "id": "B", "seq": 1, "ver": 0, "prev": ["A.0.1", "C.0.2"]} +// { "id": "B", "seq": 1, "ver": 1, "prev": ["B.1.0"]} +// { "id": "A", "seq": 2, "ver": 0, "prev": ["B.1.1"]} +// ] +``` + +C receives a 'sync' from A: +``` +Sync: C <--> A +``` + +``` +listC.add("blueberry") +// "C": [ +// { "id": "A", "seq": 0, "ver": 0, "prev": null} +// { "id": "A", "seq": 0, "ver": 1, "prev": ["A.0.0"]} +// { "id": "C", "seq": 0, "ver": 0, "prev": null} +// { "id": "C", "seq": 0, "ver": 1, "prev": ["C.0.0"]} +// { "id": "C", "seq": 0, "ver": 2, "prev": ["C.0.1"]} +// { "id": "B", "seq": 1, "ver": 0, "prev": ["A.0.1", "C.0.2"]} +// { "id": "B", "seq": 1, "ver": 1, "prev": ["B.1.0"]} +// { "id": "A", "seq": 2, "ver": 0, "prev": ["B.1.1"]} +// { "id": "C", "seq": 3, "ver": 0, "prev": ["A.2.0"]} +// ] +``` + +A receives a 'sync' from C, B receives a 'sync' from C: +``` +Sync: A <--> C +Sync: B <--> C +``` + +Data set converged (after sync ALL): +```json +{ "id": "A", "seq": 0, "ver": 0, "prev": null}, +{ "id": "A", "seq": 0, "ver": 1, "prev": ["A.0.0"]}, +{ "id": "C", "seq": 0, "ver": 0, "prev": null}, +{ "id": "C", "seq": 0, "ver": 1, "prev": ["C.0.0"]}, +{ "id": "C", "seq": 0, "ver": 2, "prev": ["C.0.1"]}, +{ "id": "B", "seq": 1, "ver": 0, "prev": ["A.0.1", "C.0.2"]}, +{ "id": "B", "seq": 1, "ver": 1, "prev": ["B.1.0"]} +{ "id": "A", "seq": 2, "ver": 0, "prev": ["B.1.1"]} +{ "id": "C", "seq": 3, "ver": 0, "prev": ["A.2.0]"]} +``` diff --git a/docs/versionclocks.json b/docs/versionclocks.json deleted file mode 100644 index 793c236..0000000 --- a/docs/versionclocks.json +++ /dev/null @@ -1,149 +0,0 @@ -// Data set grouped and sorted -"A" : [ - { "id": "A", "seq": 0, "ver": 0, "prev": null}, - { "id": "A", "seq": 0, "ver": 1, "prev": "A0.0"}, - { "id": "A", "seq": 0, "ver": 2, "prev": "A0.1"}, - { "id": "A", "seq": 0, "ver": 3, "prev": "A0.2"}, - { "id": "A", "seq": 0, "ver": 4, "prev": "A0.3"}, - { "id": "A", "seq": 2, "ver": 0, "prev": ["A0.4", "B1.1"]} -], -"B" : [ - { "id": "B", "seq": 1, "ver": 0, "prev": ["A0.4", "C0.2"]}, - { "id": "B", "seq": 1, "ver": 1, "prev": "B1.0"} -], -"C" : [ - { "id": "C", "seq": 0, "ver": 0, "prev": null}, - { "id": "C", "seq": 0, "ver": 1, "prev": "C0.0"}, - { "id": "C", "seq": 0, "ver": 2, "prev": "C0.1"}, - { "id": "C", "seq": 3, "ver": 0, "prev": ["C0.2", "A2.0]"]} -] - - A B C - | -0.0 - | -0.1 - | -0.2 0.0 - | | -0.3 0.1 - | | -0.4 0.2 - | \ / | - | 1.0 | - | | | - | 1.1 | - | / | -2.0 | - \ | - \ | - \ | - 3.0 - -// expected order A -{ "id": "A", "seq": 0, "ver": 0, "prev": null}, -{ "id": "A", "seq": 0, "ver": 1, "prev": "A0.0"}, -{ "id": "A", "seq": 0, "ver": 2, "prev": "A0.1"}, -{ "id": "A", "seq": 0, "ver": 3, "prev": "A0.2"}, -{ "id": "A", "seq": 0, "ver": 4, "prev": "A0.3"}, - { "id": "C", "seq": 0, "ver": 0, "prev": null}, - { "id": "C", "seq": 0, "ver": 1, "prev": "C0.0"}, - { "id": "C", "seq": 0, "ver": 2, "prev": "C0.1"}, - { "id": "B", "seq": 1, "ver": 0, "prev": ["A0.4", "C0.2"]}, - { "id": "B", "seq": 1, "ver": 1, "prev": "B1.0"} -{ "id": "A", "seq": 2, "ver": 0, "prev": ["A0.4", "B1.1"]} - { "id": "C", "seq": 3, "ver": 0, "prev": ["C0.2", "A2.0]"]} - -"VersionClock": { - "seq": 0, - "ver": 0 -} - -"Item": { - "id": "", - "VersionClock": "", - "prev": [] -} - -"List": { - "items": [""] -} - - - A B C - 0.0 - | -0.0 0.1 - | | -0.1 0.2 - \ / | - 1.0 | - | | - 1.1 | - / | -2.0 | - \ | - \ | - \ | - 3.0 - -// Sequence, --> syncs to -listA.add("mango") // { "id": "A", "seq": 0, "ver": 0, "prev": null} -listA.add("banana") // { "id": "A", "seq": 0, "ver": 1, "prev": "A.0.0"} ---> B - -// A -// { "id": "A", "seq": 0, "ver": 0, "prev": null} -// { "id": "A", "seq": 0, "ver": 1, "prev": "A.0.0"} - -listC.add("apple") // { "id": "C", "seq": 0, "ver": 0, "prev": null} -listC.add("strawberry") // { "id": "C", "seq": 0, "ver": 1, "prev": "C.0.0"} -listC.add("orange") // { "id": "C", "seq": 0, "ver": 2, "prev": "C.0.1"} ---> A,B - -// A -// { "id": "A", "seq": 0, "ver": 0, "prev": null} -// { "id": "A", "seq": 0, "ver": 1, "prev": "A.0.0"} -// { "id": "C", "seq": 0, "ver": 0, "prev": null} -// { "id": "C", "seq": 0, "ver": 1, "prev": "C.0.0"} -// { "id": "C", "seq": 0, "ver": 2, "prev": "C.0.1"} - -listB.add("pineapple") // { "id": "B", "seq": 1, "ver": 0, "prev": ["A.0.1", "C.0.2"]} -listB.add("papaya") // { "id": "B", "seq": 1, "ver": 1, "prev": "B.1.0"} ---> A - -// A -// { "id": "A", "seq": 0, "ver": 0, "prev": null} -// { "id": "A", "seq": 0, "ver": 1, "prev": "A.0.0"} -// { "id": "C", "seq": 0, "ver": 0, "prev": null} -// { "id": "C", "seq": 0, "ver": 1, "prev": "C.0.0"} -// { "id": "C", "seq": 0, "ver": 2, "prev": "C.0.1"} -// { "id": "B", "seq": 1, "ver": 0, "prev": ["A.0.1", "C.0.2"]} -// { "id": "B", "seq": 1, "ver": 1, "prev": "B.1.0"} - -listA.add("kiwi") // { "id": "A", "seq": 2, "ver": 0, "prev": ["A.0.1", "B1.1", "C0.2"]} ---> C - -// A -// { "id": "A", "seq": 0, "ver": 0, "prev": null} -// { "id": "A", "seq": 0, "ver": 1, "prev": "A.0.0"} -// { "id": "C", "seq": 0, "ver": 0, "prev": null} -// { "id": "C", "seq": 0, "ver": 1, "prev": "C.0.0"} -// { "id": "C", "seq": 0, "ver": 2, "prev": "C.0.1"} -// { "id": "B", "seq": 1, "ver": 0, "prev": ["A.0.0", "C.0.2"]} -// { "id": "B", "seq": 1, "ver": 1, "prev": "B.1.0"} -// { "id": "A", "seq": 2, "ver": 0, "prev": ["A.0.1", "B1.1", "C0.2"]} - -listC.add("blueberry") // { "id": "C", "seq": 3, "ver": 0, "prev": ["A.2.0", "C.0.2"]} ---> A,B - -// A -// { "id": "A", "seq": 0, "ver": 0, "prev": null} -// { "id": "A", "seq": 0, "ver": 1, "prev": "A.0.0"} -// { "id": "C", "seq": 0, "ver": 0, "prev": null} -// { "id": "C", "seq": 0, "ver": 1, "prev": "C.0.0"} -// { "id": "C", "seq": 0, "ver": 2, "prev": "C.0.1"} -// { "id": "B", "seq": 1, "ver": 0, "prev": ["A.0.0", "C.0.2"]} -// { "id": "B", "seq": 1, "ver": 1, "prev": "B.1.0"} -// { "id": "A", "seq": 2, "ver": 0, "prev": ["A.0.1", "B1.1", "C0.2"]} -// { "id": "C", "seq": 3, "ver": 0, "prev": ["A.2.0", "C.0.2"]}