From 77317178ef3304fb1b7db46db1f59dfdc58cb4f6 Mon Sep 17 00:00:00 2001 From: ryan Date: Thu, 28 Jul 2016 16:31:02 +0200 Subject: [PATCH 01/39] inital refactor --- bigchaindb/pipelines/election.py | 50 ++++++++++++++++++++++++ bigchaindb/processes.py | 7 ++-- bigchaindb/voter.py | 65 -------------------------------- tests/pipelines/test_election.py | 17 +++++++++ 4 files changed, 70 insertions(+), 69 deletions(-) create mode 100644 bigchaindb/pipelines/election.py create mode 100644 tests/pipelines/test_election.py diff --git a/bigchaindb/pipelines/election.py b/bigchaindb/pipelines/election.py new file mode 100644 index 00000000..98a19885 --- /dev/null +++ b/bigchaindb/pipelines/election.py @@ -0,0 +1,50 @@ +import rethinkdb as r +from multipipes import Pipeline, Node + +from bigchaindb.pipelines.utils import ChangeFeed +from bigchaindb import Bigchain + + +class Election: + + def __init__(self): + self.bigchain = Bigchain() + + def check_for_quorum(self, next_vote): + """ + Checks if block has enough invalid votes to make a decision + """ + next_block = r.table('bigchain')\ + .get(next_vote['vote']['voting_for_block'])\ + .run(self.bigchain.conn) + if self.bigchain.block_election_status(next_block) == self.bigchain.BLOCK_INVALID: + return next_block + + def requeue_transactions(self, invalid_block): + """ + Liquidates transactions from invalid blocks so they can be processed again + """ + for tx in invalid_block['block']['transactions']: + self.bigchain.write_transaction(tx) + + +def get_changefeed(): + return ChangeFeed(table='votes', operation='insert') + + +def create_pipeline(): + election = Election() + + election_pipeline = Pipeline([ + Node(election.check_for_quorum), + Node(election.requeue_transactions) + ]) + + return election_pipeline + + +def start(): + pipeline = create_pipeline() + pipeline.setup(indata=get_changefeed()) + pipeline.start() + return pipeline \ No newline at end of file diff --git a/bigchaindb/processes.py b/bigchaindb/processes.py index fe641750..d226778d 100644 --- a/bigchaindb/processes.py +++ b/bigchaindb/processes.py @@ -4,9 +4,9 @@ import multiprocessing as mp import rethinkdb as r import bigchaindb -from bigchaindb.pipelines import block +from bigchaindb.pipelines import block, election from bigchaindb import Bigchain -from bigchaindb.voter import Voter, Election +from bigchaindb.voter import Voter from bigchaindb.block import BlockDeleteRevert from bigchaindb.web import server @@ -70,19 +70,18 @@ class Processes(object): p_map_bigchain = mp.Process(name='bigchain_mapper', target=self.map_bigchain) p_block_delete_revert = mp.Process(name='block_delete_revert', target=delete_reverter.start) p_voter = Voter(self.q_new_block) - p_election = Election(self.q_block_new_vote) # start the processes logger.info('starting bigchain mapper') p_map_bigchain.start() logger.info('starting backlog mapper') logger.info('starting block') block.start() + election.start() p_block_delete_revert.start() logger.info('starting voter') p_voter.start() logger.info('starting election') - p_election.start() # start message p_voter.initialized.wait() diff --git a/bigchaindb/voter.py b/bigchaindb/voter.py index f7953451..4d60ecd5 100644 --- a/bigchaindb/voter.py +++ b/bigchaindb/voter.py @@ -197,68 +197,3 @@ class Voter(object): p_validate.start() p_vote.start() p_update.start() - - -class Election(object): - - def __init__(self, q_block_new_vote): - """ - Initialize the class with the needed queues. - - Initialize a queue where blocks with new votes will be held - """ - self.q_block_new_vote = q_block_new_vote - self.q_invalid_blocks = mp.Queue() - - def check_for_quorum(self): - """ - Checks if block has enough invalid votes to make a decision - """ - b = Bigchain() - - while True: - next_block = self.q_block_new_vote.get() - - # poison pill - if next_block == 'stop': - self.q_invalid_blocks.put('stop') - logger.info('clean exit') - return - - if b.block_election_status(next_block) == 'invalid': - self.q_invalid_blocks.put(next_block) - - def requeue_transactions(self): - """ - Liquidates transactions from invalid blocks so they can be processed again - """ - while True: - invalid_block = self.q_invalid_blocks.get() - - # poison pill - if invalid_block == 'stop': - logger.info('clean exit') - return - - b = Bigchain() - for tx in invalid_block['block']['transactions']: - b.write_transaction(tx) - - def kill(self): - """ - Terminate processes - """ - self.q_block_new_vote.put('stop') - - def start(self): - """ - Initialize, spawn, and start the processes - """ - - # initialize the processes - p_quorum_check = mp.Process(name='check_for_quorum', target=self.check_for_quorum) - p_requeue_tx = mp.Process(name='requeue_tx', target=self.requeue_transactions) - - # start the processes - p_quorum_check.start() - p_requeue_tx.start() diff --git a/tests/pipelines/test_election.py b/tests/pipelines/test_election.py new file mode 100644 index 00000000..f9f18285 --- /dev/null +++ b/tests/pipelines/test_election.py @@ -0,0 +1,17 @@ +import time +import random +from unittest.mock import patch + +import rethinkdb as r + +from bigchaindb.pipelines import election +from multipipes import Pipe, Pipeline + + +def test_check_for_quorum(b, user_vk): + e = election.Election() + + +def test_check_requeue_transaction(b, user_vk): + pass + From 1b204144575331d675f1dc371e78a6fb42d4c2e0 Mon Sep 17 00:00:00 2001 From: ryan Date: Mon, 1 Aug 2016 14:00:26 +0200 Subject: [PATCH 02/39] election tests --- tests/pipelines/test_election.py | 114 ++++++++++++++++++++++++++++++- 1 file changed, 112 insertions(+), 2 deletions(-) diff --git a/tests/pipelines/test_election.py b/tests/pipelines/test_election.py index f9f18285..9d71576d 100644 --- a/tests/pipelines/test_election.py +++ b/tests/pipelines/test_election.py @@ -1,5 +1,6 @@ import time import random +from bigchaindb import crypto, Bigchain from unittest.mock import patch import rethinkdb as r @@ -8,10 +9,119 @@ from bigchaindb.pipelines import election from multipipes import Pipe, Pipeline -def test_check_for_quorum(b, user_vk): +def test_check_for_quorum_invalid(b, user_vk): e = election.Election() + # create blocks with transactions + tx1 = b.create_transaction(b.me, user_vk, None, 'CREATE') + test_block = b.create_block([tx1]) + + # simulate a federation with four voters + key_pairs = [crypto.generate_key_pair() for _ in range(4)] + test_federation = [Bigchain(public_key=key_pair[1], private_key=key_pair[0]) + for key_pair in key_pairs] + + # add voters to block and write + test_block['block']['voters'] = [key_pair[1] for key_pair in key_pairs] + b.write_block(test_block) + + # split_vote (invalid) + votes = [member.vote(test_block['id'], 'abc', True) for member in test_federation[:2]] + \ + [member.vote(test_block['id'], 'abc', False) for member in test_federation[2:]] + + # cast votes + r.table('votes').insert(votes, durability='hard').run(b.conn) + + # since this block is now in valid, should pass to the next process + assert e.check_for_quorum(votes[-1]) == test_block + + +def test_check_for_quorum_valid(b, user_vk): + e = election.Election() + + # create blocks with transactions + tx1 = b.create_transaction(b.me, user_vk, None, 'CREATE') + test_block = b.create_block([tx1]) + + # simulate a federation with four voters + key_pairs = [crypto.generate_key_pair() for _ in range(4)] + test_federation = [Bigchain(public_key=key_pair[1], private_key=key_pair[0]) + for key_pair in key_pairs] + + # add voters to block and write + test_block['block']['voters'] = [key_pair[1] for key_pair in key_pairs] + b.write_block(test_block) + + # votes for block one + votes = [member.vote(test_block['id'], 'abc', True) + for member in test_federation] + # cast votes + r.table('votes').insert(votes, durability='hard').run(b.conn) + + # since this block is valid, should go nowhere + assert e.check_for_quorum(votes[-1]) is None + def test_check_requeue_transaction(b, user_vk): - pass + e = election.Election() + # create blocks with transactions + tx1 = b.create_transaction(b.me, user_vk, None, 'CREATE') + test_block = b.create_block([tx1]) + + e.requeue_transactions(test_block) + + assert r.table('backlog').get(tx1['id']).run(b.conn) == tx1 + + +@patch.object(Pipeline, 'start') +def test_start(mock_start): + # TODO: `block.start` is just a wrapper around `block.create_pipeline`, + # that is tested by `test_full_pipeline`. + # If anyone has better ideas on how to test this, please do a PR :) + election.start() + mock_start.assert_called_with() + + +def test_full_pipline(b, user_vk): + outpipe = Pipe() + + # write two blocks + txs = [] + for i in range(100): + tx = b.create_transaction(b.me, user_vk, None, 'CREATE') + tx = b.sign_transaction(tx, b.me_private) + txs.append(tx) + + valid_block = b.create_block(txs) + b.write_block(valid_block) + + txs = [] + for i in range(100): + tx = b.create_transaction(b.me, user_vk, None, 'CREATE') + tx = b.sign_transaction(tx, b.me_private) + txs.append(tx) + + invalid_block = b.create_block(txs) + b.write_block(invalid_block) + + pipeline = election.create_pipeline() + pipeline.setup(indata=election.get_changefeed(), outdata=outpipe) + pipeline.start() + + # vote one block valid, one invalid + vote_valid = b.vote(valid_block['id'], 'abc', True) + vote_invalid = b.vote(invalid_block['id'], 'abc', False) + + r.table('votes').insert(vote_valid, durability='hard').run(b.conn) + r.table('votes').insert(vote_invalid, durability='hard').run(b.conn) + + time.sleep(2) + pipeline.terminate() + + # only transactions from the invalid block should be returned to + # the backlog + assert r.table('backlog').count().run(b.conn) == 100 + tx_from_block = set([tx['id'] for tx in invalid_block['block']['transactions']]) + tx_from_backlog = set([tx['id'] for tx in list(r.table('backlog').run(b.conn))]) + assert tx_from_block == tx_from_backlog From 03454f708704445bf387a1f2ebe5959c717d7821 Mon Sep 17 00:00:00 2001 From: ryan Date: Mon, 1 Aug 2016 14:55:37 +0200 Subject: [PATCH 03/39] remove obsolete tests --- tests/db/test_voter.py | 47 +----------------------------------------- 1 file changed, 1 insertion(+), 46 deletions(-) diff --git a/tests/db/test_voter.py b/tests/db/test_voter.py index e6e200cb..1be89905 100644 --- a/tests/db/test_voter.py +++ b/tests/db/test_voter.py @@ -5,7 +5,7 @@ import multiprocessing as mp from bigchaindb import util -from bigchaindb.voter import Voter, Election, BlockStream +from bigchaindb.voter import Voter, BlockStream from bigchaindb import crypto, Bigchain @@ -471,51 +471,6 @@ class TestBlockElection(object): assert b.block_election_status(test_block) == Bigchain.BLOCK_INVALID r.table('votes').delete().run(b.conn) - def test_tx_rewritten_after_invalid(self, b, user_vk): - q_block_new_vote = mp.Queue() - - # create blocks with transactions - tx1 = b.create_transaction(b.me, user_vk, None, 'CREATE') - tx2 = b.create_transaction(b.me, user_vk, None, 'CREATE') - test_block_1 = b.create_block([tx1]) - test_block_2 = b.create_block([tx2]) - - # simulate a federation with four voters - key_pairs = [crypto.generate_key_pair() for _ in range(4)] - test_federation = [Bigchain(public_key=key_pair[1], private_key=key_pair[0]) - for key_pair in key_pairs] - - # simulate a federation with four voters - test_block_1['block']['voters'] = [key_pair[1] for key_pair in key_pairs] - test_block_2['block']['voters'] = [key_pair[1] for key_pair in key_pairs] - - # votes for block one - vote_1 = [member.vote(test_block_1['id'], 'abc', True) - for member in test_federation] - - # votes for block two - vote_2 = [member.vote(test_block_2['id'], 'abc', True) for member in test_federation[:2]] + \ - [member.vote(test_block_2['id'], 'abc', False) for member in test_federation[2:]] - - # construct valid block - r.table('votes').insert(vote_1, durability='hard').run(b.conn) - q_block_new_vote.put(test_block_1) - - # construct invalid block - r.table('votes').insert(vote_2, durability='hard').run(b.conn) - q_block_new_vote.put(test_block_2) - - election = Election(q_block_new_vote) - election.start() - time.sleep(1) - election.kill() - - # tx1 was in a valid block, and should not be in the backlog - assert r.table('backlog').get(tx1['id']).run(b.conn) is None - - # tx2 was in an invalid block and SHOULD be in the backlog - assert r.table('backlog').get(tx2['id']).run(b.conn)['id'] == tx2['id'] - class TestBlockStream(object): From 68f64ee7805719de3e8200292ef52c9143ffa3a1 Mon Sep 17 00:00:00 2001 From: ryan Date: Mon, 1 Aug 2016 15:49:50 +0200 Subject: [PATCH 04/39] return invalid block to outpipe --- bigchaindb/pipelines/election.py | 1 + tests/pipelines/test_election.py | 2 +- 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/bigchaindb/pipelines/election.py b/bigchaindb/pipelines/election.py index 98a19885..9285c9f3 100644 --- a/bigchaindb/pipelines/election.py +++ b/bigchaindb/pipelines/election.py @@ -26,6 +26,7 @@ class Election: """ for tx in invalid_block['block']['transactions']: self.bigchain.write_transaction(tx) + return invalid_block def get_changefeed(): diff --git a/tests/pipelines/test_election.py b/tests/pipelines/test_election.py index 9d71576d..36525d6e 100644 --- a/tests/pipelines/test_election.py +++ b/tests/pipelines/test_election.py @@ -116,7 +116,7 @@ def test_full_pipline(b, user_vk): r.table('votes').insert(vote_valid, durability='hard').run(b.conn) r.table('votes').insert(vote_invalid, durability='hard').run(b.conn) - time.sleep(2) + outpipe.get() pipeline.terminate() # only transactions from the invalid block should be returned to From dd0b758bfae6b2542ce5e128f2c0c2185e8e9e13 Mon Sep 17 00:00:00 2001 From: ryan Date: Mon, 1 Aug 2016 16:24:27 +0200 Subject: [PATCH 05/39] sleep --- bigchaindb/pipelines/election.py | 2 +- tests/pipelines/test_election.py | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/bigchaindb/pipelines/election.py b/bigchaindb/pipelines/election.py index 9285c9f3..26d77ac4 100644 --- a/bigchaindb/pipelines/election.py +++ b/bigchaindb/pipelines/election.py @@ -48,4 +48,4 @@ def start(): pipeline = create_pipeline() pipeline.setup(indata=get_changefeed()) pipeline.start() - return pipeline \ No newline at end of file + return pipeline diff --git a/tests/pipelines/test_election.py b/tests/pipelines/test_election.py index 36525d6e..300c896e 100644 --- a/tests/pipelines/test_election.py +++ b/tests/pipelines/test_election.py @@ -83,7 +83,7 @@ def test_start(mock_start): mock_start.assert_called_with() -def test_full_pipline(b, user_vk): +def test_full_pipeline(b, user_vk): outpipe = Pipe() # write two blocks @@ -108,7 +108,7 @@ def test_full_pipline(b, user_vk): pipeline = election.create_pipeline() pipeline.setup(indata=election.get_changefeed(), outdata=outpipe) pipeline.start() - + time.sleep(1) # vote one block valid, one invalid vote_valid = b.vote(valid_block['id'], 'abc', True) vote_invalid = b.vote(invalid_block['id'], 'abc', False) From 89661a9979c85b775c22f28fa3e7854ff919adc0 Mon Sep 17 00:00:00 2001 From: "Elad-PC\\elad" Date: Wed, 3 Aug 2016 06:34:16 +0200 Subject: [PATCH 06/39] moved transaction version inside the tx object that gets hashed moved the transaction version (currently fixed to 1.0) from the outer "transaction" object to the inside part, where it will get hashed together with the rest of the transaction. updated docs accordingly. --- bigchaindb/util.py | 4 ++-- docs/source/topic-guides/models.md | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/bigchaindb/util.py b/bigchaindb/util.py index 8492cc81..9e08b415 100644 --- a/bigchaindb/util.py +++ b/bigchaindb/util.py @@ -169,8 +169,8 @@ def create_tx(current_owners, new_owners, inputs, operation, payload=None): Reference: { "id": "", - "version": "transaction version number", "transaction": { + "version": "transaction version number", "fulfillments": [ { "current_owners": ["list of "], @@ -278,6 +278,7 @@ def create_tx(current_owners, new_owners, inputs, operation, payload=None): }) tx = { + 'version': 1, 'fulfillments': fulfillments, 'conditions': conditions, 'operation': operation, @@ -291,7 +292,6 @@ def create_tx(current_owners, new_owners, inputs, operation, payload=None): # create the transaction transaction = { 'id': tx_hash, - 'version': 1, 'transaction': tx } diff --git a/docs/source/topic-guides/models.md b/docs/source/topic-guides/models.md index c130ce83..8e0add9d 100644 --- a/docs/source/topic-guides/models.md +++ b/docs/source/topic-guides/models.md @@ -58,8 +58,8 @@ Assets can be mutable (changeable) or immutable. To change a mutable asset, you ```json { "id": "", - "version": "", "transaction": { + "version": "", "fulfillments": [""], "conditions": [""], "operation": "", @@ -75,8 +75,8 @@ Assets can be mutable (changeable) or immutable. To change a mutable asset, you Here's some explanation of the contents of a transaction: - `id`: The hash of everything inside the serialized `transaction` body (i.e. `fulfillments`, `conditions`, `operation`, `timestamp` and `data`; see below), with one wrinkle: for each fulfillment in `fulfillments`, `fulfillment` is set to `null`. The `id` is also the database primary key. -- `version`: Version number of the transaction model, so that software can support different transaction models. - `transaction`: + - `version`: Version number of the transaction model, so that software can support different transaction models. - `fulfillments`: List of fulfillments. Each _fulfillment_ contains a pointer to an unspent asset and a _crypto fulfillment_ that satisfies a spending condition set on the unspent asset. A _fulfillment_ is usually a signature proving the ownership of the asset. From 8c9d0fba3504d65e109b7a0a15cebc774c982ef3 Mon Sep 17 00:00:00 2001 From: "Elad-PC\\elad" Date: Wed, 3 Aug 2016 08:31:40 +0200 Subject: [PATCH 07/39] fixed reference to version in get_fulfillment_message --- bigchaindb/util.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bigchaindb/util.py b/bigchaindb/util.py index 9e08b415..67853ed2 100644 --- a/bigchaindb/util.py +++ b/bigchaindb/util.py @@ -479,7 +479,7 @@ def get_fulfillment_message(transaction, fulfillment, serialized=False): 'operation': transaction['transaction']['operation'], 'timestamp': transaction['transaction']['timestamp'], 'data': transaction['transaction']['data'], - 'version': transaction['version'], + 'version': transaction['transaction']['version'], 'id': transaction['id'] } # and the condition which needs to be retrieved from the output of a previous transaction From f35d5a708e2a7b3922ce561b2c65218b3f2eb12c Mon Sep 17 00:00:00 2001 From: "Elad-PC\\elad" Date: Wed, 3 Aug 2016 08:31:55 +0200 Subject: [PATCH 08/39] fixed tests to deal with new location of 'version' --- tests/db/test_bigchain_api.py | 14 +++++++------- tests/test_util.py | 2 +- 2 files changed, 8 insertions(+), 8 deletions(-) diff --git a/tests/db/test_bigchain_api.py b/tests/db/test_bigchain_api.py index 7082e52f..2eb8af7e 100644 --- a/tests/db/test_bigchain_api.py +++ b/tests/db/test_bigchain_api.py @@ -38,8 +38,8 @@ class TestBigchainApi(object): def test_create_transaction_create(self, b, user_sk): tx = b.create_transaction(b.me, user_sk, None, 'CREATE') - assert sorted(tx) == ['id', 'transaction', 'version'] - assert sorted(tx['transaction']) == ['conditions', 'data', 'fulfillments', 'operation', 'timestamp'] + assert sorted(tx) == ['id', 'transaction'] + assert sorted(tx['transaction']) == ['conditions', 'data', 'fulfillments', 'operation', 'timestamp', 'version'] def test_create_transaction_with_unsupported_payload_raises(self, b): with pytest.raises(TypeError): @@ -79,8 +79,8 @@ class TestBigchainApi(object): tx = b.create_transaction(user_vk, b.me, input_tx, 'TRANSFER') - assert sorted(tx) == ['id', 'transaction', 'version'] - assert sorted(tx['transaction']) == ['conditions', 'data', 'fulfillments', 'operation', 'timestamp'] + assert sorted(tx) == ['id', 'transaction'] + assert sorted(tx['transaction']) == ['conditions', 'data', 'fulfillments', 'operation', 'timestamp', 'version'] tx_signed = b.sign_transaction(tx, user_sk) @@ -1169,7 +1169,7 @@ class TestFulfillmentMessage(object): assert fulfillment_message['fulfillment']['input'] == original_fulfillment['input'] assert fulfillment_message['operation'] == tx['transaction']['operation'] assert fulfillment_message['timestamp'] == tx['transaction']['timestamp'] - assert fulfillment_message['version'] == tx['version'] + assert fulfillment_message['version'] == tx['transaction']['version'] @pytest.mark.usefixtures('inputs') def test_fulfillment_message_transfer(self, b, user_vk): @@ -1192,7 +1192,7 @@ class TestFulfillmentMessage(object): assert fulfillment_message['fulfillment']['input'] == original_fulfillment['input'] assert fulfillment_message['operation'] == tx['transaction']['operation'] assert fulfillment_message['timestamp'] == tx['transaction']['timestamp'] - assert fulfillment_message['version'] == tx['version'] + assert fulfillment_message['version'] == tx['transaction']['version'] def test_fulfillment_message_multiple_current_owners_multiple_new_owners_multiple_inputs(self, b, user_vk): # create a new users @@ -1230,7 +1230,7 @@ class TestFulfillmentMessage(object): assert fulfillment_message['fulfillment']['input'] == original_fulfillment['input'] assert fulfillment_message['operation'] == tx['transaction']['operation'] assert fulfillment_message['timestamp'] == tx['transaction']['timestamp'] - assert fulfillment_message['version'] == tx['version'] + assert fulfillment_message['version'] == tx['transaction']['version'] class TestTransactionMalleability(object): diff --git a/tests/test_util.py b/tests/test_util.py index 4a440d60..a4fe5a74 100644 --- a/tests/test_util.py +++ b/tests/test_util.py @@ -151,7 +151,7 @@ def test_create_tx_with_empty_inputs(): tx = create_tx(None, None, [], None) assert 'id' in tx assert 'transaction' in tx - assert 'version' in tx + assert 'version' in tx['transaction'] assert 'fulfillments' in tx['transaction'] assert 'conditions' in tx['transaction'] assert 'operation' in tx['transaction'] From 2e1abc3d6a2008f2da62cb7dbe1f6bb976c2d2a2 Mon Sep 17 00:00:00 2001 From: "Elad-PC\\elad" Date: Wed, 3 Aug 2016 08:55:37 +0200 Subject: [PATCH 09/39] fixed another test --- tests/db/test_bigchain_api.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/db/test_bigchain_api.py b/tests/db/test_bigchain_api.py index 2eb8af7e..32691468 100644 --- a/tests/db/test_bigchain_api.py +++ b/tests/db/test_bigchain_api.py @@ -1252,7 +1252,7 @@ class TestTransactionMalleability(object): assert b.is_valid_transaction(tx_changed) is False tx_changed = copy.deepcopy(tx_signed) - tx_changed['version'] = '0' + tx_changed['transaction']['version'] = '0' assert b.validate_fulfillments(tx_changed) is False assert b.is_valid_transaction(tx_changed) is False From 44aa17098d6771ccc1344c494223b0581d23560a Mon Sep 17 00:00:00 2001 From: ryan Date: Wed, 3 Aug 2016 15:41:27 +0200 Subject: [PATCH 10/39] clean up processes.py --- bigchaindb/processes.py | 8 +------- 1 file changed, 1 insertion(+), 7 deletions(-) diff --git a/bigchaindb/processes.py b/bigchaindb/processes.py index d226778d..5c7edcc3 100644 --- a/bigchaindb/processes.py +++ b/bigchaindb/processes.py @@ -31,7 +31,6 @@ class Processes(object): def __init__(self): # initialize the class self.q_new_block = mp.Queue() - self.q_block_new_vote = mp.Queue() self.q_revert_delete = mp.Queue() def map_bigchain(self): @@ -52,10 +51,6 @@ class Processes(object): # this should never happen in regular operation self.q_revert_delete.put(change['old_val']) - # update (new vote) - elif change['new_val'] is not None and change['old_val'] is not None: - self.q_block_new_vote.put(change['new_val']) - def start(self): logger.info('Initializing BigchainDB...') @@ -73,14 +68,13 @@ class Processes(object): # start the processes logger.info('starting bigchain mapper') p_map_bigchain.start() - logger.info('starting backlog mapper') logger.info('starting block') block.start() - election.start() p_block_delete_revert.start() logger.info('starting voter') p_voter.start() + election.start() logger.info('starting election') # start message From 0dca72cad8ac20bdc50612c9eb57e12fe8c65aed Mon Sep 17 00:00:00 2001 From: ryan Date: Wed, 3 Aug 2016 15:44:41 +0200 Subject: [PATCH 11/39] clean up test_election.py --- tests/pipelines/test_election.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/pipelines/test_election.py b/tests/pipelines/test_election.py index 300c896e..ad15e807 100644 --- a/tests/pipelines/test_election.py +++ b/tests/pipelines/test_election.py @@ -32,7 +32,7 @@ def test_check_for_quorum_invalid(b, user_vk): # cast votes r.table('votes').insert(votes, durability='hard').run(b.conn) - # since this block is now in valid, should pass to the next process + # since this block is now invalid, should pass to the next process assert e.check_for_quorum(votes[-1]) == test_block @@ -76,7 +76,7 @@ def test_check_requeue_transaction(b, user_vk): @patch.object(Pipeline, 'start') def test_start(mock_start): - # TODO: `block.start` is just a wrapper around `block.create_pipeline`, + # TODO: `block.election` is just a wrapper around `block.create_pipeline`, # that is tested by `test_full_pipeline`. # If anyone has better ideas on how to test this, please do a PR :) election.start() From eb10ffcba01b36ddff97659b4a000f6b4a29024f Mon Sep 17 00:00:00 2001 From: ryan Date: Wed, 3 Aug 2016 16:39:11 +0200 Subject: [PATCH 12/39] docstring --- bigchaindb/pipelines/election.py | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/bigchaindb/pipelines/election.py b/bigchaindb/pipelines/election.py index 26d77ac4..0f002f9a 100644 --- a/bigchaindb/pipelines/election.py +++ b/bigchaindb/pipelines/election.py @@ -1,3 +1,9 @@ +"""This module takes care of all the logic related to block status. + +Specifically, what happens when a block becomes invalid. The logic is +encapsulated in the ``Election`` class, while the sequence of actions +is specified in ``create_pipeline``. +""" import rethinkdb as r from multipipes import Pipeline, Node From 877fc15456440722dae745cff641a1f6375aefa4 Mon Sep 17 00:00:00 2001 From: ryan Date: Wed, 3 Aug 2016 17:38:40 +0200 Subject: [PATCH 13/39] election logging --- bigchaindb/pipelines/election.py | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/bigchaindb/pipelines/election.py b/bigchaindb/pipelines/election.py index 0f002f9a..538a5453 100644 --- a/bigchaindb/pipelines/election.py +++ b/bigchaindb/pipelines/election.py @@ -4,6 +4,8 @@ Specifically, what happens when a block becomes invalid. The logic is encapsulated in the ``Election`` class, while the sequence of actions is specified in ``create_pipeline``. """ +import logging + import rethinkdb as r from multipipes import Pipeline, Node @@ -11,6 +13,9 @@ from bigchaindb.pipelines.utils import ChangeFeed from bigchaindb import Bigchain +logger = logging.getLogger(__name__) + + class Election: def __init__(self): @@ -30,6 +35,9 @@ class Election: """ Liquidates transactions from invalid blocks so they can be processed again """ + logger.info('Rewriting %s transactions from invalid block %s', + len(invalid_block['block']['transactions']), + invalid_block['id']) for tx in invalid_block['block']['transactions']: self.bigchain.write_transaction(tx) return invalid_block From 711940af678552b3ea1e604da6596720d44ea297 Mon Sep 17 00:00:00 2001 From: David Gasparian Date: Fri, 5 Aug 2016 12:41:03 +0400 Subject: [PATCH 14/39] Make write_config() to output indented JSON into file --- bigchaindb/config_utils.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bigchaindb/config_utils.py b/bigchaindb/config_utils.py index 7d469504..81f8ed46 100644 --- a/bigchaindb/config_utils.py +++ b/bigchaindb/config_utils.py @@ -209,7 +209,7 @@ def write_config(config, filename=None): filename = CONFIG_DEFAULT_PATH with open(filename, 'w') as f: - json.dump(config, f) + json.dump(config, f, indent=4) def autoconfigure(filename=None, config=None, force=False): From bbe8d7192e7eb879a5e04035e2f53e689a85fcc9 Mon Sep 17 00:00:00 2001 From: Troy McConaghy Date: Fri, 5 Aug 2016 13:33:17 +0200 Subject: [PATCH 15/39] In Quickstart, update pip like everywhere else This makes the **Quickstart** instructions for updating `pip` (`pip3`) consistent with the other documentation and code. --- docs/source/quickstart.md | 9 ++++----- 1 file changed, 4 insertions(+), 5 deletions(-) diff --git a/docs/source/quickstart.md b/docs/source/quickstart.md index 800013ca..2ed037f3 100644 --- a/docs/source/quickstart.md +++ b/docs/source/quickstart.md @@ -15,16 +15,15 @@ sudo apt-get update sudo apt-get install g++ python3-dev ``` -D. Get the latest version of pip, wheel and setuptools: +D. Get the latest version of pip and setuptools: ```text -sudo apt-get install python3-setuptools -sudo easy_install3 pip -pip3 install --upgrade pip wheel setuptools +sudo apt-get install python3-pip +sudo pip3 install --upgrade pip setuptools ``` E. Install the `bigchaindb` Python package from PyPI: ```text -sudo pip install bigchaindb +sudo pip3 install bigchaindb ``` F. Configure and run BigchainDB: From b379d1c549eab06fba9be165995cce9a657b4a65 Mon Sep 17 00:00:00 2001 From: troymc Date: Mon, 8 Aug 2016 14:42:25 +0200 Subject: [PATCH 16/39] docs: moved 'Nodes, Clusters & Federations' earlier --- docs/source/clusters-feds/index.rst | 1 - docs/source/index.rst | 3 ++- docs/source/{clusters-feds => }/node-cluster-fed.md | 4 ++-- 3 files changed, 4 insertions(+), 4 deletions(-) rename docs/source/{clusters-feds => }/node-cluster-fed.md (64%) diff --git a/docs/source/clusters-feds/index.rst b/docs/source/clusters-feds/index.rst index ab7571d8..433aa2cd 100644 --- a/docs/source/clusters-feds/index.rst +++ b/docs/source/clusters-feds/index.rst @@ -7,7 +7,6 @@ BigchainDB Clusters & Federations .. toctree:: :maxdepth: 1 - node-cluster-fed set-up-a-federation backup deploy-on-aws diff --git a/docs/source/index.rst b/docs/source/index.rst index f54767f5..f51daf07 100644 --- a/docs/source/index.rst +++ b/docs/source/index.rst @@ -9,9 +9,10 @@ Table of Contents introduction quickstart + node-cluster-fed nodes/index - clusters-feds/index drivers-clients/index + clusters-feds/index topic-guides/index release-notes appendices/index diff --git a/docs/source/clusters-feds/node-cluster-fed.md b/docs/source/node-cluster-fed.md similarity index 64% rename from docs/source/clusters-feds/node-cluster-fed.md rename to docs/source/node-cluster-fed.md index 2987d477..32359675 100644 --- a/docs/source/clusters-feds/node-cluster-fed.md +++ b/docs/source/node-cluster-fed.md @@ -1,6 +1,6 @@ # Nodes, Clusters & Federations -A **BigchainDB node** is a server or set of closely-linked servers running RethinkDB Server, BigchainDB Server, and other BigchainDB-related software. Each node is controlled by one person or organization. +A **BigchainDB node** is a server or set of closely-linked servers running RethinkDB Server, BigchainDB Server, and related software. Each node is controlled by one person or organization. A set of BigchainDB nodes can connect to each other to form a **cluster**. Each node in the cluster runs the same software. A cluster contains one logical RethinkDB datastore. A cluster may have additional servers to do things such as cluster monitoring. @@ -8,6 +8,6 @@ The people and organizations that run the nodes in a cluster belong to a **feder **What's the Difference Between a Cluster and a Federation?** -A cluster is just a bunch of connected nodes (computers). A cluster might be operated by just one person. A federation is an organization which has a cluster, and where each node in the cluster has a different operator. +A cluster is just a bunch of connected nodes. A federation is an organization which has a cluster, and where each node in the cluster has a different operator. Confusingly, we sometimes call a federation's cluster its "federation." You can probably tell what we mean from context. \ No newline at end of file From bfd8c29ae964d55d3af5926d330395474c5a2ec3 Mon Sep 17 00:00:00 2001 From: troymc Date: Mon, 8 Aug 2016 15:02:25 +0200 Subject: [PATCH 17/39] docs: Added new 'Node Components' page --- docs/source/nodes/index.rst | 1 + docs/source/nodes/node-components.md | 17 +++++++++++++++++ 2 files changed, 18 insertions(+) create mode 100644 docs/source/nodes/node-components.md diff --git a/docs/source/nodes/index.rst b/docs/source/nodes/index.rst index 7ae86d44..b5b34876 100644 --- a/docs/source/nodes/index.rst +++ b/docs/source/nodes/index.rst @@ -7,6 +7,7 @@ BigchainDB Nodes .. toctree:: :maxdepth: 1 + node-components node-requirements setup-run-node run-with-docker diff --git a/docs/source/nodes/node-components.md b/docs/source/nodes/node-components.md new file mode 100644 index 00000000..a7b432ad --- /dev/null +++ b/docs/source/nodes/node-components.md @@ -0,0 +1,17 @@ +# Node Components + +A BigchainDB node must include, at least: + +* BigchainDB Server and +* RethinkDB Server. + +When doing development and testing, it's common to install both on the same machine, but in a production environment, it may make more sense to install them on separate machines. + +In a production environment, a BigchainDB node can have several other components, including: + +* nginx or similar, as a reverse proxy and/or load balancer for the web server +* An NTP daemon running on all machines running BigchainDB code, and possibly other machines +* A RethinkDB proxy server +* Monitoring software, to monitor all the machines in the node +* Maybe more, e.g. a configuration management server and agents on all machines + From d53523189aa48f94f0cb9beca52bcb272072ff13 Mon Sep 17 00:00:00 2001 From: troymc Date: Mon, 8 Aug 2016 15:08:31 +0200 Subject: [PATCH 18/39] docs: Updated 'Node Requirements' --- docs/source/nodes/node-requirements.md | 8 ++------ 1 file changed, 2 insertions(+), 6 deletions(-) diff --git a/docs/source/nodes/node-requirements.md b/docs/source/nodes/node-requirements.md index a68234b1..60df03df 100644 --- a/docs/source/nodes/node-requirements.md +++ b/docs/source/nodes/node-requirements.md @@ -1,14 +1,10 @@ # Node Requirements (OS, Memory, Storage, etc.) -For now, we will assume that a BigchainDB node is just one server. In the future, a node may consist of several closely-coupled servers run by one node operator (federation member). - - ## OS Requirements * RethinkDB Server [will run on any modern OS](https://www.rethinkdb.com/docs/install/). Note that the Fedora package isn't officially supported. Also, official support for Windows is fairly recent ([April 2016](https://rethinkdb.com/blog/2.3-release/)). -* Python 3.4+ [will run on any modern OS](https://docs.python.org/3.4/using/index.html). -* [Some functionality in the `multiprocessing` package doesn't work on OS X](https://docs.python.org/3.4/library/multiprocessing.html#multiprocessing.Queue.qsize). You can still use Mac OS X if you use Docker or a virtual machine. -* ZeroMQ [will run on any modern OS](http://zeromq.org/area:download). +* BigchainDB Server requires Python 3.4+ and Python 3.4+ [will run on any modern OS](https://docs.python.org/3.4/using/index.html). +* BigchaindB Server uses the Python `multiprocessing` package and [some functionality in the `multiprocessing` package doesn't work on OS X](https://docs.python.org/3.4/library/multiprocessing.html#multiprocessing.Queue.qsize). You can still use Mac OS X if you use Docker or a virtual machine. The BigchainDB core dev team uses Ubuntu 14.04 or Fedora 23. From d9321a83957bc08aca91c88fe658b46ec84ed785 Mon Sep 17 00:00:00 2001 From: troymc Date: Mon, 8 Aug 2016 15:26:27 +0200 Subject: [PATCH 19/39] docs: Moved 'Configuration Settings' to new 'Server Reference' section --- docs/source/appendices/firewall-notes.md | 2 +- docs/source/clusters-feds/backup.md | 2 +- docs/source/clusters-feds/deploy-on-aws.md | 2 +- docs/source/index.rst | 1 + docs/source/nodes/index.rst | 1 - docs/source/nodes/setup-run-node.md | 2 +- docs/source/{nodes => server-reference}/configuration.md | 4 +++- 7 files changed, 8 insertions(+), 6 deletions(-) rename docs/source/{nodes => server-reference}/configuration.md (94%) diff --git a/docs/source/appendices/firewall-notes.md b/docs/source/appendices/firewall-notes.md index 4f1e780c..dca89ac6 100644 --- a/docs/source/appendices/firewall-notes.md +++ b/docs/source/appendices/firewall-notes.md @@ -47,7 +47,7 @@ Port 8080 is the default port used by RethinkDB for its adminstrative web (HTTP) Port 9984 is the default port for the BigchainDB client-server HTTP API (TCP), which is served by Gunicorn HTTP Server. It's _possible_ allow port 9984 to accept inbound traffic from anyone, but we recommend against doing that. Instead, set up a reverse proxy server (e.g. using Nginx) and only allow traffic from there. Information about how to do that can be found [in the Gunicorn documentation](http://docs.gunicorn.org/en/stable/deploy.html). (They call it a proxy.) -If Gunicorn and the reverse proxy are running on the same server, then you'll have to tell Gunicorn to listen on some port other than 9984 (so that the reverse proxy can listen on port 9984). You can do that by setting `server.bind` to 'localhost:PORT' in the [BigchainDB Configuration Settings](../nodes/configuration.html), where PORT is whatever port you chose (e.g. 9983). +If Gunicorn and the reverse proxy are running on the same server, then you'll have to tell Gunicorn to listen on some port other than 9984 (so that the reverse proxy can listen on port 9984). You can do that by setting `server.bind` to 'localhost:PORT' in the [BigchainDB Configuration Settings](../server-reference/configuration.html), where PORT is whatever port you chose (e.g. 9983). You may want to have Gunicorn and the reverse proxy running on different servers, so that both can listen on port 9984. That would also help isolate the effects of a denial-of-service attack. diff --git a/docs/source/clusters-feds/backup.md b/docs/source/clusters-feds/backup.md index ff45ef47..e42a115b 100644 --- a/docs/source/clusters-feds/backup.md +++ b/docs/source/clusters-feds/backup.md @@ -39,7 +39,7 @@ rethinkdb dump -e bigchain.bigchain -e bigchain.votes ``` That should write a file named `rethinkdb_dump__