Tendermint v0.22.8 backward compatibility (#2677)

Problem statement:

BigchainDB v2.0.0b9 has been around for quite a while. Recently we have updated
Tendermint supported version to v0.31.5 which has incompatible blockchain.
Despite the fact that we have defined instructions on chain migration, no one
expected to migrate to incompatible chain within patch version range. So there
is a demand for Tendermint v0.22.8 compatibility among BigchainDB users.

Work has been done:

bigchaindb-abci package was upgraded to support multiple API versions.
New configuration field stating tendermint version was added.

Signed-off-by: David Dashyan <mail@davie.li>
This commit is contained in:
David Dashyan 2020-01-24 01:17:27 +03:00 committed by Jürgen Eckel
parent 86472157db
commit 7df59994e9
12 changed files with 132 additions and 121 deletions

View File

@ -70,6 +70,7 @@ config = {
'tendermint': {
'host': 'localhost',
'port': 26657,
'version': 'v0.31.5', # look for __tm_supported_versions__
},
# FIXME: hardcoding to localmongodb for now
'database': _database_map['localmongodb'],

View File

@ -9,15 +9,7 @@ import logging
import sys
from abci.application import BaseApplication
from abci import (
ResponseInitChain,
ResponseInfo,
ResponseCheckTx,
ResponseBeginBlock,
ResponseDeliverTx,
ResponseEndBlock,
ResponseCommit,
)
from abci import CodeTypeOk
from bigchaindb import BigchainDB
from bigchaindb.elections.election import Election
@ -30,7 +22,6 @@ import bigchaindb.upsert_validator.validator_utils as vutils
from bigchaindb.events import EventTypes, Event
CodeTypeOk = 0
CodeTypeError = 1
logger = logging.getLogger(__name__)
@ -42,7 +33,8 @@ class App(BaseApplication):
transaction logic to Tendermint Core.
"""
def __init__(self, bigchaindb=None, events_queue=None):
def __init__(self, abci, bigchaindb=None, events_queue=None,):
super().__init__(abci)
self.events_queue = events_queue
self.bigchaindb = bigchaindb or BigchainDB()
self.block_txn_ids = []
@ -108,7 +100,7 @@ class App(BaseApplication):
genesis.chain_id, True)
self.chain = {'height': abci_chain_height, 'is_synced': True,
'chain_id': genesis.chain_id}
return ResponseInitChain()
return self.abci.ResponseInitChain()
def info(self, request):
"""Return height of the latest committed block."""
@ -123,7 +115,7 @@ class App(BaseApplication):
logger.info(f"Tendermint version: {request.version}")
r = ResponseInfo()
r = self.abci.ResponseInfo()
block = self.bigchaindb.get_latest_block()
if block:
chain_shift = 0 if self.chain is None else self.chain['height']
@ -148,10 +140,10 @@ class App(BaseApplication):
transaction = decode_transaction(raw_transaction)
if self.bigchaindb.is_valid_transaction(transaction):
logger.debug('check_tx: VALID')
return ResponseCheckTx(code=CodeTypeOk)
return self.abci.ResponseCheckTx(code=CodeTypeOk)
else:
logger.debug('check_tx: INVALID')
return ResponseCheckTx(code=CodeTypeError)
return self.abci.ResponseCheckTx(code=CodeTypeError)
def begin_block(self, req_begin_block):
"""Initialize list of transaction.
@ -168,7 +160,7 @@ class App(BaseApplication):
self.block_txn_ids = []
self.block_transactions = []
return ResponseBeginBlock()
return self.abci.ResponseBeginBlock()
def deliver_tx(self, raw_transaction):
"""Validate the transaction before mutating the state.
@ -185,12 +177,12 @@ class App(BaseApplication):
if not transaction:
logger.debug('deliver_tx: INVALID')
return ResponseDeliverTx(code=CodeTypeError)
return self.abci.ResponseDeliverTx(code=CodeTypeError)
else:
logger.debug('storing tx')
self.block_txn_ids.append(transaction.id)
self.block_transactions.append(transaction)
return ResponseDeliverTx(code=CodeTypeOk)
return self.abci.ResponseDeliverTx(code=CodeTypeOk)
def end_block(self, request_end_block):
"""Calculate block hash using transaction ids and previous block
@ -226,7 +218,7 @@ class App(BaseApplication):
self.new_height,
self.block_transactions)
return ResponseEndBlock(validator_updates=validator_update)
return self.abci.ResponseEndBlock(validator_updates=validator_update)
def commit(self):
"""Store the new height and along with block hash."""
@ -257,7 +249,7 @@ class App(BaseApplication):
})
self.events_queue.put(event)
return ResponseCommit(data=data)
return self.abci.ResponseCommit(data=data)
def rollback(b):

View File

@ -5,27 +5,23 @@
import multiprocessing as mp
from collections import defaultdict
from abci import ResponseCheckTx, ResponseDeliverTx
from bigchaindb import BigchainDB, App
from bigchaindb import App, BigchainDB
from bigchaindb.tendermint_utils import decode_transaction
CodeTypeOk = 0
from abci import CodeTypeOk
class ParallelValidationApp(App):
def __init__(self, bigchaindb=None, events_queue=None):
super().__init__(bigchaindb, events_queue)
def __init__(self, bigchaindb=None, events_queue=None, abci=None):
super().__init__(bigchaindb, events_queue, abci=abci)
self.parallel_validator = ParallelValidator()
self.parallel_validator.start()
def check_tx(self, raw_transaction):
return ResponseCheckTx(code=CodeTypeOk)
return self.abci.ResponseCheckTx(code=CodeTypeOk)
def deliver_tx(self, raw_transaction):
self.parallel_validator.validate(raw_transaction)
return ResponseDeliverTx(code=CodeTypeOk)
return self.abci.ResponseDeliverTx(code=CodeTypeOk)
def end_block(self, request_end_block):
result = self.parallel_validator.result(timeout=30)

View File

@ -5,6 +5,8 @@
import logging
import setproctitle
from abci import TmVersion, ABCI
import bigchaindb
from bigchaindb.lib import BigchainDB
from bigchaindb.core import App
@ -62,15 +64,26 @@ def start(args):
# We need to import this after spawning the web server
# because import ABCIServer will monkeypatch all sockets
# for gevent.
from abci import ABCIServer
from abci.server import ABCIServer
setproctitle.setproctitle('bigchaindb')
# Start the ABCIServer
abci = ABCI(TmVersion(bigchaindb.config['tendermint']['version']))
if args.experimental_parallel_validation:
app = ABCIServer(app=ParallelValidationApp(events_queue=exchange.get_publisher_queue()))
app = ABCIServer(
app=ParallelValidationApp(
abci=abci.types,
events_queue=exchange.get_publisher_queue(),
)
)
else:
app = ABCIServer(app=App(events_queue=exchange.get_publisher_queue()))
app = ABCIServer(
app=App(
abci=abci.types,
events_queue=exchange.get_publisher_queue(),
)
)
app.run()

View File

@ -1,18 +1,28 @@
import codecs
import base64
import binascii
import codecs
from abci import (ValidatorUpdate, PubKey)
from bigchaindb.common.exceptions import InvalidPublicKey
import bigchaindb
from abci import types_v0_22_8, types_v0_31_5, TmVersion
from bigchaindb.common.exceptions import InvalidPublicKey, BigchainDBError
def encode_validator(v):
ed25519_public_key = v['public_key']['value']
# NOTE: tendermint expects public to be encoded in go-amino format
pub_key = PubKey(type='ed25519',
data=bytes.fromhex(ed25519_public_key))
return ValidatorUpdate(pub_key=pub_key,
power=v['power'])
try:
version = TmVersion(bigchaindb.config["tendermint"]["version"])
except ValueError:
raise BigchainDBError('Invalid tendermint version, '
'check BigchainDB configuration file')
validator_update_t, pubkey_t = {
TmVersion.v0_22_8: (types_v0_22_8.Validator, types_v0_22_8.PubKey),
TmVersion.v0_31_5: (types_v0_31_5.ValidatorUpdate, types_v0_31_5.PubKey)
}[version]
pub_key = pubkey_t(type='ed25519', data=bytes.fromhex(ed25519_public_key))
return validator_update_t(pub_key=pub_key, power=v['power'])
def decode_validator(v):

View File

@ -6,4 +6,4 @@ __version__ = '2.1.0'
__short_version__ = '2.1'
# Supported Tendermint versions
__tm_supported_versions__ = ["0.31.5"]
__tm_supported_versions__ = ["0.31.5", "0.22.8"]

View File

@ -85,7 +85,7 @@ install_requires = [
'jsonschema~=2.5.1',
'pyyaml>=4.2b1',
'aiohttp~=3.0',
'bigchaindb-abci==0.7.1',
'bigchaindb-abci==1.0.1',
'setproctitle~=1.1.0',
'packaging~=18.0',
]

View File

@ -233,6 +233,12 @@ def merlin():
return generate_key_pair()
@pytest.fixture
def a():
from abci import types_v0_31_5
return types_v0_31_5
@pytest.fixture
def b():
from bigchaindb import BigchainDB
@ -434,11 +440,12 @@ def event_loop():
@pytest.fixture(scope='session')
def abci_server():
from abci import ABCIServer
from abci.server import ABCIServer
from abci import types_v0_31_5
from bigchaindb.core import App
from bigchaindb.utils import Process
app = ABCIServer(app=App())
app = ABCIServer(app=App(types_v0_31_5))
abci_proxy = Process(name='ABCI', target=app.run)
yield abci_proxy.start()
abci_proxy.terminate()

View File

@ -5,7 +5,7 @@
import pytest
import codecs
import abci as types
from abci import types_v0_31_5 as types
@pytest.fixture

View File

@ -6,23 +6,14 @@ import json
import pytest
import random
from abci import (
PubKey,
ResponseInitChain,
RequestInitChain,
RequestInfo,
RequestBeginBlock,
RequestEndBlock,
ValidatorUpdate,
)
from abci import types_v0_31_5 as types
from bigchaindb import App
from bigchaindb.backend.localmongodb import query
from bigchaindb.common.crypto import generate_key_pair
from bigchaindb.core import (CodeTypeOk,
CodeTypeError,
rollback
)
rollback)
from bigchaindb.elections.election import Election
from bigchaindb.lib import Block
from bigchaindb.migrations.chain_migration_election import ChainMigrationElection
@ -48,20 +39,20 @@ def generate_address():
def generate_validator():
pk, _ = generate_key_pair()
pub_key = PubKey(type='ed25519', data=pk.encode())
val = ValidatorUpdate(power=10, pub_key=pub_key)
pub_key = types.PubKey(type='ed25519', data=pk.encode())
val = types.ValidatorUpdate(power=10, pub_key=pub_key)
return val
def generate_init_chain_request(chain_id, vals=None):
vals = vals if vals is not None else [generate_validator()]
return RequestInitChain(validators=vals, chain_id=chain_id)
return types.RequestInitChain(validators=vals, chain_id=chain_id)
def test_init_chain_successfully_registers_chain(b):
def test_init_chain_successfully_registers_chain(a, b):
request = generate_init_chain_request('chain-XYZ')
res = App(b).init_chain(request)
assert res == ResponseInitChain()
res = App(a, b).init_chain(request)
assert res == types.ResponseInitChain()
chain = query.get_latest_abci_chain(b.connection)
assert chain == {'height': 0, 'chain_id': 'chain-XYZ', 'is_synced': True}
assert query.get_latest_block(b.connection) == {
@ -71,11 +62,11 @@ def test_init_chain_successfully_registers_chain(b):
}
def test_init_chain_ignores_invalid_init_chain_requests(b):
def test_init_chain_ignores_invalid_init_chain_requests(a, b):
validators = [generate_validator()]
request = generate_init_chain_request('chain-XYZ', validators)
res = App(b).init_chain(request)
assert res == ResponseInitChain()
res = App(a, b).init_chain(request)
assert res == types.ResponseInitChain()
validator_set = query.get_validator_set(b.connection)
@ -88,7 +79,7 @@ def test_init_chain_ignores_invalid_init_chain_requests(b):
]
for r in invalid_requests:
with pytest.raises(SystemExit):
App(b).init_chain(r)
App(a, b).init_chain(r)
# assert nothing changed - neither validator set, nor chain ID
new_validator_set = query.get_validator_set(b.connection)
assert new_validator_set == validator_set
@ -101,11 +92,11 @@ def test_init_chain_ignores_invalid_init_chain_requests(b):
}
def test_init_chain_recognizes_new_chain_after_migration(b):
def test_init_chain_recognizes_new_chain_after_migration(a, b):
validators = [generate_validator()]
request = generate_init_chain_request('chain-XYZ', validators)
res = App(b).init_chain(request)
assert res == ResponseInitChain()
res = App(a, b).init_chain(request)
assert res == types.ResponseInitChain()
validator_set = query.get_validator_set(b.connection)['validators']
@ -123,7 +114,7 @@ def test_init_chain_recognizes_new_chain_after_migration(b):
]
for r in invalid_requests:
with pytest.raises(SystemExit):
App(b).init_chain(r)
App(a, b).init_chain(r)
assert query.get_latest_abci_chain(b.connection) == {
'chain_id': 'chain-XYZ-migrated-at-height-1',
'is_synced': False,
@ -136,8 +127,8 @@ def test_init_chain_recognizes_new_chain_after_migration(b):
# completes the migration
request = generate_init_chain_request('chain-XYZ-migrated-at-height-1',
validators)
res = App(b).init_chain(request)
assert res == ResponseInitChain()
res = App(a, b).init_chain(request)
assert res == types.ResponseInitChain()
assert query.get_latest_abci_chain(b.connection) == {
'chain_id': 'chain-XYZ-migrated-at-height-1',
'is_synced': True,
@ -157,7 +148,7 @@ def test_init_chain_recognizes_new_chain_after_migration(b):
]
for r in invalid_requests:
with pytest.raises(SystemExit):
App(b).init_chain(r)
App(a, b).init_chain(r)
assert query.get_latest_abci_chain(b.connection) == {
'chain_id': 'chain-XYZ-migrated-at-height-1',
'is_synced': True,
@ -172,9 +163,9 @@ def test_init_chain_recognizes_new_chain_after_migration(b):
}
def test_info(b):
r = RequestInfo(version=__tm_supported_versions__[0])
app = App(b)
def test_info(a, b):
r = types.RequestInfo(version=__tm_supported_versions__[0])
app = App(a, b)
res = app.info(r)
assert res.last_block_height == 0
@ -187,7 +178,7 @@ def test_info(b):
# simulate a migration and assert the height is shifted
b.store_abci_chain(2, 'chain-XYZ')
app = App(b)
app = App(a, b)
b.store_block(Block(app_hash='2', height=2, transactions=[])._asdict())
res = app.info(r)
assert res.last_block_height == 0
@ -200,14 +191,14 @@ def test_info(b):
# it's always the latest migration that is taken into account
b.store_abci_chain(4, 'chain-XYZ-new')
app = App(b)
app = App(a, b)
b.store_block(Block(app_hash='4', height=4, transactions=[])._asdict())
res = app.info(r)
assert res.last_block_height == 0
assert res.last_block_app_hash == b'4'
def test_check_tx__signed_create_is_ok(b):
def test_check_tx__signed_create_is_ok(a, b):
from bigchaindb import App
from bigchaindb.models import Transaction
from bigchaindb.common.crypto import generate_key_pair
@ -219,12 +210,12 @@ def test_check_tx__signed_create_is_ok(b):
[([bob.public_key], 1)])\
.sign([alice.private_key])
app = App(b)
app = App(a, b)
result = app.check_tx(encode_tx_to_bytes(tx))
assert result.code == CodeTypeOk
def test_check_tx__unsigned_create_is_error(b):
def test_check_tx__unsigned_create_is_error(a, b):
from bigchaindb import App
from bigchaindb.models import Transaction
from bigchaindb.common.crypto import generate_key_pair
@ -235,12 +226,12 @@ def test_check_tx__unsigned_create_is_error(b):
tx = Transaction.create([alice.public_key],
[([bob.public_key], 1)])
app = App(b)
app = App(a, b)
result = app.check_tx(encode_tx_to_bytes(tx))
assert result.code == CodeTypeError
def test_deliver_tx__valid_create_updates_db_and_emits_event(b, init_chain_request):
def test_deliver_tx__valid_create_updates_db_and_emits_event(a, b, init_chain_request):
import multiprocessing as mp
from bigchaindb import App
from bigchaindb.models import Transaction
@ -254,17 +245,17 @@ def test_deliver_tx__valid_create_updates_db_and_emits_event(b, init_chain_reque
[([bob.public_key], 1)])\
.sign([alice.private_key])
app = App(b, events)
app = App(a, b, events)
app.init_chain(init_chain_request)
begin_block = RequestBeginBlock()
begin_block = types.RequestBeginBlock()
app.begin_block(begin_block)
result = app.deliver_tx(encode_tx_to_bytes(tx))
assert result.code == CodeTypeOk
app.end_block(RequestEndBlock(height=99))
app.end_block(types.RequestEndBlock(height=99))
app.commit()
assert b.get_transaction(tx.id).id == tx.id
block_event = events.get()
@ -278,7 +269,7 @@ def test_deliver_tx__valid_create_updates_db_and_emits_event(b, init_chain_reque
# next(unspent_outputs)
def test_deliver_tx__double_spend_fails(b, init_chain_request):
def test_deliver_tx__double_spend_fails(a, b, init_chain_request):
from bigchaindb import App
from bigchaindb.models import Transaction
from bigchaindb.common.crypto import generate_key_pair
@ -290,16 +281,16 @@ def test_deliver_tx__double_spend_fails(b, init_chain_request):
[([bob.public_key], 1)])\
.sign([alice.private_key])
app = App(b)
app = App(a, b)
app.init_chain(init_chain_request)
begin_block = RequestBeginBlock()
begin_block = types.RequestBeginBlock()
app.begin_block(begin_block)
result = app.deliver_tx(encode_tx_to_bytes(tx))
assert result.code == CodeTypeOk
app.end_block(RequestEndBlock(height=99))
app.end_block(types.RequestEndBlock(height=99))
app.commit()
assert b.get_transaction(tx.id).id == tx.id
@ -307,15 +298,15 @@ def test_deliver_tx__double_spend_fails(b, init_chain_request):
assert result.code == CodeTypeError
def test_deliver_transfer_tx__double_spend_fails(b, init_chain_request):
def test_deliver_transfer_tx__double_spend_fails(a, b, init_chain_request):
from bigchaindb import App
from bigchaindb.models import Transaction
from bigchaindb.common.crypto import generate_key_pair
app = App(b)
app = App(a, b)
app.init_chain(init_chain_request)
begin_block = RequestBeginBlock()
begin_block = types.RequestBeginBlock()
app.begin_block(begin_block)
alice = generate_key_pair()
@ -351,11 +342,11 @@ def test_deliver_transfer_tx__double_spend_fails(b, init_chain_request):
assert result.code == CodeTypeError
def test_end_block_return_validator_updates(b, init_chain_request):
app = App(b)
def test_end_block_return_validator_updates(a, b, init_chain_request):
app = App(a, b)
app.init_chain(init_chain_request)
begin_block = RequestBeginBlock()
begin_block = types.RequestBeginBlock()
app.begin_block(begin_block)
# generate a block containing a concluded validator election
@ -380,13 +371,13 @@ def test_end_block_return_validator_updates(b, init_chain_request):
app.block_transactions = votes
resp = app.end_block(RequestEndBlock(height=2))
resp = app.end_block(types.RequestEndBlock(height=2))
assert resp.validator_updates[0].power == new_validator['election']['power']
expected = bytes.fromhex(new_validator['election']['public_key']['value'])
assert expected == resp.validator_updates[0].pub_key.data
def test_store_pre_commit_state_in_end_block(b, alice, init_chain_request):
def test_store_pre_commit_state_in_end_block(a, b, alice, init_chain_request):
from bigchaindb import App
from bigchaindb.backend import query
from bigchaindb.models import Transaction
@ -396,13 +387,13 @@ def test_store_pre_commit_state_in_end_block(b, alice, init_chain_request):
asset={'msg': 'live long and prosper'})\
.sign([alice.private_key])
app = App(b)
app = App(a, b)
app.init_chain(init_chain_request)
begin_block = RequestBeginBlock()
begin_block = types.RequestBeginBlock()
app.begin_block(begin_block)
app.deliver_tx(encode_tx_to_bytes(tx))
app.end_block(RequestEndBlock(height=99))
app.end_block(types.RequestEndBlock(height=99))
resp = query.get_pre_commit_state(b.connection)
assert resp['height'] == 99
@ -410,17 +401,17 @@ def test_store_pre_commit_state_in_end_block(b, alice, init_chain_request):
app.begin_block(begin_block)
app.deliver_tx(encode_tx_to_bytes(tx))
app.end_block(RequestEndBlock(height=100))
app.end_block(types.RequestEndBlock(height=100))
resp = query.get_pre_commit_state(b.connection)
assert resp['height'] == 100
assert resp['transactions'] == [tx.id]
# simulate a chain migration and assert the height is shifted
b.store_abci_chain(100, 'new-chain')
app = App(b)
app = App(a, b)
app.begin_block(begin_block)
app.deliver_tx(encode_tx_to_bytes(tx))
app.end_block(RequestEndBlock(height=1))
app.end_block(types.RequestEndBlock(height=1))
resp = query.get_pre_commit_state(b.connection)
assert resp['height'] == 101
assert resp['transactions'] == [tx.id]
@ -510,43 +501,43 @@ def test_new_validator_set(b):
assert updated_validator_set == updated_validators
def test_info_aborts_if_chain_is_not_synced(b):
def test_info_aborts_if_chain_is_not_synced(a, b):
b.store_abci_chain(0, 'chain-XYZ', False)
with pytest.raises(SystemExit):
App(b).info(RequestInfo())
App(a, b).info(types.RequestInfo())
def test_check_tx_aborts_if_chain_is_not_synced(b):
def test_check_tx_aborts_if_chain_is_not_synced(a, b):
b.store_abci_chain(0, 'chain-XYZ', False)
with pytest.raises(SystemExit):
App(b).check_tx('some bytes')
App(a, b).check_tx('some bytes')
def test_begin_aborts_if_chain_is_not_synced(b):
def test_begin_aborts_if_chain_is_not_synced(a, b):
b.store_abci_chain(0, 'chain-XYZ', False)
with pytest.raises(SystemExit):
App(b).info(RequestBeginBlock())
App(a, b).info(types.RequestBeginBlock())
def test_deliver_tx_aborts_if_chain_is_not_synced(b):
def test_deliver_tx_aborts_if_chain_is_not_synced(a, b):
b.store_abci_chain(0, 'chain-XYZ', False)
with pytest.raises(SystemExit):
App(b).deliver_tx('some bytes')
App(a, b).deliver_tx('some bytes')
def test_end_block_aborts_if_chain_is_not_synced(b):
def test_end_block_aborts_if_chain_is_not_synced(a, b):
b.store_abci_chain(0, 'chain-XYZ', False)
with pytest.raises(SystemExit):
App(b).info(RequestEndBlock())
App(a, b).info(types.RequestEndBlock())
def test_commit_aborts_if_chain_is_not_synced(b):
def test_commit_aborts_if_chain_is_not_synced(a, b):
b.store_abci_chain(0, 'chain-XYZ', False)
with pytest.raises(SystemExit):
App(b).commit()
App(a, b).commit()

View File

@ -4,7 +4,7 @@
import codecs
import abci as types
from abci import types_v0_31_5 as types
import json
import pytest
@ -18,13 +18,13 @@ from io import BytesIO
@pytest.mark.bdb
def test_app(b, init_chain_request):
def test_app(a, b, init_chain_request):
from bigchaindb import App
from bigchaindb.tendermint_utils import calculate_hash
from bigchaindb.common.crypto import generate_key_pair
from bigchaindb.models import Transaction
app = App(b)
app = App(a, b)
p = ProtocolHandler(app)
data = p.process('info',
@ -146,10 +146,10 @@ def test_post_transaction_responses(tendermint_ws_url, b):
@pytest.mark.bdb
def test_exit_when_tm_ver_not_supported(b):
def test_exit_when_tm_ver_not_supported(a, b):
from bigchaindb import App
app = App(b)
app = App(a, b)
p = ProtocolHandler(app)
with pytest.raises(SystemExit):

View File

@ -220,6 +220,7 @@ def test_autoconfigure_read_both_from_file_and_env(monkeypatch, request):
'tendermint': {
'host': 'localhost',
'port': 26657,
'version': 'v0.31.5'
},
'log': {
'file': LOG_FILE,