From 1118ddffa2df7b112b79bd88db3d89b969d76348 Mon Sep 17 00:00:00 2001 From: vrde Date: Tue, 10 Jul 2018 13:51:59 +0200 Subject: [PATCH 1/5] Problem: performance issue on transaction lookup (#2378) Solution: create the correct index for transaction id. Close #2377 --- bigchaindb/backend/localmongodb/schema.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bigchaindb/backend/localmongodb/schema.py b/bigchaindb/backend/localmongodb/schema.py index a030afc5..58ee582f 100644 --- a/bigchaindb/backend/localmongodb/schema.py +++ b/bigchaindb/backend/localmongodb/schema.py @@ -55,7 +55,7 @@ def create_transactions_secondary_index(conn, dbname): logger.info('Create `transactions` secondary index.') # to query the transactions for a transaction id, this field is unique - conn.conn[dbname]['transactions'].create_index('transactions.id', + conn.conn[dbname]['transactions'].create_index('id', name='transaction_id') # secondary index for asset uuid, this field is unique From c7503f56891c521b1c8d9094c351e99b0f6aac82 Mon Sep 17 00:00:00 2001 From: Vanshdeep Singh Date: Tue, 10 Jul 2018 14:16:02 +0200 Subject: [PATCH 2/5] Problem: Support for latest Tendermint missing (#2375) Solution: Upgrade to py-abci 0.5.1 to use latest Tendermint --- Dockerfile-alpine | 2 +- Dockerfile-dev | 2 +- bigchaindb/__init__.py | 2 +- bigchaindb/tendermint/core.py | 50 ++++++++++++------- docker-compose.yml | 16 +++--- .../source/appendices/firewall-notes.md | 14 +++--- .../architecture.rst | 6 +-- .../node-on-kubernetes.rst | 2 +- .../source/server-reference/configuration.md | 4 +- docs/server/source/simple-network-setup.md | 10 ++-- k8s/bigchaindb/bigchaindb-ext-conn-svc.yaml | 4 +- k8s/bigchaindb/bigchaindb-ss.yaml | 6 +-- k8s/bigchaindb/bigchaindb-svc.yaml | 12 ++--- .../tendermint_container/Dockerfile | 2 +- k8s/configuration/config-map.yaml | 6 +-- k8s/nginx-http/container/Dockerfile | 2 +- k8s/nginx-http/nginx-http-dep.yaml | 2 +- k8s/nginx-https/container/Dockerfile | 2 +- k8s/nginx-https/nginx-https-dep.yaml | 2 +- k8s/nginx-https/nginx-https-svc.yaml | 4 +- k8s/scripts/functions | 6 +-- .../roles/bigchaindb/defaults/main.yml | 4 +- .../roles/bigchaindb/tasks/common.yml | 4 +- .../roles/tendermint/defaults/main.yml | 8 +-- .../roles/tendermint/files/config.toml | 6 +-- .../roles/tendermint/tasks/start.yml | 6 +-- .../tendermint/templates/start_tendermint.j2 | 4 +- setup.py | 2 +- tests/conftest.py | 4 +- tests/tendermint/test_core.py | 33 ++++++------ tests/tendermint/test_integration.py | 30 ++++++----- tmdata/config.toml | 6 +-- 32 files changed, 141 insertions(+), 122 deletions(-) diff --git a/Dockerfile-alpine b/Dockerfile-alpine index 66c95f73..4ed7a492 100644 --- a/Dockerfile-alpine +++ b/Dockerfile-alpine @@ -25,6 +25,6 @@ ENV BIGCHAINDB_WSSERVER_SCHEME ws ENV BIGCHAINDB_WSSERVER_ADVERTISED_HOST 0.0.0.0 ENV BIGCHAINDB_WSSERVER_ADVERTISED_SCHEME ws -ENV BIGCHAINDB_TENDERMINT_PORT 46657 +ENV BIGCHAINDB_TENDERMINT_PORT 26657 ARG backend RUN bigchaindb -y configure "$backend" \ No newline at end of file diff --git a/Dockerfile-dev b/Dockerfile-dev index db5c7070..2902a08f 100644 --- a/Dockerfile-dev +++ b/Dockerfile-dev @@ -25,7 +25,7 @@ ENV BIGCHAINDB_WSSERVER_SCHEME ws ENV BIGCHAINDB_WSSERVER_ADVERTISED_HOST 0.0.0.0 ENV BIGCHAINDB_WSSERVER_ADVERTISED_SCHEME ws -ENV BIGCHAINDB_TENDERMINT_PORT 46657 +ENV BIGCHAINDB_TENDERMINT_PORT 26657 ENV BIGCHAINDB_CI_ABCI ${abci_status} diff --git a/bigchaindb/__init__.py b/bigchaindb/__init__.py index b9f096ec..0991e511 100644 --- a/bigchaindb/__init__.py +++ b/bigchaindb/__init__.py @@ -61,7 +61,7 @@ config = { }, 'tendermint': { 'host': 'localhost', - 'port': 46657, + 'port': 26657, }, # FIXME: hardcoding to localmongodb for now 'database': _database_map['localmongodb'], diff --git a/bigchaindb/tendermint/core.py b/bigchaindb/tendermint/core.py index d82e0266..e6e43484 100644 --- a/bigchaindb/tendermint/core.py +++ b/bigchaindb/tendermint/core.py @@ -2,16 +2,28 @@ with Tendermint.""" import logging -from abci.application import BaseApplication, Result -from abci.types_pb2 import ResponseEndBlock, ResponseInfo, Validator +from abci.application import BaseApplication +from abci.types_pb2 import ( + ResponseInitChain, + ResponseInfo, + ResponseCheckTx, + ResponseBeginBlock, + ResponseDeliverTx, + ResponseEndBlock, + ResponseCommit, + Validator, + PubKey +) from bigchaindb.tendermint import BigchainDB from bigchaindb.tendermint.utils import (decode_transaction, - calculate_hash, - amino_encoded_public_key) + calculate_hash) from bigchaindb.tendermint.lib import Block, PreCommitState from bigchaindb.backend.query import PRE_COMMIT_ID + +CodeTypeOk = 0 +CodeTypeError = 1 logger = logging.getLogger(__name__) @@ -35,8 +47,9 @@ class App(BaseApplication): block = Block(app_hash='', height=0, transactions=[]) self.bigchaindb.store_block(block._asdict()) + return ResponseInitChain() - def info(self): + def info(self, request): """Return height of the latest committed block.""" r = ResponseInfo() block = self.bigchaindb.get_latest_block() @@ -61,11 +74,11 @@ class App(BaseApplication): if self.bigchaindb.is_valid_transaction(transaction): logger.debug('check_tx: VALID') logger.benchmark('CHECK_TX_END, tx_id:%s', transaction['id']) - return Result.ok() + return ResponseCheckTx(code=CodeTypeOk) else: logger.debug('check_tx: INVALID') logger.benchmark('CHECK_TX_END, tx_id:%s', transaction['id']) - return Result.error() + return ResponseCheckTx(code=CodeTypeError) def begin_block(self, req_begin_block): """Initialize list of transaction. @@ -79,6 +92,7 @@ class App(BaseApplication): self.block_txn_ids = [] self.block_transactions = [] + return ResponseBeginBlock() def deliver_tx(self, raw_transaction): """Validate the transaction before mutating the state. @@ -91,19 +105,21 @@ class App(BaseApplication): if not transaction: logger.debug('deliver_tx: INVALID') - return Result.error(log='Invalid transaction') + return ResponseDeliverTx(code=CodeTypeError) else: logger.debug('storing tx') self.block_txn_ids.append(transaction.id) self.block_transactions.append(transaction) - return Result.ok() + return ResponseDeliverTx(code=CodeTypeOk) - def end_block(self, height): + def end_block(self, request_end_block): """Calculate block hash using transaction ids and previous block hash to be stored in the next block. Args: height (int): new height of the chain.""" + + height = request_end_block.height self.new_height = height block_txn_hash = calculate_hash(self.block_txn_ids) block = self.bigchaindb.get_latest_block() @@ -126,15 +142,11 @@ class App(BaseApplication): transactions=self.block_txn_ids) logger.debug('Updating PreCommitState: %s', self.new_height) self.bigchaindb.store_pre_commit_state(pre_commit_state._asdict()) - - # NOTE: interface for `ResponseEndBlock` has be changed in the latest - # version of py-abci i.e. the validator updates should be return - # as follows: - # ResponseEndBlock(validator_updates=validator_updates) return ResponseEndBlock(validator_updates=validator_updates) def commit(self): """Store the new height and along with block hash.""" + data = self.block_txn_hash.encode('utf-8') # register a new block only when new transactions are received @@ -151,12 +163,16 @@ class App(BaseApplication): 'height=%s, txn ids=%s', data, self.new_height, self.block_txn_ids) logger.benchmark('COMMIT_BLOCK, height:%s', self.new_height) - return data + return ResponseCommit(data=data) def encode_validator(v): ed25519_public_key = v['pub_key']['data'] # NOTE: tendermint expects public to be encoded in go-amino format - pub_key = amino_encoded_public_key(ed25519_public_key) + + pub_key = PubKey(type='ed25519', + data=bytes.fromhex(ed25519_public_key)) + return Validator(pub_key=pub_key, + address=b'', power=v['power']) diff --git a/docker-compose.yml b/docker-compose.yml index 391e2c95..160d776c 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -34,26 +34,26 @@ services: BIGCHAINDB_WSSERVER_HOST: 0.0.0.0 BIGCHAINDB_WSSERVER_ADVERTISED_HOST: bigchaindb BIGCHAINDB_TENDERMINT_HOST: tendermint - BIGCHAINDB_TENDERMINT_PORT: 46657 + BIGCHAINDB_TENDERMINT_PORT: 26657 ports: - "9984:9984" - "9985:9985" - - "46658" + - "26658" healthcheck: - test: ["CMD", "bash", "-c", "curl http://bigchaindb:9984 && curl http://tendermint:46657/abci_query"] + test: ["CMD", "bash", "-c", "curl http://bigchaindb:9984 && curl http://tendermint:26657/abci_query"] interval: 3s timeout: 5s retries: 3 command: '.ci/entrypoint.sh' tendermint: - image: tendermint/tendermint:0.19.9 + image: tendermint/tendermint:0.22.0 # volumes: # - ./tmdata:/tendermint entrypoint: '' ports: - - "46656:46656" - - "46657:46657" - command: sh -c "tendermint init && tendermint node --consensus.create_empty_blocks=false --proxy_app=tcp://bigchaindb:46658" + - "26656:26656" + - "26657:26657" + command: sh -c "tendermint init && tendermint node --consensus.create_empty_blocks=false --proxy_app=tcp://bigchaindb:26658" bdb: image: busybox depends_on: @@ -64,7 +64,7 @@ services: # curl client to check the health of development env curl-client: image: appropriate/curl - command: /bin/sh -c "curl -s http://bigchaindb:9984/ > /dev/null && curl -s http://tendermint:46657/ > /dev/null" + command: /bin/sh -c "curl -s http://bigchaindb:9984/ > /dev/null && curl -s http://tendermint:26657/ > /dev/null" # BigchainDB setup to do acceptance testing with Python python-acceptance: diff --git a/docs/server/source/appendices/firewall-notes.md b/docs/server/source/appendices/firewall-notes.md index 2a2de78a..a84b3105 100644 --- a/docs/server/source/appendices/firewall-notes.md +++ b/docs/server/source/appendices/firewall-notes.md @@ -9,7 +9,7 @@ The following ports should expect unsolicited inbound traffic: 1. **Port 22** can expect inbound SSH (TCP) traffic from the node administrator (i.e. a small set of IP addresses). 1. **Port 9984** can expect inbound HTTP (TCP) traffic from BigchainDB clients sending transactions to the BigchainDB HTTP API. 1. **Port 9985** can expect inbound WebSocket traffic from BigchainDB clients. -1. **Port 46656** can expect inbound Tendermint P2P traffic from other Tendermint peers. +1. **Port 26656** can expect inbound Tendermint P2P traffic from other Tendermint peers. 1. **Port 9986** can expect inbound HTTP (TCP) traffic from clients accessing the Public Key of a Tendermint instance. All other ports should only get inbound traffic in response to specific requests from inside the node. @@ -56,17 +56,17 @@ Port 9985 is the default port for the [BigchainDB WebSocket Event Stream API](.. Port 9986 is the default port to access the Public Key of a Tendermint instance, it is used by a NGINX instance that runs with Tendermint instance(Pod), and only hosts the Public Key. -## Port 46656 +## Port 26656 -Port 46656 is the default port used by Tendermint Core to communicate with other instances of Tendermint Core (peers). +Port 26656 is the default port used by Tendermint Core to communicate with other instances of Tendermint Core (peers). -## Port 46657 +## Port 26657 -Port 46657 is the default port used by Tendermint Core for RPC traffic. BigchainDB nodes use that internally; they don't expect incoming traffic from the outside world on port 46657. +Port 26657 is the default port used by Tendermint Core for RPC traffic. BigchainDB nodes use that internally; they don't expect incoming traffic from the outside world on port 26657. -## Port 46658 +## Port 26658 -Port 46658 is the default port used by Tendermint Core for ABCI traffic. BigchainDB nodes use that internally; they don't expect incoming traffic from the outside world on port 46658. +Port 26658 is the default port used by Tendermint Core for ABCI traffic. BigchainDB nodes use that internally; they don't expect incoming traffic from the outside world on port 26658. ## Other Ports diff --git a/docs/server/source/production-deployment-template/architecture.rst b/docs/server/source/production-deployment-template/architecture.rst index 7778e45f..4d029f5c 100644 --- a/docs/server/source/production-deployment-template/architecture.rst +++ b/docs/server/source/production-deployment-template/architecture.rst @@ -56,7 +56,7 @@ BigchainDB Node | | | | v | | | - | "443" +----------+ "46656/9986" | + | "443" +----------+ "26656/9986" | | | "Rate" | | | +---------------------------+"Limiting"+-----------------------+ | | | | "Logic" | | | @@ -70,7 +70,7 @@ BigchainDB Node | v | v | | +-------------+ | +------------+ | | |"HTTPS" | | +------------------> |"Tendermint"| | - | |"Termination"| | | "9986" |"Service" | "46656" | + | |"Termination"| | | "9986" |"Service" | "26656" | | | | | | +-------+ | <----+ | | +-----+-------+ | | | +------------+ | | | | | | | | | @@ -205,6 +205,6 @@ In a multi-node deployment, Tendermint nodes/peers communicate with each other v the public ports exposed by the NGINX gateway. We use port **9986** (configurable) to allow tendermint nodes to access the public keys -of the peers and port **46656** (configurable) for the rest of the communications between +of the peers and port **26656** (configurable) for the rest of the communications between the peers. diff --git a/docs/server/source/production-deployment-template/node-on-kubernetes.rst b/docs/server/source/production-deployment-template/node-on-kubernetes.rst index 15f25e4a..b6fbeedd 100644 --- a/docs/server/source/production-deployment-template/node-on-kubernetes.rst +++ b/docs/server/source/production-deployment-template/node-on-kubernetes.rst @@ -674,7 +674,7 @@ To test the BigchainDB instance: $ curl -X GET http://bdb-instance-0:9986/pub_key.json - $ curl -X GET http://bdb-instance-0:46657/abci_info + $ curl -X GET http://bdb-instance-0:26657/abci_info $ wsc -er ws://bdb-instance-0:9985/api/v1/streams/valid_transactions diff --git a/docs/server/source/server-reference/configuration.md b/docs/server/source/server-reference/configuration.md index 1e9849fa..9807ebcf 100644 --- a/docs/server/source/server-reference/configuration.md +++ b/docs/server/source/server-reference/configuration.md @@ -440,7 +440,7 @@ consensus(Tendermint) backend that we are using: **Example using environment variables** ```text export BIGCHAINDB_TENDERMINT_HOST=tendermint -export BIGCHAINDB_TENDERMINT_PORT=46657 +export BIGCHAINDB_TENDERMINT_PORT=26657 ``` **Default values** @@ -448,6 +448,6 @@ export BIGCHAINDB_TENDERMINT_PORT=46657 ```js "tendermint": { "host": "localhost", - "port": 46657, + "port": 26657, } ``` diff --git a/docs/server/source/simple-network-setup.md b/docs/server/source/simple-network-setup.md index ac8d4960..a744d9f3 100644 --- a/docs/server/source/simple-network-setup.md +++ b/docs/server/source/simple-network-setup.md @@ -86,13 +86,13 @@ sudo mv tendermint /usr/local/bin ### Set Up the Firewall -Make sure to accept inbound connections on ports `9984`, `9985`, and `46656`. You might also want to add port `22` so that you can continue to access the machine via SSH. +Make sure to accept inbound connections on ports `9984`, `9985`, and `26656`. You might also want to add port `22` so that you can continue to access the machine via SSH. ``` sudo ufw allow 22/tcp sudo ufw allow 9984/tcp sudo ufw allow 9985/tcp -sudo ufw allow 46656/tcp +sudo ufw allow 26656/tcp sudo ufw enable ``` @@ -214,9 +214,9 @@ The Member must edit the `.tendermint/config/config.toml` file and make the foll create_empty_blocks = false ... -persistent_peers = "@:46656,\ -@:46656,\ -@:46656," +persistent_peers = "@:26656,\ +@:26656,\ +@:26656," ``` ## Member: Start MongoDB, BigchainDB and Tendermint diff --git a/k8s/bigchaindb/bigchaindb-ext-conn-svc.yaml b/k8s/bigchaindb/bigchaindb-ext-conn-svc.yaml index 85beabb4..5845e382 100644 --- a/k8s/bigchaindb/bigchaindb-ext-conn-svc.yaml +++ b/k8s/bigchaindb/bigchaindb-ext-conn-svc.yaml @@ -9,9 +9,9 @@ spec: ports: spec: ports: - - port: 46656 + - port: 26656 name: p2p - - port: 46657 + - port: 26657 name: pubkey - port: 9986 name: nginx diff --git a/k8s/bigchaindb/bigchaindb-ss.yaml b/k8s/bigchaindb/bigchaindb-ss.yaml index 04dbe86c..a61e037a 100644 --- a/k8s/bigchaindb/bigchaindb-ss.yaml +++ b/k8s/bigchaindb/bigchaindb-ss.yaml @@ -126,9 +126,9 @@ spec: - name: bdb-config-data mountPath: /tendermint_node_data ports: - - containerPort: 46656 + - containerPort: 26656 name: p2p - - containerPort: 46657 + - containerPort: 26657 name: rpc livenessProbe: exec: @@ -266,7 +266,7 @@ spec: - containerPort: 9985 protocol: TCP name: bdb-ws-port - - containerPort: 46658 + - containerPort: 26658 protocol: TCP name: bdb-abci-port volumeMounts: diff --git a/k8s/bigchaindb/bigchaindb-svc.yaml b/k8s/bigchaindb/bigchaindb-svc.yaml index 183d49e0..a5fea38d 100644 --- a/k8s/bigchaindb/bigchaindb-svc.yaml +++ b/k8s/bigchaindb/bigchaindb-svc.yaml @@ -17,16 +17,16 @@ spec: targetPort: 9985 name: bdb-ws-port protocol: TCP - - port: 46658 - targetPort: 46658 + - port: 26658 + targetPort: 26658 name: tm-abci-port protocol: TCP - - port: 46656 - targetPort: 46656 + - port: 26656 + targetPort: 26656 name: tm-p2p-port protocol: TCP - - port: 46657 - targetPort: 46657 + - port: 26657 + targetPort: 26657 name: tm-rpc-port protocol: TCP - port: 9986 diff --git a/k8s/bigchaindb/tendermint_container/Dockerfile b/k8s/bigchaindb/tendermint_container/Dockerfile index ed0e06a6..ab389f3c 100644 --- a/k8s/bigchaindb/tendermint_container/Dockerfile +++ b/k8s/bigchaindb/tendermint_container/Dockerfile @@ -6,5 +6,5 @@ RUN apk --update add bash COPY genesis.json.template /etc/tendermint/genesis.json COPY tendermint_entrypoint.bash / VOLUME /tendermint /tendermint_node_data -EXPOSE 46656 46657 +EXPOSE 26656 26657 ENTRYPOINT ["/tendermint_entrypoint.bash"] diff --git a/k8s/configuration/config-map.yaml b/k8s/configuration/config-map.yaml index d93074b6..7166b376 100644 --- a/k8s/configuration/config-map.yaml +++ b/k8s/configuration/config-map.yaml @@ -146,15 +146,15 @@ data: # bdb-abci-port is used by Tendermint Core for ABCI traffic. BigchainDB nodes # use that internally. - bdb-abci-port: "46658" + bdb-abci-port: "26658" # bdb-p2p-port is used by Tendermint Core to communicate with # other peers in the network. This port is accessible publicly. - bdb-p2p-port: "46656" + bdb-p2p-port: "26656" # bdb-rpc-port is used by Tendermint Core to rpc. BigchainDB nodes # use this port internally. - bbd-rpc-port: "46657" + bbd-rpc-port: "26657" # bdb-pub-key-access is the port number used to host/publish the # public key of the tendemrint node in this cluster. diff --git a/k8s/nginx-http/container/Dockerfile b/k8s/nginx-http/container/Dockerfile index 0cf6a05e..f0fad61b 100644 --- a/k8s/nginx-http/container/Dockerfile +++ b/k8s/nginx-http/container/Dockerfile @@ -7,5 +7,5 @@ RUN apt-get update \ && apt-get clean COPY nginx.conf.template /etc/nginx/nginx.conf COPY nginx_entrypoint.bash / -EXPOSE 80 27017 9986 46656 +EXPOSE 80 27017 9986 26656 ENTRYPOINT ["/nginx_entrypoint.bash"] diff --git a/k8s/nginx-http/nginx-http-dep.yaml b/k8s/nginx-http/nginx-http-dep.yaml index 1b523737..8d41a04f 100644 --- a/k8s/nginx-http/nginx-http-dep.yaml +++ b/k8s/nginx-http/nginx-http-dep.yaml @@ -79,7 +79,7 @@ spec: - containerPort: 9986 protocol: TCP name: bdb-pub-key - - containerPort: 46656 + - containerPort: 26656 protocol: TCP name: bdb-p2p-port livenessProbe: diff --git a/k8s/nginx-https/container/Dockerfile b/k8s/nginx-https/container/Dockerfile index 7606ae43..b2211ef9 100644 --- a/k8s/nginx-https/container/Dockerfile +++ b/k8s/nginx-https/container/Dockerfile @@ -8,5 +8,5 @@ RUN apt-get update \ COPY nginx.conf.threescale.template /etc/nginx/nginx-threescale.conf COPY nginx.conf.template /etc/nginx/nginx.conf COPY nginx_entrypoint.bash / -EXPOSE 80 443 27017 9986 46656 +EXPOSE 80 443 27017 9986 26656 ENTRYPOINT ["/nginx_entrypoint.bash"] diff --git a/k8s/nginx-https/nginx-https-dep.yaml b/k8s/nginx-https/nginx-https-dep.yaml index a28d685a..1e9da3a9 100644 --- a/k8s/nginx-https/nginx-https-dep.yaml +++ b/k8s/nginx-https/nginx-https-dep.yaml @@ -103,7 +103,7 @@ spec: - containerPort: 9986 protocol: TCP name: bdb-pub-key - - containerPort: 46656 + - containerPort: 26656 protocol: TCP name: bdb-p2p-port livenessProbe: diff --git a/k8s/nginx-https/nginx-https-svc.yaml b/k8s/nginx-https/nginx-https-svc.yaml index 1a79ce10..9a689f2b 100644 --- a/k8s/nginx-https/nginx-https-svc.yaml +++ b/k8s/nginx-https/nginx-https-svc.yaml @@ -25,8 +25,8 @@ spec: targetPort: 9986 name: tm-pub-key-access protocol: TCP - - port: 46656 - targetPort: 46656 + - port: 26656 + targetPort: 26656 protocol: TCP name: tm-p2p-port - port: 80 diff --git a/k8s/scripts/functions b/k8s/scripts/functions index c8d02f0e..ce683671 100755 --- a/k8s/scripts/functions +++ b/k8s/scripts/functions @@ -379,15 +379,15 @@ data: # bdb-abci-port is used by Tendermint Core for ABCI traffic. BigchainDB nodes # use that internally. - bdb-abci-port: "46658" + bdb-abci-port: "26658" # bdb-p2p-port is used by Tendermint Core to communicate with # other peers in the network. This port is accessible publicly. - bdb-p2p-port: "46656" + bdb-p2p-port: "26656" # bdb-rpc-port is used by Tendermint Core to rpc. BigchainDB nodes # use this port internally. - bdb-rpc-port: "46657" + bdb-rpc-port: "26657" # bdb-pub-key-access is the port number used to host/publish the # public key of the tendemrint node in this cluster. diff --git a/pkg/configuration/roles/bigchaindb/defaults/main.yml b/pkg/configuration/roles/bigchaindb/defaults/main.yml index 37afca6e..5203a544 100644 --- a/pkg/configuration/roles/bigchaindb/defaults/main.yml +++ b/pkg/configuration/roles/bigchaindb/defaults/main.yml @@ -38,6 +38,6 @@ mongodb_docker_name: "mongodb" tendermint_docker_name: "tendermint" bigchaindb_default_server_port: 9984 bigchaindb_default_ws_port: 9985 -bigchaindb_tendermint_port: 46657 +bigchaindb_tendermint_port: 26657 tendermint_abci_port: 45558 -bigchaindb_docker_net: "bigchaindb_docker_net" \ No newline at end of file +bigchaindb_docker_net: "bigchaindb_docker_net" diff --git a/pkg/configuration/roles/bigchaindb/tasks/common.yml b/pkg/configuration/roles/bigchaindb/tasks/common.yml index 3fc4da50..d944c06f 100644 --- a/pkg/configuration/roles/bigchaindb/tasks/common.yml +++ b/pkg/configuration/roles/bigchaindb/tasks/common.yml @@ -36,6 +36,6 @@ BIGCHAINDB_WSSERVER_HOST: 0.0.0.0 BIGCHAINDB_WSSERVER_PORT: 9985 BIGCHAINDB_TENDERMINT_HOST: 127.0.0.1 - BIGCHAINDB_TENDERMINT_PORT: 46657 + BIGCHAINDB_TENDERMINT_PORT: 26657 when: mdb_pchk.stdout| int != 0 and bdb_pchk.stdout| int == 0 and tm_pchk.stdout| int != 0 - tags: [bigchaindb] \ No newline at end of file + tags: [bigchaindb] diff --git a/pkg/configuration/roles/tendermint/defaults/main.yml b/pkg/configuration/roles/tendermint/defaults/main.yml index 2fff2d58..01225915 100644 --- a/pkg/configuration/roles/tendermint/defaults/main.yml +++ b/pkg/configuration/roles/tendermint/defaults/main.yml @@ -13,9 +13,9 @@ bigchaindb_docker_net: "bigchaindb_docker_net" tendermint_host_mount_dir: "{{ home_dir }}/tendermint_docker" tendermint_host_mount_config_dir: "{{ home_dir }}/tendermint_config" tendermint_home: /tendermint/config -tendermint_p2p_port: 46656 -tendermint_rpc_port: 46657 -tendermint_abci_port: 46658 +tendermint_p2p_port: 26656 +tendermint_rpc_port: 26657 +tendermint_abci_port: 26658 directories: - "{{ tendermint_home }}" @@ -24,4 +24,4 @@ tendermint_conf_files: [ { src: "genesis.json", dest: "{{ tendermint_home }}/genesis.json"}, { src: "config.toml", dest: "{{ tendermint_home}}/config.toml"}, { src: "access_pub_key.conf", dest: "/etc/nginx/conf.d/access_pub_key.conf"} -] \ No newline at end of file +] diff --git a/pkg/configuration/roles/tendermint/files/config.toml b/pkg/configuration/roles/tendermint/files/config.toml index bc6a1fe3..c2ed6792 100644 --- a/pkg/configuration/roles/tendermint/files/config.toml +++ b/pkg/configuration/roles/tendermint/files/config.toml @@ -5,7 +5,7 @@ # TCP or UNIX socket address of the ABCI application, # or the name of an ABCI application compiled in with the Tendermint binary -proxy_app = "tcp://127.0.0.1:46658" +proxy_app = "tcp://127.0.0.1:26658" # A custom human readable name for this node moniker = "d16137710ef8" @@ -51,7 +51,7 @@ filter_peers = false [rpc] # TCP or UNIX socket address for the RPC server to listen on -laddr = "tcp://0.0.0.0:46657" +laddr = "tcp://0.0.0.0:26657" # TCP or UNIX socket address for the gRPC server to listen on # NOTE: This server only supports /broadcast_tx_commit @@ -64,7 +64,7 @@ unsafe = false [p2p] # Address to listen for incoming connections -laddr = "tcp://0.0.0.0:46656" +laddr = "tcp://0.0.0.0:26656" # Comma separated list of seed nodes to connect to seeds = "" diff --git a/pkg/configuration/roles/tendermint/tasks/start.yml b/pkg/configuration/roles/tendermint/tasks/start.yml index e9128cf2..3f253d50 100644 --- a/pkg/configuration/roles/tendermint/tasks/start.yml +++ b/pkg/configuration/roles/tendermint/tasks/start.yml @@ -56,8 +56,8 @@ mv /tendermint_config/node_key"{{ item|string }}".json /tendermint/config/node_key.json && peers=() && for i in $( seq {{ stack_size }} );do peers+=($(cat /tendermint_config/node_id$i)@"{{ tendermint_docker_name }}$i:{{ tendermint_p2p_port }}");done && peers=$(IFS=","; echo "${peers[*]}") && echo $peers && - tendermint node --p2p.persistent_peers="$peers" --p2p.laddr "tcp://"{{ tendermint_docker_name }}{{ item|string }}":46656" - --proxy_app="tcp://"{{ bigchaindb_docker_name }}{{ item|string }}":46658" --consensus.create_empty_blocks=false --p2p.pex=false' + tendermint node --p2p.persistent_peers="$peers" --p2p.laddr "tcp://"{{ tendermint_docker_name }}{{ item|string }}":26656" + --proxy_app="tcp://"{{ bigchaindb_docker_name }}{{ item|string }}":26658" --consensus.create_empty_blocks=false --p2p.pex=false' state: started keep_volumes: true with_sequence: start=1 end="{{ stack_size|int }}" stride=1 @@ -87,4 +87,4 @@ - import_tasks: common.yml when: stack_type|lower == "local" - tags: [tendermint] \ No newline at end of file + tags: [tendermint] diff --git a/pkg/configuration/roles/tendermint/templates/start_tendermint.j2 b/pkg/configuration/roles/tendermint/templates/start_tendermint.j2 index 15aeb619..35961df2 100644 --- a/pkg/configuration/roles/tendermint/templates/start_tendermint.j2 +++ b/pkg/configuration/roles/tendermint/templates/start_tendermint.j2 @@ -94,5 +94,5 @@ done peers=$(IFS=','; echo "${peers[*]}") echo "INFO: starting tendermint..." -/usr/local/bin/tendermint node --home "/tendermint" --p2p.persistent_peers="$peers" --proxy_app="tcp://0.0.0.0:46658" --consensus.create_empty_blocks=false --p2p.pex=false -{% endraw %} \ No newline at end of file +/usr/local/bin/tendermint node --home "/tendermint" --p2p.persistent_peers="$peers" --proxy_app="tcp://0.0.0.0:26658" --consensus.create_empty_blocks=false --p2p.pex=false +{% endraw %} diff --git a/setup.py b/setup.py index 5e1f8874..02286b47 100644 --- a/setup.py +++ b/setup.py @@ -87,7 +87,7 @@ install_requires = [ 'pyyaml~=3.12', 'aiohttp~=2.3', 'python-rapidjson-schema==0.1.1', - 'bigchaindb-abci==0.4.5', + 'bigchaindb-abci==0.5.1', 'setproctitle~=1.1.0', ] diff --git a/tests/conftest.py b/tests/conftest.py index d17afd5c..d1ad5368 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -109,7 +109,7 @@ def _configure_bigchaindb(request): 'database': bigchaindb._database_map[backend], 'tendermint': { 'host': 'localhost', - 'port': 46657, + 'port': 26657, } } config['database']['name'] = test_db_name @@ -445,7 +445,7 @@ def tendermint_host(): @pytest.fixture def tendermint_port(): - return int(os.getenv('BIGCHAINDB_TENDERMINT_PORT', 46657)) + return int(os.getenv('BIGCHAINDB_TENDERMINT_PORT', 26657)) @pytest.fixture diff --git a/tests/tendermint/test_core.py b/tests/tendermint/test_core.py index ef34d4b6..cb67a065 100644 --- a/tests/tendermint/test_core.py +++ b/tests/tendermint/test_core.py @@ -1,7 +1,12 @@ import json import pytest -from abci.types_pb2 import RequestBeginBlock +from abci.types_pb2 import ( + RequestBeginBlock, + RequestEndBlock +) + +from bigchaindb.tendermint.core import CodeTypeOk, CodeTypeError pytestmark = [pytest.mark.tendermint, pytest.mark.bdb] @@ -25,7 +30,7 @@ def test_check_tx__signed_create_is_ok(b): app = App(b) result = app.check_tx(encode_tx_to_bytes(tx)) - assert result.is_ok() + assert result.code == CodeTypeOk def test_check_tx__unsigned_create_is_error(b): @@ -41,7 +46,7 @@ def test_check_tx__unsigned_create_is_error(b): app = App(b) result = app.check_tx(encode_tx_to_bytes(tx)) - assert result.is_error() + assert result.code == CodeTypeError @pytest.mark.bdb @@ -64,9 +69,9 @@ def test_deliver_tx__valid_create_updates_db(b): app.begin_block(begin_block) result = app.deliver_tx(encode_tx_to_bytes(tx)) - assert result.is_ok() + assert result.code == CodeTypeOk - app.end_block(99) + app.end_block(RequestEndBlock(height=99)) app.commit() assert b.get_transaction(tx.id).id == tx.id @@ -97,14 +102,14 @@ def test_deliver_tx__double_spend_fails(b): app.begin_block(begin_block) result = app.deliver_tx(encode_tx_to_bytes(tx)) - assert result.is_ok() + assert result.code == CodeTypeOk - app.end_block(99) + app.end_block(RequestEndBlock(height=99)) app.commit() assert b.get_transaction(tx.id).id == tx.id result = app.deliver_tx(encode_tx_to_bytes(tx)) - assert result.is_error() + assert result.code == CodeTypeError def test_deliver_transfer_tx__double_spend_fails(b): @@ -132,7 +137,7 @@ def test_deliver_transfer_tx__double_spend_fails(b): .sign([alice.private_key]) result = app.deliver_tx(encode_tx_to_bytes(tx)) - assert result.is_ok() + assert result.code == CodeTypeOk tx_transfer = Transaction.transfer(tx.to_inputs(), [([bob.public_key], 1)], @@ -140,7 +145,7 @@ def test_deliver_transfer_tx__double_spend_fails(b): .sign([alice.private_key]) result = app.deliver_tx(encode_tx_to_bytes(tx_transfer)) - assert result.is_ok() + assert result.code == CodeTypeOk double_spend = Transaction.transfer(tx.to_inputs(), [([carly.public_key], 1)], @@ -148,7 +153,7 @@ def test_deliver_transfer_tx__double_spend_fails(b): .sign([alice.private_key]) result = app.deliver_tx(encode_tx_to_bytes(double_spend)) - assert result.is_error() + assert result.code == CodeTypeError def test_end_block_return_validator_updates(b): @@ -170,7 +175,7 @@ def test_end_block_return_validator_updates(b): 'update_id': VALIDATOR_UPDATE_ID} query.store_validator_update(b.connection, validator_update) - resp = app.end_block(99) + resp = app.end_block(RequestEndBlock(height=99)) assert resp.validator_updates[0] == encode_validator(validator) updates = b.get_validator_update() @@ -194,7 +199,7 @@ def test_store_pre_commit_state_in_end_block(b, alice): begin_block = RequestBeginBlock() app.begin_block(begin_block) app.deliver_tx(encode_tx_to_bytes(tx)) - app.end_block(99) + app.end_block(RequestEndBlock(height=99)) resp = query.get_pre_commit_state(b.connection, PRE_COMMIT_ID) assert resp['commit_id'] == PRE_COMMIT_ID @@ -203,7 +208,7 @@ def test_store_pre_commit_state_in_end_block(b, alice): app.begin_block(begin_block) app.deliver_tx(encode_tx_to_bytes(tx)) - app.end_block(100) + app.end_block(RequestEndBlock(height=100)) resp = query.get_pre_commit_state(b.connection, PRE_COMMIT_ID) assert resp['commit_id'] == PRE_COMMIT_ID assert resp['height'] == 100 diff --git a/tests/tendermint/test_integration.py b/tests/tendermint/test_integration.py index c1d86684..edb8cfa2 100644 --- a/tests/tendermint/test_integration.py +++ b/tests/tendermint/test_integration.py @@ -4,8 +4,7 @@ import pytest from abci.server import ProtocolHandler -from abci.encoding import read_message -from abci.messages import to_request_deliver_tx, to_request_check_tx +from abci.encoding import read_messages from copy import deepcopy from io import BytesIO @@ -22,14 +21,14 @@ def test_app(tb): app = App(b) p = ProtocolHandler(app) - data = p.process('info', None) - res, err = read_message(BytesIO(data), types.Response) + data = p.process('info', types.Request(info=types.RequestInfo(version='2'))) + res = next(read_messages(BytesIO(data), types.Response)) assert res assert res.info.last_block_app_hash == b'' assert res.info.last_block_height == 0 assert not b.get_latest_block() - p.process('init_chain', None) + p.process('init_chain', types.Request(init_chain=types.RequestInitChain())) block0 = b.get_latest_block() assert block0 assert block0['height'] == 0 @@ -42,9 +41,9 @@ def test_app(tb): .sign([alice.private_key]) etxn = json.dumps(tx.to_dict()).encode('utf8') - r = to_request_check_tx(etxn) + r = types.Request(check_tx=types.RequestCheckTx(tx=etxn)) data = p.process('check_tx', r) - res, err = read_message(BytesIO(data), types.Response) + res = next(read_messages(BytesIO(data), types.Response)) assert res assert res.check_tx.code == 0 @@ -52,25 +51,24 @@ def test_app(tb): r.begin_block.hash = b'' p.process('begin_block', r) - r = to_request_deliver_tx(etxn) + r = types.Request(deliver_tx=types.RequestDeliverTx(tx=etxn)) data = p.process('deliver_tx', r) - res, err = read_message(BytesIO(data), types.Response) + res = next(read_messages(BytesIO(data), types.Response)) assert res assert res.deliver_tx.code == 0 new_block_txn_hash = calculate_hash([tx.id]) - r = types.Request() - r.end_block.height = 1 + r = types.Request(end_block=types.RequestEndBlock(height=1)) data = p.process('end_block', r) - res, err = read_message(BytesIO(data), types.Response) + res = next(read_messages(BytesIO(data), types.Response)) assert res assert 'end_block' == res.WhichOneof('value') new_block_hash = calculate_hash([block0['app_hash'], new_block_txn_hash]) data = p.process('commit', None) - res, err = read_message(BytesIO(data), types.Response) + res = next(read_messages(BytesIO(data), types.Response)) assert res.commit.data == new_block_hash.encode('utf-8') assert b.get_transaction(tx.id).id == tx.id @@ -89,7 +87,7 @@ def test_app(tb): p.process('end_block', r) data = p.process('commit', None) - res, err = read_message(BytesIO(data), types.Response) + res = next(read_messages(BytesIO(data), types.Response)) assert res.commit.data == new_block_hash.encode('utf-8') block0 = b.get_latest_block() @@ -109,8 +107,8 @@ def test_upsert_validator(b, alice): import time conn = connect() - public_key = '1718D2DBFF00158A0852A17A01C78F4DCF3BA8E4FB7B8586807FAC182A535034' power = 1 + public_key = '9B3119650DF82B9A5D8A12E38953EA47475C09F0C48A4E6A0ECE182944B24403' validator = {'pub_key': {'type': 'AC26791624DE60', 'data': public_key}, @@ -133,7 +131,7 @@ def test_upsert_validator(b, alice): validators = [(v['pub_key']['value'], v['voting_power']) for v in validators] public_key64 = public_key_to_base64(public_key) - assert ((public_key64, power) in validators) + assert ((public_key64, str(power)) in validators) @pytest.mark.abci diff --git a/tmdata/config.toml b/tmdata/config.toml index 75ac5e26..e03a0796 100644 --- a/tmdata/config.toml +++ b/tmdata/config.toml @@ -1,7 +1,7 @@ # This is a TOML config file. # For more information, see https://github.com/toml-lang/toml -proxy_app = "tcp://bigchaindb:46658" +proxy_app = "tcp://bigchaindb:26658" moniker = "anonymous" fast_sync = true db_backend = "leveldb" @@ -11,8 +11,8 @@ log_level = "state:debug,*:error" create_empty_blocks = false [rpc] -laddr = "tcp://0.0.0.0:46657" +laddr = "tcp://0.0.0.0:26657" [p2p] -laddr = "tcp://0.0.0.0:46656" +laddr = "tcp://0.0.0.0:26656" seeds = "" From 10953a64e7fe2499999fe3f8ce5f5141e03966f1 Mon Sep 17 00:00:00 2001 From: Troy McConaghy Date: Tue, 10 Jul 2018 14:19:45 +0200 Subject: [PATCH 3/5] Problem: "IPDB Transaction Spec" referenced in .../schema/README.md (#2376) Solution: Update `bigchaindb/common/schema/README.md` to link to the BigchainDB Transactions Specs instead. --- bigchaindb/common/schema/README.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/bigchaindb/common/schema/README.md b/bigchaindb/common/schema/README.md index ab8d8e9d..1de4bd41 100644 --- a/bigchaindb/common/schema/README.md +++ b/bigchaindb/common/schema/README.md @@ -13,9 +13,9 @@ The aim is to provide: ## Sources The files defining the JSON Schema for transactions (`transaction_*.yaml`) -are based on the [IPDB Transaction Spec](https://github.com/ipdb/ipdb-tx-spec). +are based on the [BigchainDB Transactions Specs](https://github.com/bigchaindb/BEPs/tree/master/tx-specs). If you want to add a new transaction version, -you must add it to the IPDB Transaction Spec first. +you must write a spec for it first. (You can't change the JSON Schema files for old versions. Those were used to validate old transactions and are needed to re-check those transactions.) From f243b252ca25885b8681365b239b68686ed9754a Mon Sep 17 00:00:00 2001 From: Zachary Bowen Date: Tue, 10 Jul 2018 17:30:00 +0200 Subject: [PATCH 4/5] Problem: we don't have any tests to cover malicious strings (#2334) * Problem: we don't have any tests for malicious strings Solution: Use a parameterized test that fuzzes over a library of potentially hazardous strings * Needed to update the Dockerfile to install blns * Removed unnecessary import from Dockerfile-dev * Made the conditions under which the test expects an error more explicit * Improved reporting in case of a bad status code * Removed some over-zealous assertions. * Removed `sent_transaction` from assertation error response * *Problem: blns was removed as a dependency while fixing a merge conflict Solution: added it back to the Dockerfile * *Problem: made a typo when adding blns to the Dockerfile Solution: fixed the typo --- Dockerfile-dev | 2 +- acceptance/python/Dockerfile | 3 +- acceptance/python/src/test_naughty_strings.py | 91 +++++++++++++++++++ docker-compose.yml | 2 - 4 files changed, 94 insertions(+), 4 deletions(-) create mode 100644 acceptance/python/src/test_naughty_strings.py diff --git a/Dockerfile-dev b/Dockerfile-dev index 2902a08f..05e4bf98 100644 --- a/Dockerfile-dev +++ b/Dockerfile-dev @@ -33,5 +33,5 @@ RUN mkdir -p /usr/src/app COPY . /usr/src/app/ WORKDIR /usr/src/app RUN pip install --no-cache-dir --process-dependency-links -e .[dev] -RUN bigchaindb -y configure "$backend" +RUN bigchaindb -y configure diff --git a/acceptance/python/Dockerfile b/acceptance/python/Dockerfile index 9e06a7b0..b01105cb 100644 --- a/acceptance/python/Dockerfile +++ b/acceptance/python/Dockerfile @@ -5,4 +5,5 @@ RUN pip install --upgrade \ pycco \ websocket-client~=0.47.0 \ pytest~=3.0 \ - bigchaindb-driver==0.5.0 + bigchaindb-driver==0.5.0 \ + blns \ No newline at end of file diff --git a/acceptance/python/src/test_naughty_strings.py b/acceptance/python/src/test_naughty_strings.py new file mode 100644 index 00000000..54664183 --- /dev/null +++ b/acceptance/python/src/test_naughty_strings.py @@ -0,0 +1,91 @@ +# ## Testing potentially hazardous strings +# This test uses a library of `naughty` strings (code injections, weird unicode chars., etc.) as both keys and values. +# We look for either a successful tx, or in the case that we use a naughty string as a key, and it violates some key +# constraints, we expect to receive a well formatted error message. + +# ## Imports +# We need some utils from the `os` package, we will interact with +# env variables. +import os + +# Since the naughty strings get encoded and decoded in odd ways, we'll use a regex to sweep those details under the rug. +import re + +# We'll use a nice library of naughty strings... +from blns import blns + +# And parameterize our test so each one is treated as a separate test case +import pytest + +# For this test case we import and use the Python Driver. +from bigchaindb_driver import BigchainDB +from bigchaindb_driver.crypto import generate_keypair +from bigchaindb_driver.exceptions import BadRequest + +naughty_strings = blns.all() + + +# This is our base test case, but we'll reuse it to send naughty strings as both keys and values. +def send_naughty_tx(asset, metadata): + # ## Set up a connection to BigchainDB + # Check [test_basic.py](./test_basic.html) to get some more details + # about the endpoint. + bdb = BigchainDB(os.environ.get('BIGCHAINDB_ENDPOINT')) + + # Here's Alice. + alice = generate_keypair() + + # Alice is in a naughty mood today, so she creates a tx with some naughty strings + prepared_transaction = bdb.transactions.prepare( + operation='CREATE', + signers=alice.public_key, + asset=asset, + metadata=metadata) + + # She fulfills the transaction + fulfilled_transaction = bdb.transactions.fulfill( + prepared_transaction, + private_keys=alice.private_key) + + # The fulfilled tx gets sent to the BDB network + try: + sent_transaction = bdb.transactions.send(fulfilled_transaction) + except BadRequest as e: + sent_transaction = e + + # If her key contained a '.', began with a '$', or contained a NUL character + regex = '.*\..*|\$.*|.*\x00.*' + key = next(iter(metadata)) + if re.match(regex, key): + # Then she expects a nicely formatted error code + status_code = sent_transaction.status_code + error = sent_transaction.error + regex = '\{"message":"Invalid transaction \\(ValidationError\\): Invalid key name .* in asset object. ' \ + 'The key name cannot contain characters .* or null characters","status":400\}\n' + assert status_code == 400 + assert re.fullmatch(regex, error), sent_transaction + # Otherwise, she expects to see her transaction in the database + elif 'id' in sent_transaction.keys(): + tx_id = sent_transaction['id'] + assert bdb.transactions.retrieve(tx_id) + # If neither condition was true, then something weird happened... + else: + raise TypeError(sent_transaction) + + +@pytest.mark.parametrize("naughty_string", naughty_strings, ids=naughty_strings) +def test_naughty_keys(naughty_string): + + asset = {'data': {naughty_string: 'nice_value'}} + metadata = {naughty_string: 'nice_value'} + + send_naughty_tx(asset, metadata) + + +@pytest.mark.parametrize("naughty_string", naughty_strings, ids=naughty_strings) +def test_naughty_values(naughty_string): + + asset = {'data': {'nice_key': naughty_string}} + metadata = {'nice_key': naughty_string} + + send_naughty_tx(asset, metadata) diff --git a/docker-compose.yml b/docker-compose.yml index 160d776c..30bb23b6 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -15,8 +15,6 @@ services: build: context: . dockerfile: Dockerfile-dev - args: - backend: localmongodb volumes: - ./bigchaindb:/usr/src/app/bigchaindb - ./tests:/usr/src/app/tests From 4d2e58416c0e68838ae2dca313535cb5fc0085b5 Mon Sep 17 00:00:00 2001 From: Zachary Bowen Date: Tue, 10 Jul 2018 17:34:51 +0200 Subject: [PATCH 5/5] Problem: Bigchain class is unused and redundant (#2366) * Problem: core.py contains an unused class, `Bigchain` Solution: Remove core.py. Refactor BigchainDB Class to remove inheritance from Bigchain. * Problem: core.py contains an unused class, `Bigchain` Solution: Remove core.py. Refactor BigchainDB Class to remove inheritance from Bigchain. * Fixed flake8 complaint about too many blank lines * Attempting to fix Sphinx docs. This may result in some redundant commits, as I don't know what I'm doing, and I can't experiment without running the CI... Sorry in advance! * Attempting to fix Sphinx docs. This may result in some redundant commits, as I don't know what I'm doing, and I can't experiment without running the CI... Sorry in advance! * Updating from master changed BigchainDB.process_post_response to a private method, so I had to align with that. * Fixed a couple stale references to bigchaindb.Bigchain in docstrings * Missed a reference to `Bigchain` in a patch call... * Merge with master and re-do some changes --- bigchaindb/README.md | 4 +- bigchaindb/__init__.py | 2 +- bigchaindb/commands/bigchaindb.py | 4 +- bigchaindb/core.py | 434 ------------------ bigchaindb/models.py | 6 +- bigchaindb/tendermint/lib.py | 126 ++++- bigchaindb/web/server.py | 4 +- .../generate_http_server_api_documentation.py | 2 +- .../appendices/tendermint-integration.rst | 1 + tests/commands/test_commands.py | 2 +- tests/db/test_bigchain_api.py | 14 +- tests/test_config_utils.py | 3 +- tests/test_core.py | 14 +- tests/web/test_transactions.py | 14 +- 14 files changed, 156 insertions(+), 474 deletions(-) diff --git a/bigchaindb/README.md b/bigchaindb/README.md index 59fbe77c..dbb08afb 100644 --- a/bigchaindb/README.md +++ b/bigchaindb/README.md @@ -4,9 +4,9 @@ A high-level description of the files and subdirectories of BigchainDB. ## Files -### [`core.py`](./core.py) +### [`tendermint/lib.py`](./tendermint/lib.py) -The `Bigchain` class is defined here. Most node-level operations and database interactions are found in this file. This is the place to start if you are interested in implementing a server API, since many of these class methods concern BigchainDB interacting with the outside world. +The `BigchainDB` class is defined here. Most node-level operations and database interactions are found in this file. This is the place to start if you are interested in implementing a server API, since many of these class methods concern BigchainDB interacting with the outside world. ### [`models.py`](./models.py) diff --git a/bigchaindb/__init__.py b/bigchaindb/__init__.py index 0991e511..011737fa 100644 --- a/bigchaindb/__init__.py +++ b/bigchaindb/__init__.py @@ -84,5 +84,5 @@ config = { # the user wants to reconfigure the node. Check ``bigchaindb.config_utils`` # for more info. _config = copy.deepcopy(config) -from bigchaindb.core import Bigchain # noqa +from bigchaindb.tendermint import BigchainDB # noqa from bigchaindb.version import __version__ # noqa diff --git a/bigchaindb/commands/bigchaindb.py b/bigchaindb/commands/bigchaindb.py index 72d31bce..188cb7b3 100644 --- a/bigchaindb/commands/bigchaindb.py +++ b/bigchaindb/commands/bigchaindb.py @@ -97,7 +97,7 @@ def run_configure(args): def run_upsert_validator(args): """Store validators which should be synced with Tendermint""" - b = bigchaindb.tendermint.lib.BigchainDB() + b = bigchaindb.tendermint.BigchainDB() public_key = public_key_from_base64(args.public_key) validator = {'pub_key': {'type': 'ed25519', 'data': public_key}, @@ -113,7 +113,7 @@ def run_upsert_validator(args): def _run_init(): - bdb = bigchaindb.tendermint.lib.BigchainDB() + bdb = bigchaindb.tendermint.BigchainDB() schema.init_database(connection=bdb.connection) diff --git a/bigchaindb/core.py b/bigchaindb/core.py index 14dce4fe..e69de29b 100644 --- a/bigchaindb/core.py +++ b/bigchaindb/core.py @@ -1,434 +0,0 @@ -from bigchaindb import exceptions as core_exceptions - -import bigchaindb - -from bigchaindb import backend, config_utils -from bigchaindb.consensus import BaseConsensusRules -from bigchaindb.models import Transaction - - -class Bigchain(object): - """Bigchain API - - Create, read, sign, write transactions to the database - """ - - BLOCK_INVALID = 'invalid' - """return if a block is invalid""" - - BLOCK_VALID = TX_VALID = 'valid' - """return if a block is valid, or tx is in valid block""" - - BLOCK_UNDECIDED = TX_UNDECIDED = 'undecided' - """return if block is undecided, or tx is in undecided block""" - - TX_IN_BACKLOG = 'backlog' - """return if transaction is in backlog""" - - def __init__(self, connection=None): - """Initialize the Bigchain instance - - A Bigchain instance has several configuration parameters (e.g. host). - If a parameter value is passed as an argument to the Bigchain - __init__ method, then that is the value it will have. - Otherwise, the parameter value will come from an environment variable. - If that environment variable isn't set, then the value - will come from the local configuration file. And if that variable - isn't in the local configuration file, then the parameter will have - its default value (defined in bigchaindb.__init__). - - Args: - connection (:class:`~bigchaindb.backend.connection.Connection`): - A connection to the database. - """ - consensusPlugin = bigchaindb.config.get('consensus_plugin') - - if consensusPlugin: - self.consensus = config_utils.load_consensus_plugin(consensusPlugin) - else: - self.consensus = BaseConsensusRules - - def delete_transaction(self, *transaction_id): - """Delete a transaction from the backlog. - - Args: - *transaction_id (str): the transaction(s) to delete - - Returns: - The database response. - """ - - return backend.query.delete_transaction(self.connection, *transaction_id) - - def get_stale_transactions(self): - """Get a cursor of stale transactions. - - Transactions are considered stale if they have been assigned a node, but are still in the - backlog after some amount of time specified in the configuration - """ - - return backend.query.get_stale_transactions(self.connection) - - def validate_transaction(self, transaction): - """Validate a transaction. - - Args: - transaction (Transaction): transaction to validate. - - Returns: - The transaction if the transaction is valid else it raises an - exception describing the reason why the transaction is invalid. - """ - - return self.consensus.validate_transaction(self, transaction) - - def is_new_transaction(self, txid, exclude_block_id=None): - """Return True if the transaction does not exist in any - VALID or UNDECIDED block. Return False otherwise. - - Args: - txid (str): Transaction ID - exclude_block_id (str): Exclude block from search - """ - block_statuses = self.get_blocks_status_containing_tx(txid) - block_statuses.pop(exclude_block_id, None) - for status in block_statuses.values(): - if status != self.BLOCK_INVALID: - return False - return True - - def get_transaction(self, txid, include_status=False): - """Get the transaction with the specified `txid` (and optionally its status) - - This query begins by looking in the bigchain table for all blocks containing - a transaction with the specified `txid`. If one of those blocks is valid, it - returns the matching transaction from that block. Else if some of those - blocks are undecided, it returns a matching transaction from one of them. If - the transaction was found in invalid blocks only, or in no blocks, then this - query looks for a matching transaction in the backlog table, and if it finds - one there, it returns that. - - Args: - txid (str): transaction id of the transaction to get - include_status (bool): also return the status of the transaction - the return value is then a tuple: (tx, status) - - Returns: - A :class:`~.models.Transaction` instance if the transaction - was found in a valid block, an undecided block, or the backlog table, - otherwise ``None``. - If :attr:`include_status` is ``True``, also returns the - transaction's status if the transaction was found. - """ - - response, tx_status = None, None - - blocks_validity_status = self.get_blocks_status_containing_tx(txid) - check_backlog = True - - if blocks_validity_status: - # Disregard invalid blocks, and return if there are no valid or undecided blocks - blocks_validity_status = { - _id: status for _id, status in blocks_validity_status.items() - if status != Bigchain.BLOCK_INVALID - } - if blocks_validity_status: - - # The transaction _was_ found in an undecided or valid block, - # so there's no need to look in the backlog table - check_backlog = False - - tx_status = self.TX_UNDECIDED - # If the transaction is in a valid or any undecided block, return it. Does not check - # if transactions in undecided blocks are consistent, but selects the valid block - # before undecided ones - for target_block_id in blocks_validity_status: - if blocks_validity_status[target_block_id] == Bigchain.BLOCK_VALID: - tx_status = self.TX_VALID - break - - # Query the transaction in the target block and return - response = backend.query.get_transaction_from_block(self.connection, txid, target_block_id) - - if check_backlog: - response = backend.query.get_transaction_from_backlog(self.connection, txid) - - if response: - tx_status = self.TX_IN_BACKLOG - - if response: - if tx_status == self.TX_IN_BACKLOG: - response = Transaction.from_dict(response) - else: - # If we are reading from the bigchain collection the asset is - # not in the transaction so we need to fetch the asset and - # reconstruct the transaction. - response = Transaction.from_db(self, response) - - if include_status: - return response, tx_status - else: - return response - - def get_status(self, txid): - """Retrieve the status of a transaction with `txid` from bigchain. - - Args: - txid (str): transaction id of the transaction to query - - Returns: - (string): transaction status ('valid', 'undecided', - or 'backlog'). If no transaction with that `txid` was found it - returns `None` - """ - _, status = self.get_transaction(txid, include_status=True) - return status - - def get_blocks_status_containing_tx(self, txid): - """Retrieve block ids and statuses related to a transaction - - Transactions may occur in multiple blocks, but no more than one valid block. - - Args: - txid (str): transaction id of the transaction to query - - Returns: - A dict of blocks containing the transaction, - e.g. {block_id_1: 'valid', block_id_2: 'invalid' ...}, or None - """ - - # First, get information on all blocks which contain this transaction - blocks = backend.query.get_blocks_status_from_transaction(self.connection, txid) - if blocks: - # Determine the election status of each block - blocks_validity_status = { - block['id']: self.block_election_status(block) - for block in blocks - } - - # NOTE: If there are multiple valid blocks with this transaction, - # something has gone wrong - if list(blocks_validity_status.values()).count(Bigchain.BLOCK_VALID) > 1: - block_ids = str([ - block for block in blocks_validity_status - if blocks_validity_status[block] == Bigchain.BLOCK_VALID - ]) - raise core_exceptions.CriticalDoubleInclusion( - 'Transaction {tx} is present in ' - 'multiple valid blocks: {block_ids}' - .format(tx=txid, block_ids=block_ids)) - - return blocks_validity_status - - else: - return None - - def get_asset_by_id(self, asset_id): - """Returns the asset associated with an asset_id. - - Args: - asset_id (str): The asset id. - - Returns: - dict if the asset exists else None. - """ - cursor = backend.query.get_asset_by_id(self.connection, asset_id) - cursor = list(cursor) - if cursor: - return cursor[0]['asset'] - - def get_spent(self, txid, output): - """Check if a `txid` was already used as an input. - - A transaction can be used as an input for another transaction. Bigchain - needs to make sure that a given `(txid, output)` is only used once. - - This method will check if the `(txid, output)` has already been - spent in a transaction that is in either the `VALID`, `UNDECIDED` or - `BACKLOG` state. - - Args: - txid (str): The id of the transaction - output (num): the index of the output in the respective transaction - - Returns: - The transaction (Transaction) that used the `(txid, output)` as an - input else `None` - - Raises: - CriticalDoubleSpend: If the given `(txid, output)` was spent in - more than one valid transaction. - """ - # checks if an input was already spent - # checks if the bigchain has any transaction with input {'txid': ..., - # 'output': ...} - transactions = list(backend.query.get_spent(self.connection, txid, - output)) - - # a transaction_id should have been spent at most one time - # determine if these valid transactions appear in more than one valid - # block - num_valid_transactions = 0 - non_invalid_transactions = [] - for transaction in transactions: - # ignore transactions in invalid blocks - # FIXME: Isn't there a faster solution than doing I/O again? - txn, status = self.get_transaction(transaction['id'], - include_status=True) - if status == self.TX_VALID: - num_valid_transactions += 1 - # `txid` can only have been spent in at most on valid block. - if num_valid_transactions > 1: - raise core_exceptions.CriticalDoubleSpend( - '`{}` was spent more than once. There is a problem' - ' with the chain'.format(txid)) - # if its not and invalid transaction - if status is not None: - transaction.update({'metadata': txn.metadata}) - non_invalid_transactions.append(transaction) - - if non_invalid_transactions: - return Transaction.from_dict(non_invalid_transactions[0]) - - # Either no transaction was returned spending the `(txid, output)` as - # input or the returned transactions are not valid. - - def get_owned_ids(self, owner): - """Retrieve a list of ``txid`` s that can be used as inputs. - - Args: - owner (str): base58 encoded public key. - - Returns: - :obj:`list` of TransactionLink: list of ``txid`` s and ``output`` s - pointing to another transaction's condition - """ - return self.get_outputs_filtered(owner, spent=False) - - def get_outputs_filtered(self, owner, spent=None): - """Get a list of output links filtered on some criteria - - Args: - owner (str): base58 encoded public_key. - spent (bool): If ``True`` return only the spent outputs. If - ``False`` return only unspent outputs. If spent is - not specified (``None``) return all outputs. - - Returns: - :obj:`list` of TransactionLink: list of ``txid`` s and ``output`` s - pointing to another transaction's condition - """ - outputs = self.fastquery.get_outputs_by_public_key(owner) - if spent is None: - return outputs - elif spent is True: - return self.fastquery.filter_unspent_outputs(outputs) - elif spent is False: - return self.fastquery.filter_spent_outputs(outputs) - - def get_transactions_filtered(self, asset_id, operation=None): - """Get a list of transactions filtered on some criteria - """ - txids = backend.query.get_txids_filtered(self.connection, asset_id, - operation) - for txid in txids: - tx, status = self.get_transaction(txid, True) - if status == self.TX_VALID: - yield tx - - # TODO: check that the votings structure is correctly constructed - def validate_block(self, block): - """Validate a block. - - Args: - block (Block): block to validate. - - Returns: - The block if the block is valid else it raises and exception - describing the reason why the block is invalid. - """ - return self.consensus.validate_block(self, block) - - def write_block(self, block): - """Write a block to bigchain. - - Args: - block (Block): block to write to bigchain. - """ - - # Decouple assets from block - assets, block_dict = block.decouple_assets() - metadatas, block_dict = block.decouple_metadata(block_dict) - - # write the assets - if assets: - self.write_assets(assets) - - if metadatas: - self.write_metadata(metadatas) - - # write the block - return backend.query.store_block(self.connection, block_dict) - - def get_assets(self, asset_ids): - """Return a list of assets that match the asset_ids - - Args: - asset_ids (:obj:`list` of :obj:`str`): A list of asset_ids to - retrieve from the database. - - Returns: - list: The list of assets returned from the database. - """ - return backend.query.get_assets(self.connection, asset_ids) - - def get_metadata(self, txn_ids): - """Return a list of metadata that match the transaction ids (txn_ids) - - Args: - txn_ids (:obj:`list` of :obj:`str`): A list of txn_ids to - retrieve from the database. - - Returns: - list: The list of metadata returned from the database. - """ - return backend.query.get_metadata(self.connection, txn_ids) - - def write_assets(self, assets): - """Writes a list of assets into the database. - - Args: - assets (:obj:`list` of :obj:`dict`): A list of assets to write to - the database. - """ - return backend.query.store_assets(self.connection, assets) - - def write_metadata(self, metadata): - """Writes a list of metadata into the database. - - Args: - metadata (:obj:`list` of :obj:`dict`): A list of metadata to write to - the database. - """ - return backend.query.write_metadata(self.connection, metadata) - - def text_search(self, search, *, limit=0, table='assets'): - """Return an iterator of assets that match the text search - - Args: - search (str): Text search string to query the text index - limit (int, optional): Limit the number of returned documents. - - Returns: - iter: An iterator of assets that match the text search. - """ - objects = backend.query.text_search(self.connection, search, limit=limit, - table=table) - - # TODO: This is not efficient. There may be a more efficient way to - # query by storing block ids with the assets and using fastquery. - # See https://github.com/bigchaindb/bigchaindb/issues/1496 - for obj in objects: - tx, status = self.get_transaction(obj['id'], True) - if status == self.TX_VALID: - yield obj diff --git a/bigchaindb/models.py b/bigchaindb/models.py index 8d9aae6f..33689d0c 100644 --- a/bigchaindb/models.py +++ b/bigchaindb/models.py @@ -14,7 +14,7 @@ class Transaction(Transaction): """Validate transaction spend Args: - bigchain (Bigchain): an instantiated bigchaindb.Bigchain object. + bigchain (BigchainDB): an instantiated bigchaindb.tendermint.BigchainDB object. Returns: The transaction (Transaction) if the transaction is valid else it @@ -108,8 +108,8 @@ class Transaction(Transaction): asset from the asset table and reconstructs the transaction. Args: - bigchain (:class:`~bigchaindb.Bigchain`): An instance of Bigchain - used to perform database queries. + bigchain (:class:`~bigchaindb.tendermint.BigchainDB`): An instance + of BigchainDB used to perform database queries. tx_dict_list (:list:`dict` or :obj:`dict`): The transaction dict or list of transaction dict as returned from the database. diff --git a/bigchaindb/tendermint/lib.py b/bigchaindb/tendermint/lib.py index 7275eed6..ed7aec15 100644 --- a/bigchaindb/tendermint/lib.py +++ b/bigchaindb/tendermint/lib.py @@ -14,9 +14,9 @@ except ImportError: from sha3 import sha3_256 import requests + import bigchaindb from bigchaindb import backend, config_utils -from bigchaindb import Bigchain from bigchaindb.models import Transaction from bigchaindb.common.exceptions import (SchemaValidationError, ValidationError, @@ -24,15 +24,45 @@ from bigchaindb.common.exceptions import (SchemaValidationError, from bigchaindb.tendermint.utils import encode_transaction, merkleroot from bigchaindb.tendermint import fastquery from bigchaindb import exceptions as core_exceptions +from bigchaindb.consensus import BaseConsensusRules logger = logging.getLogger(__name__) -class BigchainDB(Bigchain): +class BigchainDB(object): + """Bigchain API - def __init__(self, connection=None, **kwargs): - super().__init__(**kwargs) + Create, read, sign, write transactions to the database + """ + BLOCK_INVALID = 'invalid' + """return if a block is invalid""" + + BLOCK_VALID = TX_VALID = 'valid' + """return if a block is valid, or tx is in valid block""" + + BLOCK_UNDECIDED = TX_UNDECIDED = 'undecided' + """return if block is undecided, or tx is in undecided block""" + + TX_IN_BACKLOG = 'backlog' + """return if transaction is in backlog""" + + def __init__(self, connection=None): + """Initialize the Bigchain instance + + A Bigchain instance has several configuration parameters (e.g. host). + If a parameter value is passed as an argument to the Bigchain + __init__ method, then that is the value it will have. + Otherwise, the parameter value will come from an environment variable. + If that environment variable isn't set, then the value + will come from the local configuration file. And if that variable + isn't in the local configuration file, then the parameter will have + its default value (defined in bigchaindb.__init__). + + Args: + connection (:class:`~bigchaindb.backend.connection.Connection`): + A connection to the database. + """ config_utils.autoconfigure() self.mode_list = ('broadcast_tx_async', 'broadcast_tx_sync', @@ -40,12 +70,20 @@ class BigchainDB(Bigchain): self.tendermint_host = bigchaindb.config['tendermint']['host'] self.tendermint_port = bigchaindb.config['tendermint']['port'] self.endpoint = 'http://{}:{}/'.format(self.tendermint_host, self.tendermint_port) + + consensusPlugin = bigchaindb.config.get('consensus_plugin') + + if consensusPlugin: + self.consensus = config_utils.load_consensus_plugin(consensusPlugin) + else: + self.consensus = BaseConsensusRules + self.connection = connection if connection else backend.connect(**bigchaindb.config['database']) def post_transaction(self, transaction, mode): """Submit a valid transaction to the mempool.""" if not mode or mode not in self.mode_list: - raise ValidationError(('Mode must be one of the following {}.') + raise ValidationError('Mode must be one of the following {}.' .format(', '.join(self.mode_list))) payload = { @@ -86,7 +124,7 @@ class BigchainDB(Bigchain): # else: # return (500, 'Error while validating the transaction') - def _process_status_code(self, status_code, failure_msg): + def process_status_code(self, status_code, failure_msg): return (202, '') if status_code == 0 else (500, failure_msg) def store_transaction(self, transaction): @@ -237,6 +275,37 @@ class BigchainDB(Bigchain): else: return transaction + def get_transactions_filtered(self, asset_id, operation=None): + """Get a list of transactions filtered on some criteria + """ + txids = backend.query.get_txids_filtered(self.connection, asset_id, + operation) + for txid in txids: + tx, status = self.get_transaction(txid, True) + if status == self.TX_VALID: + yield tx + + def get_outputs_filtered(self, owner, spent=None): + """Get a list of output links filtered on some criteria + + Args: + owner (str): base58 encoded public_key. + spent (bool): If ``True`` return only the spent outputs. If + ``False`` return only unspent outputs. If spent is + not specified (``None``) return all outputs. + + Returns: + :obj:`list` of TransactionLink: list of ``txid`` s and ``output`` s + pointing to another transaction's condition + """ + outputs = self.fastquery.get_outputs_by_public_key(owner) + if spent is None: + return outputs + elif spent is True: + return self.fastquery.filter_unspent_outputs(outputs) + elif spent is False: + return self.fastquery.filter_spent_outputs(outputs) + def get_spent(self, txid, output, current_transactions=[]): transactions = backend.query.get_spent(self.connection, txid, output) @@ -343,6 +412,51 @@ class BigchainDB(Bigchain): logger.warning('Invalid transaction (%s): %s', type(e).__name__, e) return False + def text_search(self, search, *, limit=0, table='assets'): + """Return an iterator of assets that match the text search + + Args: + search (str): Text search string to query the text index + limit (int, optional): Limit the number of returned documents. + + Returns: + iter: An iterator of assets that match the text search. + """ + objects = backend.query.text_search(self.connection, search, limit=limit, + table=table) + + # TODO: This is not efficient. There may be a more efficient way to + # query by storing block ids with the assets and using fastquery. + # See https://github.com/bigchaindb/bigchaindb/issues/1496 + for obj in objects: + tx, status = self.get_transaction(obj['id'], True) + if status == self.TX_VALID: + yield obj + + def get_assets(self, asset_ids): + """Return a list of assets that match the asset_ids + + Args: + asset_ids (:obj:`list` of :obj:`str`): A list of asset_ids to + retrieve from the database. + + Returns: + list: The list of assets returned from the database. + """ + return backend.query.get_assets(self.connection, asset_ids) + + def get_metadata(self, txn_ids): + """Return a list of metadata that match the transaction ids (txn_ids) + + Args: + txn_ids (:obj:`list` of :obj:`str`): A list of txn_ids to + retrieve from the database. + + Returns: + list: The list of metadata returned from the database. + """ + return backend.query.get_metadata(self.connection, txn_ids) + @property def fastquery(self): return fastquery.FastQuery(self.connection) diff --git a/bigchaindb/web/server.py b/bigchaindb/web/server.py index 05d378b6..646f6f17 100644 --- a/bigchaindb/web/server.py +++ b/bigchaindb/web/server.py @@ -11,7 +11,7 @@ from flask_cors import CORS import gunicorn.app.base from bigchaindb import utils -from bigchaindb import Bigchain +from bigchaindb.tendermint import BigchainDB from bigchaindb.web.routes import add_routes from bigchaindb.web.strip_content_type_middleware import StripContentTypeMiddleware @@ -67,7 +67,7 @@ def create_app(*, debug=False, threads=1, bigchaindb_factory=None): """ if not bigchaindb_factory: - bigchaindb_factory = Bigchain + bigchaindb_factory = BigchainDB app = Flask(__name__) app.wsgi_app = StripContentTypeMiddleware(app.wsgi_app) diff --git a/docs/server/generate_http_server_api_documentation.py b/docs/server/generate_http_server_api_documentation.py index 69bcd699..2256c355 100644 --- a/docs/server/generate_http_server_api_documentation.py +++ b/docs/server/generate_http_server_api_documentation.py @@ -5,7 +5,7 @@ import os import os.path from bigchaindb.common.transaction import Transaction, Input, TransactionLink -from bigchaindb.core import Bigchain +from bigchaindb.tendermint import BigchainDB from bigchaindb.tendermint import lib from bigchaindb.web import server diff --git a/docs/server/source/appendices/tendermint-integration.rst b/docs/server/source/appendices/tendermint-integration.rst index 175cbe80..bc800400 100644 --- a/docs/server/source/appendices/tendermint-integration.rst +++ b/docs/server/source/appendices/tendermint-integration.rst @@ -8,6 +8,7 @@ Tendermint Integration .. automodule:: bigchaindb.tendermint.lib :special-members: __init__ + :noindex: .. automodule:: bigchaindb.tendermint.core :special-members: __init__ diff --git a/tests/commands/test_commands.py b/tests/commands/test_commands.py index a81a6938..17403e8c 100644 --- a/tests/commands/test_commands.py +++ b/tests/commands/test_commands.py @@ -90,7 +90,7 @@ def test_bigchain_run_init_when_db_exists(mocker, capsys): def test__run_init(mocker): from bigchaindb.commands.bigchaindb import _run_init bigchain_mock = mocker.patch( - 'bigchaindb.commands.bigchaindb.bigchaindb.tendermint.lib.BigchainDB') + 'bigchaindb.commands.bigchaindb.bigchaindb.tendermint.BigchainDB') init_db_mock = mocker.patch( 'bigchaindb.commands.bigchaindb.schema.init_database', autospec=True, diff --git a/tests/db/test_bigchain_api.py b/tests/db/test_bigchain_api.py index b6c8a2ad..c285401a 100644 --- a/tests/db/test_bigchain_api.py +++ b/tests/db/test_bigchain_api.py @@ -280,7 +280,7 @@ class TestBigchainApi(object): @pytest.mark.usefixtures('inputs') def test_write_transaction(self, b, user_pk, user_sk): - from bigchaindb import Bigchain + from bigchaindb.tendermint import BigchainDB from bigchaindb.models import Transaction input_tx = b.get_owned_ids(user_pk).pop() @@ -294,7 +294,7 @@ class TestBigchainApi(object): tx_from_db, status = b.get_transaction(tx.id, include_status=True) assert tx_from_db.to_dict() == tx.to_dict() - assert status == Bigchain.TX_IN_BACKLOG + assert status == BigchainDB.TX_IN_BACKLOG @pytest.mark.usefixtures('inputs') def test_read_transaction(self, b, user_pk, user_sk): @@ -439,7 +439,7 @@ class TestBigchainApi(object): from bigchaindb.common.exceptions import InputDoesNotExist from bigchaindb.common.transaction import Input, TransactionLink from bigchaindb.models import Transaction - from bigchaindb import Bigchain + from bigchaindb.tendermint import BigchainDB # Create an input for a non existing transaction input = Input(Ed25519Sha256(public_key=b58decode(user_pk)), @@ -449,7 +449,7 @@ class TestBigchainApi(object): asset_id='mock_asset_link') with pytest.raises(InputDoesNotExist): - tx.validate(Bigchain()) + tx.validate(BigchainDB()) def test_count_backlog(self, b, user_pk, alice): from bigchaindb.backend import query @@ -970,9 +970,9 @@ class TestMultipleInputs(object): def test_get_owned_ids_calls_get_outputs_filtered(): - from bigchaindb.core import Bigchain - with patch('bigchaindb.core.Bigchain.get_outputs_filtered') as gof: - b = Bigchain() + from bigchaindb.tendermint import BigchainDB + with patch('bigchaindb.tendermint.BigchainDB.get_outputs_filtered') as gof: + b = BigchainDB() res = b.get_owned_ids('abc') gof.assert_called_once_with('abc', spent=False) assert res == gof() diff --git a/tests/test_config_utils.py b/tests/test_config_utils.py index a96e1405..30dcfd60 100644 --- a/tests/test_config_utils.py +++ b/tests/test_config_utils.py @@ -35,6 +35,7 @@ def test_bigchain_instance_raises_when_not_configured(request, monkeypatch): import bigchaindb from bigchaindb import config_utils from bigchaindb.common import exceptions + from bigchaindb.tendermint import BigchainDB assert 'CONFIGURED' not in bigchaindb.config # We need to disable ``bigchaindb.config_utils.autoconfigure`` to avoid reading @@ -42,7 +43,7 @@ def test_bigchain_instance_raises_when_not_configured(request, monkeypatch): monkeypatch.setattr(config_utils, 'autoconfigure', lambda: 0) with pytest.raises(exceptions.ConfigurationError): - bigchaindb.Bigchain() + BigchainDB() def test_load_consensus_plugin_loads_default_rules_without_name(): diff --git a/tests/test_core.py b/tests/test_core.py index b563c385..9f6150e6 100644 --- a/tests/test_core.py +++ b/tests/test_core.py @@ -26,10 +26,10 @@ def config(request, monkeypatch): def test_bigchain_class_default_initialization(config): - from bigchaindb.core import Bigchain + from bigchaindb.tendermint import BigchainDB from bigchaindb.consensus import BaseConsensusRules from bigchaindb.backend.connection import Connection - bigchain = Bigchain() + bigchain = BigchainDB() assert isinstance(bigchain.connection, Connection) assert bigchain.connection.host == config['database']['host'] assert bigchain.connection.port == config['database']['port'] @@ -38,7 +38,7 @@ def test_bigchain_class_default_initialization(config): def test_bigchain_class_initialization_with_parameters(config): - from bigchaindb.core import Bigchain + from bigchaindb.tendermint import BigchainDB from bigchaindb.backend import connect from bigchaindb.consensus import BaseConsensusRules init_db_kwargs = { @@ -48,7 +48,7 @@ def test_bigchain_class_initialization_with_parameters(config): 'name': 'this_is_the_db_name', } connection = connect(**init_db_kwargs) - bigchain = Bigchain(connection=connection, **init_db_kwargs) + bigchain = BigchainDB(connection=connection, **init_db_kwargs) assert bigchain.connection == connection assert bigchain.connection.host == init_db_kwargs['host'] assert bigchain.connection.port == init_db_kwargs['port'] @@ -58,13 +58,13 @@ def test_bigchain_class_initialization_with_parameters(config): def test_get_blocks_status_containing_tx(monkeypatch): from bigchaindb.backend import query as backend_query - from bigchaindb.core import Bigchain + from bigchaindb.tendermint import BigchainDB blocks = [ {'id': 1}, {'id': 2} ] monkeypatch.setattr(backend_query, 'get_blocks_status_from_transaction', lambda x: blocks) - monkeypatch.setattr(Bigchain, 'block_election_status', lambda x, y, z: Bigchain.BLOCK_VALID) - bigchain = Bigchain(public_key='pubkey', private_key='privkey') + monkeypatch.setattr(BigchainDB, 'block_election_status', lambda x, y, z: BigchainDB.BLOCK_VALID) + bigchain = BigchainDB(public_key='pubkey', private_key='privkey') with pytest.raises(Exception): bigchain.get_blocks_status_containing_tx('txid') diff --git a/tests/web/test_transactions.py b/tests/web/test_transactions.py index c81a9db8..7c52ea94 100644 --- a/tests/web/test_transactions.py +++ b/tests/web/test_transactions.py @@ -372,7 +372,7 @@ def test_transactions_get_list_good(client): asset_id = '1' * 64 - with patch('bigchaindb.core.Bigchain.get_transactions_filtered', get_txs_patched): + with patch('bigchaindb.tendermint.BigchainDB.get_transactions_filtered', get_txs_patched): url = TX_ENDPOINT + '?asset_id=' + asset_id assert client.get(url).json == [ ['asset_id', asset_id], @@ -389,7 +389,7 @@ def test_transactions_get_list_good(client): def test_transactions_get_list_bad(client): def should_not_be_called(): assert False - with patch('bigchaindb.core.Bigchain.get_transactions_filtered', + with patch('bigchaindb.tendermint.BigchainDB.get_transactions_filtered', lambda *_, **__: should_not_be_called()): # Test asset id validated url = TX_ENDPOINT + '?asset_id=' + '1' * 63 @@ -404,7 +404,7 @@ def test_transactions_get_list_bad(client): @pytest.mark.tendermint def test_return_only_valid_transaction(client): - from bigchaindb import Bigchain + from bigchaindb.tendermint import BigchainDB def get_transaction_patched(status): def inner(self, tx_id, include_status): @@ -415,13 +415,13 @@ def test_return_only_valid_transaction(client): # UNDECIDED or VALID block, as well as transactions from the backlog. # As the endpoint uses `get_transaction`, we don't have to test # against invalid transactions here. - with patch('bigchaindb.core.Bigchain.get_transaction', - get_transaction_patched(Bigchain.TX_UNDECIDED)): + with patch('bigchaindb.tendermint.BigchainDB.get_transaction', + get_transaction_patched(BigchainDB.TX_UNDECIDED)): url = '{}{}'.format(TX_ENDPOINT, '123') assert client.get(url).status_code == 404 - with patch('bigchaindb.core.Bigchain.get_transaction', - get_transaction_patched(Bigchain.TX_IN_BACKLOG)): + with patch('bigchaindb.tendermint.BigchainDB.get_transaction', + get_transaction_patched(BigchainDB.TX_IN_BACKLOG)): url = '{}{}'.format(TX_ENDPOINT, '123') assert client.get(url).status_code == 404