mirror of
https://github.com/kaspanet/kaspad.git
synced 2025-03-30 15:08:33 +00:00

* [NOD-828] Reimplement FFLDB (#663) * [NOD-828] Create the database2 package that will some day replace the database package. * [NOD-828] Implement a "bucket" key mechanism. * [NOD-828] Move bucket.go into the ffldb2 package. * [NOD-828] Delete the un-interfaced ffldb package from database2, since we aren't going to be using it anyway. * [NOD-828] Copy over + fixup flat file structs from the old ffldb. * [NOD-828] Implement flatFilePath. * [NOD-828] Implement flatFileStore.write(). * [NOD-828] Implement flatFileStore.read(). * [NOD-828] Implement flatFileStore.rollback(). * [NOD-828] Sync the file to disk at the end of write(). * [NOD-828] Extract crc32ByteOrder to a separate variable. * [NOD-828] Add a sanity test. * [NOD-828] Remove context-unrelated methods from the Database interface. * [NOD-828] Create an ffldb object. Simply work against a context. * [NOD-828] Open the new database on start. * [NOD-828] Create the leveldb package. * [NOD-828] Implement opening/closing leveldb. * [NOD-828] Implement get/put out of/into leveldb. * [NOD-828] Implement transactions and make them implement a generic database interface. * [NOD-828] Write sanity tests for leveldb with and without transactions. * [NOD-828] Add another case to the transaction sanity test. * [NOD-828] Implement AppendBlock/RetrieveBlock. * [NOD-828] Refactor so that concepts such as "block" and "metadata" don't leak into the database package. * [NOD-828] Add RollbackFlatData to the database interface. * [NOD-828] Remove anything from dbaccess that I'm not planning to implement as part of this ticket. * [NOD-828] Implement StoreBlock. * [NOD-828] Implement FetchBlock. * [NOD-828] Implement HasBlock. * [NOD-828] Write a sanity test for block insertion. * [NOD-828] Implement CurrentFlatDataLocation. * [NOD-828] Implement storing the current block location. * [NOD-828] Implement initializing/syncing the flat file block store and the "metadata". * [NOD-828] Add InitBlockStore to TestBlockStoreSanity. * [NOD-828] Fix rename errors. * [NOD-828] Fix lint errors in the root database package. * [NOD-828] Fix lint errors in the ffldb.go. * [NOD-828] Fix lint errors in the flatfile/db.go. * [NOD-828] Rename packages in such a way to make the linter happy. * [NOD-828] Finish satisfying the linter. * [NOD-828] Fix doc.go. * [NOD-828] Fix comments in block.go. * [NOD-828] Move dbaccess out of the database package. * [NOD-828] Move opening/closing the database to dbaccess. * [NOD-828] Move the Database interface to the root database package, since it's meant to be generic. * [NOD-828] Make ffldb generic to later support additional databases. * [NOD-828] Make ffldb.Open return DatabaseHandle, since ffldb is no longer exported. * [NOD-828] Fix comments. * [NOD-828] Rename AppendFlatData to AppendToStore and RetrieveFlatData to RetrieveFromStore. * [NOD-828] Make buckets nicer to use. * [NOD-828] Implement cursors that iterate over some bucket. * [NOD-828] Generalize flat-file repairing and move block database repairing into ffldb. * [NOD-828] Write a test making sure that flat file repair works. * [NOD-828] Properly close the database in TestRepairFlatFiles. * [NOD-828] Add a comment warning against putting and getting the same data within the same transaction. * [NOD-828] Fix the flatFilesBucket description. * [NOD-828] Remove superfluous whitespace. * [NOD-828] Fix the ffldb description. * [NOD-828] Rename Database to DataAccessor and rename Handle to Database. * [NOD-828] Make Get return nil if the value doesn't exist. * [NOD-828] Attempt to close leveldb even if closing ffdb failed. * [NOD-828] Fix a bug where the wrong location would be written to the current store location bucket. * [NOD-828] Fix not updating the store location in ffldb transactions. * [NOD-828] Make scanFlatFiles return an error if os.Stat fails for any reason other than file-not-found. * [NOD-828] Update the README and doc.go. * [NOD-828] Simplify Bucket.Path(). * [NOD-828] Since LevelDBCursor satisfied the database2.Cursor interface, use it directly. * [NOD-828] Combine two lines into one. * [NOD-828] Combine another two lines into one. * [NOD-828] Move a misplaced comment. * [NOD-828] Use Wrapf instead of Errorf where appropriate. * [NOD-828] Fix a comment. * [NOD-828] Fix a comment. * [NOD-828] Fix Wrapf calls. * [NOD-828] Fix comments. * [NOD-828] Remove superfluous whitespace. * [NOD-828] Add a comment explaining the use of batches and snapshots. * [NOD-828] Implement RollbackUnlessClosed(). * [NOD-828] Return both errors in StoreBlock rollback. * [NOD-828] Move rollback-on-error logic into ffldb. Remove CurrentStoreLocation and RollbackStore from DataAccessor. * [NOD-828] Make bucket a type alias instead of a struct. * [NOD-828] Fix a typo. * [NOD-828] Use copy instead of append in Bucket. * [NOD-828] Extract flatFileLocationSerializedSize to a const. * [NOD-828] Debugf -> Warnf in rollback.go. * [NOD-828] Fix a comment. * [NOD-828] Fix a comment. * [NOD-828] Remove data length from flat file data format. * [NOD-828] Rearrange TestLevelDBTransactionSanity a bit. * [NOD-828] Add stack traces to all errors that come out of library functions. * [NOD-828] Return errors from rollback(). * [NOD-828] Remove an irrelevant comment. * [NOD-828] Remove redundant whitespace. * [NOD-828] Handle nil in FetchBlock. * [NOD-828] Move the explanation about batches and snapshots to the LevelDBTransaction struct. * Revert "[NOD-828] Make bucket a type alias instead of a struct." This reverts commit 1fd39652 * [NOD-828] Fix revert errors. * Revert "[NOD-828] Remove data length from flat file data format." This reverts commit ef408e32 * [NOD-862] Move Cursor() into the DataAccessor interface. * [NOD-828] Add Delete to DataAccessor. * [NOD-828] Fix a couple of places that erroneously referenced blocks. * [NOD-828] Add a comment on top of flatFileLocationSerializedSize. * [NOD-828] Add Seek to Cursor. * [NOD-828] Add First to Cursor. * [NOD-828] Rename db to accessor in Context. * [NOD-828] Make Get/Fetch calls return a boolean to indicate whether the requested item was found. * [NOD-828] Name the output parameters of all Get functions. * [NOD-828] Make RetrieveFromStore return whether the data was found. * [NOD-887] Add a couple of QoL features to Cursor (#674) * [NOD-887] Changed First to not return an error. * [NOD-887] Fix merge error. * [NOD-887] Make Cursor.Key not return the entire key path. * [NOD-888] Add RollbackUnlessClosed to Context (#676) * [NOD-888] Add RollbackUnlessClosed to Context. * [NOD-888] Fix copy+paste error. * [NOD-889] Instead of returning a boolean for not-found, return an error (#677) * [NOD-889] Instead of returning a boolean for not-found, return an error. * [NOD-889] Wrapped ErrNotFound for Get calls with nicer error messages. * [NOD-889] Fix format. * [NOD-889] Fix double space in a comment. * [NOD-889] Add IsNotFoundError to dbaccess. * [NOD-862] Replace calls to Tx.StoreBlock, Tx.HasBlock, Tx.FetchBlock with appropriate calls in dbaccess (#672) * [NOD-828] Create the database2 package that will some day replace the database package. * [NOD-828] Implement a "bucket" key mechanism. * [NOD-828] Move bucket.go into the ffldb2 package. * [NOD-828] Delete the un-interfaced ffldb package from database2, since we aren't going to be using it anyway. * [NOD-828] Copy over + fixup flat file structs from the old ffldb. * [NOD-828] Implement flatFilePath. * [NOD-828] Implement flatFileStore.write(). * [NOD-828] Implement flatFileStore.read(). * [NOD-828] Implement flatFileStore.rollback(). * [NOD-828] Sync the file to disk at the end of write(). * [NOD-828] Extract crc32ByteOrder to a separate variable. * [NOD-828] Add a sanity test. * [NOD-828] Remove context-unrelated methods from the Database interface. * [NOD-828] Create an ffldb object. Simply work against a context. * [NOD-828] Open the new database on start. * [NOD-828] Create the leveldb package. * [NOD-828] Implement opening/closing leveldb. * [NOD-828] Implement get/put out of/into leveldb. * [NOD-828] Implement transactions and make them implement a generic database interface. * [NOD-828] Write sanity tests for leveldb with and without transactions. * [NOD-828] Add another case to the transaction sanity test. * [NOD-828] Implement AppendBlock/RetrieveBlock. * [NOD-828] Refactor so that concepts such as "block" and "metadata" don't leak into the database package. * [NOD-828] Add RollbackFlatData to the database interface. * [NOD-828] Remove anything from dbaccess that I'm not planning to implement as part of this ticket. * [NOD-828] Implement StoreBlock. * [NOD-828] Implement FetchBlock. * [NOD-828] Implement HasBlock. * [NOD-828] Write a sanity test for block insertion. * [NOD-828] Implement CurrentFlatDataLocation. * [NOD-828] Implement storing the current block location. * [NOD-828] Implement initializing/syncing the flat file block store and the "metadata". * [NOD-828] Add InitBlockStore to TestBlockStoreSanity. * [NOD-828] Fix rename errors. * [NOD-828] Fix lint errors in the root database package. * [NOD-828] Fix lint errors in the ffldb.go. * [NOD-828] Fix lint errors in the flatfile/db.go. * [NOD-828] Rename packages in such a way to make the linter happy. * [NOD-828] Finish satisfying the linter. * [NOD-828] Fix doc.go. * [NOD-828] Fix comments in block.go. * [NOD-828] Move dbaccess out of the database package. * [NOD-828] Move opening/closing the database to dbaccess. * [NOD-828] Move the Database interface to the root database package, since it's meant to be generic. * [NOD-828] Make ffldb generic to later support additional databases. * [NOD-828] Make ffldb.Open return DatabaseHandle, since ffldb is no longer exported. * [NOD-828] Fix comments. * [NOD-828] Rename AppendFlatData to AppendToStore and RetrieveFlatData to RetrieveFromStore. * [NOD-828] Make buckets nicer to use. * [NOD-828] Implement cursors that iterate over some bucket. * [NOD-828] Generalize flat-file repairing and move block database repairing into ffldb. * [NOD-828] Write a test making sure that flat file repair works. * [NOD-828] Properly close the database in TestRepairFlatFiles. * [NOD-828] Add a comment warning against putting and getting the same data within the same transaction. * [NOD-862] Use dbaccess.HasBlock instead of Tx.HasBlock in initDAGState. * [NOD-862] Use dbaccess.StoreBlock instead of dbStoreBlock. * [NOD-862] Use dbaccess.FetchBlock instead of various block fetching mechanisms. * [NOD-828] Fix the flatFilesBucket description. * [NOD-828] Remove superfluous whitespace. * [NOD-828] Fix the ffldb description. * [NOD-828] Rename Database to DataAccessor and rename Handle to Database. * [NOD-828] Make Get return nil if the value doesn't exist. * [NOD-828] Attempt to close leveldb even if closing ffdb failed. * [NOD-828] Fix a bug where the wrong location would be written to the current store location bucket. * [NOD-828] Fix not updating the store location in ffldb transactions. * [NOD-828] Make scanFlatFiles return an error if os.Stat fails for any reason other than file-not-found. * [NOD-828] Update the README and doc.go. * [NOD-828] Simplify Bucket.Path(). * [NOD-828] Since LevelDBCursor satisfied the database2.Cursor interface, use it directly. * [NOD-828] Combine two lines into one. * [NOD-828] Combine another two lines into one. * [NOD-828] Move a misplaced comment. * [NOD-828] Use Wrapf instead of Errorf where appropriate. * [NOD-828] Fix a comment. * [NOD-828] Fix a comment. * [NOD-828] Fix Wrapf calls. * [NOD-828] Fix comments. * [NOD-828] Remove superfluous whitespace. * [NOD-828] Add a comment explaining the use of batches and snapshots. * [NOD-828] Implement RollbackUnlessClosed(). * [NOD-828] Return both errors in StoreBlock rollback. * [NOD-828] Move rollback-on-error logic into ffldb. Remove CurrentStoreLocation and RollbackStore from DataAccessor. * [NOD-828] Make bucket a type alias instead of a struct. * [NOD-828] Fix a typo. * [NOD-828] Use copy instead of append in Bucket. * [NOD-828] Extract flatFileLocationSerializedSize to a const. * [NOD-828] Debugf -> Warnf in rollback.go. * [NOD-828] Fix a comment. * [NOD-828] Fix a comment. * [NOD-828] Remove data length from flat file data format. * [NOD-828] Rearrange TestLevelDBTransactionSanity a bit. * [NOD-828] Add stack traces to all errors that come out of library functions. * [NOD-828] Return errors from rollback(). * [NOD-828] Remove an irrelevant comment. * [NOD-828] Remove redundant whitespace. * [NOD-828] Handle nil in FetchBlock. * [NOD-828] Implement a dbaccess.BlockNode struct. * [NOD-828] Move the explanation about batches and snapshots to the LevelDBTransaction struct. * [NOD-828] Implement toDBBlockNode and fromDBBlockNode. * Revert "[NOD-828] Make bucket a type alias instead of a struct." This reverts commit 1fd39652 * [NOD-828] Fix revert errors. * Revert "[NOD-828] Remove data length from flat file data format." This reverts commit ef408e32 * [NOD-862] Implement storing index blocks. * [NOD-862] Use database transactions where appropriate. * [NOD-862] Fix tests failing on DAGSetup. * [NOD-862] Fix bad make call. * [NOD-862] Fix remaining database opening problems in tests. * [NOD-862] Move Cursor() into the DataAccessor interface. * [NOD-862] Move Cursor() into the DataAccessor interface. * [NOD-862] Iterate over the new block index in dagio. * [NOD-862] Fix block index key. * [NOD-828] Add Delete to DataAccessor. * [NOD-862] Pass byte slices to dbaccess instead of objects. * [NOD-862] Fix errors. * [NOD-862] Fix maybeAcceptBlock not checking block existence. * [NOD-862] Fix TestAcceptanceIndexRecover. * [NOD-862] Add comments to StoreBlockIndex and BlockIndexCursor. * [NOD-828] Fix a couple of places that erroneously referenced blocks. * [NOD-828] Add a comment on top of flatFileLocationSerializedSize. * [NOD-828] Add Seek to Cursor. * [NOD-828] Add First to Cursor. * [NOD-828] Rename db to accessor in Context. * [NOD-828] Make Get/Fetch calls return a boolean to indicate whether the requested item was found. * [NOD-828] Name the output parameters of all Get functions. * [NOD-828] Make RetrieveFromStore return whether the data was found. * [NOD-862] Fix merge errors. * [NOD-862] Fix DAGSetup using bad temp directories. * [NOD-862] Fix TestProcessDelayedBlocks not closing the database properly. * [NOD-862] Fix merge errors. * [NOD-862] Merge flushToDBWithContext and flushToDB. * [NOD-862] Remove TODO. * [NOD-862] Add prefix to the temp dir in DAGSetup. * [NOD-862] Bring back dbFetchBlockByHash. * [NOD-862] Use BlockDAG.BlockByHash in p2p and rpc. * [NOD-862] Use daghash.Hash in dbaccess. * [NOD-862] Add defer to RollbackUnlessClosed after NewTx(). * [NOD-862] Extract dbStoreBlock to a separate function. * [NOD-862] Fix grammar in comment. * [NOD-862] Fix merge errors. * [NOD-867] Migrate database logic in blockdag/dagio.go to dbaccess (#675) * [NOD-828] Create the database2 package that will some day replace the database package. * [NOD-828] Implement a "bucket" key mechanism. * [NOD-828] Move bucket.go into the ffldb2 package. * [NOD-828] Delete the un-interfaced ffldb package from database2, since we aren't going to be using it anyway. * [NOD-828] Copy over + fixup flat file structs from the old ffldb. * [NOD-828] Implement flatFilePath. * [NOD-828] Implement flatFileStore.write(). * [NOD-828] Implement flatFileStore.read(). * [NOD-828] Implement flatFileStore.rollback(). * [NOD-828] Sync the file to disk at the end of write(). * [NOD-828] Extract crc32ByteOrder to a separate variable. * [NOD-828] Add a sanity test. * [NOD-828] Remove context-unrelated methods from the Database interface. * [NOD-828] Create an ffldb object. Simply work against a context. * [NOD-828] Open the new database on start. * [NOD-828] Create the leveldb package. * [NOD-828] Implement opening/closing leveldb. * [NOD-828] Implement get/put out of/into leveldb. * [NOD-828] Implement transactions and make them implement a generic database interface. * [NOD-828] Write sanity tests for leveldb with and without transactions. * [NOD-828] Add another case to the transaction sanity test. * [NOD-828] Implement AppendBlock/RetrieveBlock. * [NOD-828] Refactor so that concepts such as "block" and "metadata" don't leak into the database package. * [NOD-828] Add RollbackFlatData to the database interface. * [NOD-828] Remove anything from dbaccess that I'm not planning to implement as part of this ticket. * [NOD-828] Implement StoreBlock. * [NOD-828] Implement FetchBlock. * [NOD-828] Implement HasBlock. * [NOD-828] Write a sanity test for block insertion. * [NOD-828] Implement CurrentFlatDataLocation. * [NOD-828] Implement storing the current block location. * [NOD-828] Implement initializing/syncing the flat file block store and the "metadata". * [NOD-828] Add InitBlockStore to TestBlockStoreSanity. * [NOD-828] Fix rename errors. * [NOD-828] Fix lint errors in the root database package. * [NOD-828] Fix lint errors in the ffldb.go. * [NOD-828] Fix lint errors in the flatfile/db.go. * [NOD-828] Rename packages in such a way to make the linter happy. * [NOD-828] Finish satisfying the linter. * [NOD-828] Fix doc.go. * [NOD-828] Fix comments in block.go. * [NOD-828] Move dbaccess out of the database package. * [NOD-828] Move opening/closing the database to dbaccess. * [NOD-828] Move the Database interface to the root database package, since it's meant to be generic. * [NOD-828] Make ffldb generic to later support additional databases. * [NOD-828] Make ffldb.Open return DatabaseHandle, since ffldb is no longer exported. * [NOD-828] Fix comments. * [NOD-828] Rename AppendFlatData to AppendToStore and RetrieveFlatData to RetrieveFromStore. * [NOD-828] Make buckets nicer to use. * [NOD-828] Implement cursors that iterate over some bucket. * [NOD-828] Generalize flat-file repairing and move block database repairing into ffldb. * [NOD-828] Write a test making sure that flat file repair works. * [NOD-828] Properly close the database in TestRepairFlatFiles. * [NOD-828] Add a comment warning against putting and getting the same data within the same transaction. * [NOD-862] Use dbaccess.HasBlock instead of Tx.HasBlock in initDAGState. * [NOD-862] Use dbaccess.StoreBlock instead of dbStoreBlock. * [NOD-862] Use dbaccess.FetchBlock instead of various block fetching mechanisms. * [NOD-828] Fix the flatFilesBucket description. * [NOD-828] Remove superfluous whitespace. * [NOD-828] Fix the ffldb description. * [NOD-828] Rename Database to DataAccessor and rename Handle to Database. * [NOD-828] Make Get return nil if the value doesn't exist. * [NOD-828] Attempt to close leveldb even if closing ffdb failed. * [NOD-828] Fix a bug where the wrong location would be written to the current store location bucket. * [NOD-828] Fix not updating the store location in ffldb transactions. * [NOD-828] Make scanFlatFiles return an error if os.Stat fails for any reason other than file-not-found. * [NOD-828] Update the README and doc.go. * [NOD-828] Simplify Bucket.Path(). * [NOD-828] Since LevelDBCursor satisfied the database2.Cursor interface, use it directly. * [NOD-828] Combine two lines into one. * [NOD-828] Combine another two lines into one. * [NOD-828] Move a misplaced comment. * [NOD-828] Use Wrapf instead of Errorf where appropriate. * [NOD-828] Fix a comment. * [NOD-828] Fix a comment. * [NOD-828] Fix Wrapf calls. * [NOD-828] Fix comments. * [NOD-828] Remove superfluous whitespace. * [NOD-828] Add a comment explaining the use of batches and snapshots. * [NOD-828] Implement RollbackUnlessClosed(). * [NOD-828] Return both errors in StoreBlock rollback. * [NOD-828] Move rollback-on-error logic into ffldb. Remove CurrentStoreLocation and RollbackStore from DataAccessor. * [NOD-828] Make bucket a type alias instead of a struct. * [NOD-828] Fix a typo. * [NOD-828] Use copy instead of append in Bucket. * [NOD-828] Extract flatFileLocationSerializedSize to a const. * [NOD-828] Debugf -> Warnf in rollback.go. * [NOD-828] Fix a comment. * [NOD-828] Fix a comment. * [NOD-828] Remove data length from flat file data format. * [NOD-828] Rearrange TestLevelDBTransactionSanity a bit. * [NOD-828] Add stack traces to all errors that come out of library functions. * [NOD-828] Return errors from rollback(). * [NOD-828] Remove an irrelevant comment. * [NOD-828] Remove redundant whitespace. * [NOD-828] Handle nil in FetchBlock. * [NOD-828] Implement a dbaccess.BlockNode struct. * [NOD-828] Move the explanation about batches and snapshots to the LevelDBTransaction struct. * [NOD-828] Implement toDBBlockNode and fromDBBlockNode. * Revert "[NOD-828] Make bucket a type alias instead of a struct." This reverts commit 1fd39652 * [NOD-828] Fix revert errors. * Revert "[NOD-828] Remove data length from flat file data format." This reverts commit ef408e32 * [NOD-862] Implement storing index blocks. * [NOD-862] Use database transactions where appropriate. * [NOD-862] Fix tests failing on DAGSetup. * [NOD-862] Fix bad make call. * [NOD-862] Fix remaining database opening problems in tests. * [NOD-862] Move Cursor() into the DataAccessor interface. * [NOD-862] Move Cursor() into the DataAccessor interface. * [NOD-862] Iterate over the new block index in dagio. * [NOD-862] Fix block index key. * [NOD-828] Add Delete to DataAccessor. * [NOD-862] Pass byte slices to dbaccess instead of objects. * [NOD-862] Fix errors. * [NOD-862] Fix maybeAcceptBlock not checking block existence. * [NOD-862] Fix TestAcceptanceIndexRecover. * [NOD-862] Add comments to StoreBlockIndex and BlockIndexCursor. * [NOD-828] Fix a couple of places that erroneously referenced blocks. * [NOD-828] Add a comment on top of flatFileLocationSerializedSize. * [NOD-828] Add Seek to Cursor. * [NOD-828] Add First to Cursor. * [NOD-828] Rename db to accessor in Context. * [NOD-828] Make Get/Fetch calls return a boolean to indicate whether the requested item was found. * [NOD-828] Name the output parameters of all Get functions. * [NOD-828] Make RetrieveFromStore return whether the data was found. * [NOD-862] Fix merge errors. * [NOD-862] Fix DAGSetup using bad temp directories. * [NOD-862] Fix TestProcessDelayedBlocks not closing the database properly. * [NOD-867] Remove blockIndexBucket from dagio. * [NOD-867] Fix wrong key in StoreIndexBucket. * [NOD-867] Migrate DAG state to dbaccess. * [NOD-867] Remove utxoSetVersionKeyName. * [NOD-862] Fix merge errors. * [NOD-867] Move localSubnetworkID into dagState. * [NOD-867] Fix a comment. * [NOD-867] Remove an unused function. * [NOD-867] Migrate the database's UTXO set to dbaccess. * [NOD-867] Add missing error check. * [NOD-867] Changed First to not return an error. * [NOD-867] Make Cursor.Key not return the entire key path. * [NOD-887] Fix the comment above BlockIndexCursorFrom. * [NOD-862] Merge flushToDBWithContext and flushToDB. * [NOD-862] Remove TODO. * [NOD-862] Add prefix to the temp dir in DAGSetup. * [NOD-862] Bring back dbFetchBlockByHash. * [NOD-862] Use BlockDAG.BlockByHash in p2p and rpc. * [NOD-862] Use daghash.Hash in dbaccess. * [NOD-862] Add defer to RollbackUnlessClosed after NewTx(). * [NOD-862] Extract dbStoreBlock to a separate function. * [NOD-867] Remove TODOs. * [NOD-867] Fix merge errors. * [NOD-867] Fix comments and errors. * [NOD-867] Unexport blockIndexKey. * [NOD-867] Fix merge errors. * [NOD-867] Move a misplaced comment. * [NOD-867] Fix an error message. * [NOD-867] Remove preallocation in initDAGState. * [NOD-866] Migrate database logic in blockdag/indexers package to dbaccess (#682) * [NOD-865] Delete blockidhash.go. * [NOD-865] Remove a lot of no-longer relevant logic from indexers. * [NOD-865] Pass TxContext to ConnectBlock. * [NOD-865] Migrate the acceptance index to dbaccess. * [NOD-865] Fix a block not being sent to ConnectBlock. * [NOD-865] Pass the block's hash instead of the whole block. * [NOD-865] Add forgotten Commit call. * [NOD-865] Add comments. * [NOD-866] Fix a comment. * [NOD-866] Fix a comment. * [NOD-866] Remove pointless indirection in acceptanceindex. * [NOD-866] Fix comment over ForEachHash. * [NOD-866] Rename ClearAcceptanceIndex to DropAcceptanceIndex. * [NOD-866] Explain collecting keys before deleting them. * [NOD-865] Move misc db logic to db access (#681) * [NOD-828] Create the database2 package that will some day replace the database package. * [NOD-828] Implement a "bucket" key mechanism. * [NOD-828] Move bucket.go into the ffldb2 package. * [NOD-828] Delete the un-interfaced ffldb package from database2, since we aren't going to be using it anyway. * [NOD-828] Copy over + fixup flat file structs from the old ffldb. * [NOD-828] Implement flatFilePath. * [NOD-828] Implement flatFileStore.write(). * [NOD-828] Implement flatFileStore.read(). * [NOD-828] Implement flatFileStore.rollback(). * [NOD-828] Sync the file to disk at the end of write(). * [NOD-828] Extract crc32ByteOrder to a separate variable. * [NOD-828] Add a sanity test. * [NOD-828] Remove context-unrelated methods from the Database interface. * [NOD-828] Create an ffldb object. Simply work against a context. * [NOD-828] Open the new database on start. * [NOD-828] Create the leveldb package. * [NOD-828] Implement opening/closing leveldb. * [NOD-828] Implement get/put out of/into leveldb. * [NOD-828] Implement transactions and make them implement a generic database interface. * [NOD-828] Write sanity tests for leveldb with and without transactions. * [NOD-828] Add another case to the transaction sanity test. * [NOD-828] Implement AppendBlock/RetrieveBlock. * [NOD-828] Refactor so that concepts such as "block" and "metadata" don't leak into the database package. * [NOD-828] Add RollbackFlatData to the database interface. * [NOD-828] Remove anything from dbaccess that I'm not planning to implement as part of this ticket. * [NOD-828] Implement StoreBlock. * [NOD-828] Implement FetchBlock. * [NOD-828] Implement HasBlock. * [NOD-828] Write a sanity test for block insertion. * [NOD-828] Implement CurrentFlatDataLocation. * [NOD-828] Implement storing the current block location. * [NOD-828] Implement initializing/syncing the flat file block store and the "metadata". * [NOD-828] Add InitBlockStore to TestBlockStoreSanity. * [NOD-828] Fix rename errors. * [NOD-828] Fix lint errors in the root database package. * [NOD-828] Fix lint errors in the ffldb.go. * [NOD-828] Fix lint errors in the flatfile/db.go. * [NOD-828] Rename packages in such a way to make the linter happy. * [NOD-828] Finish satisfying the linter. * [NOD-828] Fix doc.go. * [NOD-828] Fix comments in block.go. * [NOD-828] Move dbaccess out of the database package. * [NOD-828] Move opening/closing the database to dbaccess. * [NOD-828] Move the Database interface to the root database package, since it's meant to be generic. * [NOD-828] Make ffldb generic to later support additional databases. * [NOD-828] Make ffldb.Open return DatabaseHandle, since ffldb is no longer exported. * [NOD-828] Fix comments. * [NOD-828] Rename AppendFlatData to AppendToStore and RetrieveFlatData to RetrieveFromStore. * [NOD-828] Make buckets nicer to use. * [NOD-828] Implement cursors that iterate over some bucket. * [NOD-828] Generalize flat-file repairing and move block database repairing into ffldb. * [NOD-828] Write a test making sure that flat file repair works. * [NOD-828] Properly close the database in TestRepairFlatFiles. * [NOD-828] Add a comment warning against putting and getting the same data within the same transaction. * [NOD-828] Fix the flatFilesBucket description. * [NOD-828] Remove superfluous whitespace. * [NOD-828] Fix the ffldb description. * [NOD-828] Rename Database to DataAccessor and rename Handle to Database. * [NOD-828] Make Get return nil if the value doesn't exist. * [NOD-828] Attempt to close leveldb even if closing ffdb failed. * [NOD-828] Fix a bug where the wrong location would be written to the current store location bucket. * [NOD-828] Fix not updating the store location in ffldb transactions. * [NOD-828] Make scanFlatFiles return an error if os.Stat fails for any reason other than file-not-found. * [NOD-828] Update the README and doc.go. * [NOD-828] Simplify Bucket.Path(). * [NOD-828] Since LevelDBCursor satisfied the database2.Cursor interface, use it directly. * [NOD-828] Combine two lines into one. * [NOD-828] Combine another two lines into one. * [NOD-828] Move a misplaced comment. * [NOD-828] Use Wrapf instead of Errorf where appropriate. * [NOD-828] Fix a comment. * [NOD-828] Fix a comment. * [NOD-828] Fix Wrapf calls. * [NOD-828] Fix comments. * [NOD-828] Remove superfluous whitespace. * [NOD-828] Add a comment explaining the use of batches and snapshots. * [NOD-828] Implement RollbackUnlessClosed(). * [NOD-828] Return both errors in StoreBlock rollback. * [NOD-828] Move rollback-on-error logic into ffldb. Remove CurrentStoreLocation and RollbackStore from DataAccessor. * [NOD-828] Make bucket a type alias instead of a struct. * [NOD-828] Fix a typo. * [NOD-828] Use copy instead of append in Bucket. * [NOD-828] Extract flatFileLocationSerializedSize to a const. * [NOD-828] Debugf -> Warnf in rollback.go. * [NOD-828] Fix a comment. * [NOD-828] Fix a comment. * [NOD-828] Remove data length from flat file data format. * [NOD-828] Rearrange TestLevelDBTransactionSanity a bit. * [NOD-828] Add stack traces to all errors that come out of library functions. * [NOD-828] Return errors from rollback(). * [NOD-828] Remove an irrelevant comment. * [NOD-828] Remove redundant whitespace. * [NOD-828] Handle nil in FetchBlock. * [NOD-828] Move the explanation about batches and snapshots to the LevelDBTransaction struct. * Revert "[NOD-828] Make bucket a type alias instead of a struct." This reverts commit 1fd39652 * [NOD-828] Fix revert errors. * Revert "[NOD-828] Remove data length from flat file data format." This reverts commit ef408e32 * [NOD-862] Move Cursor() into the DataAccessor interface. * [NOD-828] Add Delete to DataAccessor. * [NOD-865] Move fee data db operations to dbaccess * [NOD-865] Move reachability data db operations to dbaccess * [NOD-865] Move UTXO diff data db operations to dbaccess * [NOD-865] Move subnetwork data db operations to dbaccess * [NOD-865] Fix createDAGState * [NOD-865] Remove old Get signature with "exists" * [NOD-865] Move multiset db operations to dbaccess * [NOD-865] Use dbaccess transactions where possible * [NOD-865] Remove old Get signature with "exists" * [NOD-881] Recover TestGHOSTDAGErrors * [NOD-865] Create function for db keys * [NOD-865] Change Exists to Has, and use accessor.Has where possible * [NOD-865] Make ClearReachabilityData transactive * [NOD-865] Don't iterate cursors while changing db data * [NOD-865] Rename RegisterSubnetwork -> StoreSubnetwork * [NOD-865] Change bucket from utxodiffs to utxo-diffs * [NOD-865] Rename SubnetworkExists->HasSubnetwork * [NOD-865] Change a comment * [NOD-865] Fix tests * [NOD-865] Fix comment * [NOD-865] Remove the prefix "db" from some functions * [NOD-865] Remove redundant comments * [NOD-865] Make clearBucket function * [NOD-865] Make clear functions get a dbTx as an arg * [NOD-865] Remove erroneous tx commit Co-authored-by: stasatdaglabs <stas@daglabs.com> * [NOD-868] Delete the old database package (#683) * [NOD-828] Create the database2 package that will some day replace the database package. * [NOD-828] Implement a "bucket" key mechanism. * [NOD-828] Move bucket.go into the ffldb2 package. * [NOD-828] Delete the un-interfaced ffldb package from database2, since we aren't going to be using it anyway. * [NOD-828] Copy over + fixup flat file structs from the old ffldb. * [NOD-828] Implement flatFilePath. * [NOD-828] Implement flatFileStore.write(). * [NOD-828] Implement flatFileStore.read(). * [NOD-828] Implement flatFileStore.rollback(). * [NOD-828] Sync the file to disk at the end of write(). * [NOD-828] Extract crc32ByteOrder to a separate variable. * [NOD-828] Add a sanity test. * [NOD-828] Remove context-unrelated methods from the Database interface. * [NOD-828] Create an ffldb object. Simply work against a context. * [NOD-828] Open the new database on start. * [NOD-828] Create the leveldb package. * [NOD-828] Implement opening/closing leveldb. * [NOD-828] Implement get/put out of/into leveldb. * [NOD-828] Implement transactions and make them implement a generic database interface. * [NOD-828] Write sanity tests for leveldb with and without transactions. * [NOD-828] Add another case to the transaction sanity test. * [NOD-828] Implement AppendBlock/RetrieveBlock. * [NOD-828] Refactor so that concepts such as "block" and "metadata" don't leak into the database package. * [NOD-828] Add RollbackFlatData to the database interface. * [NOD-828] Remove anything from dbaccess that I'm not planning to implement as part of this ticket. * [NOD-828] Implement StoreBlock. * [NOD-828] Implement FetchBlock. * [NOD-828] Implement HasBlock. * [NOD-828] Write a sanity test for block insertion. * [NOD-828] Implement CurrentFlatDataLocation. * [NOD-828] Implement storing the current block location. * [NOD-828] Implement initializing/syncing the flat file block store and the "metadata". * [NOD-828] Add InitBlockStore to TestBlockStoreSanity. * [NOD-828] Fix rename errors. * [NOD-828] Fix lint errors in the root database package. * [NOD-828] Fix lint errors in the ffldb.go. * [NOD-828] Fix lint errors in the flatfile/db.go. * [NOD-828] Rename packages in such a way to make the linter happy. * [NOD-828] Finish satisfying the linter. * [NOD-828] Fix doc.go. * [NOD-828] Fix comments in block.go. * [NOD-828] Move dbaccess out of the database package. * [NOD-828] Move opening/closing the database to dbaccess. * [NOD-828] Move the Database interface to the root database package, since it's meant to be generic. * [NOD-828] Make ffldb generic to later support additional databases. * [NOD-828] Make ffldb.Open return DatabaseHandle, since ffldb is no longer exported. * [NOD-828] Fix comments. * [NOD-828] Rename AppendFlatData to AppendToStore and RetrieveFlatData to RetrieveFromStore. * [NOD-828] Make buckets nicer to use. * [NOD-828] Implement cursors that iterate over some bucket. * [NOD-828] Generalize flat-file repairing and move block database repairing into ffldb. * [NOD-828] Write a test making sure that flat file repair works. * [NOD-828] Properly close the database in TestRepairFlatFiles. * [NOD-828] Add a comment warning against putting and getting the same data within the same transaction. * [NOD-828] Fix the flatFilesBucket description. * [NOD-828] Remove superfluous whitespace. * [NOD-828] Fix the ffldb description. * [NOD-828] Rename Database to DataAccessor and rename Handle to Database. * [NOD-828] Make Get return nil if the value doesn't exist. * [NOD-828] Attempt to close leveldb even if closing ffdb failed. * [NOD-828] Fix a bug where the wrong location would be written to the current store location bucket. * [NOD-828] Fix not updating the store location in ffldb transactions. * [NOD-828] Make scanFlatFiles return an error if os.Stat fails for any reason other than file-not-found. * [NOD-828] Update the README and doc.go. * [NOD-828] Simplify Bucket.Path(). * [NOD-828] Since LevelDBCursor satisfied the database2.Cursor interface, use it directly. * [NOD-828] Combine two lines into one. * [NOD-828] Combine another two lines into one. * [NOD-828] Move a misplaced comment. * [NOD-828] Use Wrapf instead of Errorf where appropriate. * [NOD-828] Fix a comment. * [NOD-828] Fix a comment. * [NOD-828] Fix Wrapf calls. * [NOD-828] Fix comments. * [NOD-828] Remove superfluous whitespace. * [NOD-828] Add a comment explaining the use of batches and snapshots. * [NOD-828] Implement RollbackUnlessClosed(). * [NOD-828] Return both errors in StoreBlock rollback. * [NOD-828] Move rollback-on-error logic into ffldb. Remove CurrentStoreLocation and RollbackStore from DataAccessor. * [NOD-828] Make bucket a type alias instead of a struct. * [NOD-828] Fix a typo. * [NOD-828] Use copy instead of append in Bucket. * [NOD-828] Extract flatFileLocationSerializedSize to a const. * [NOD-828] Debugf -> Warnf in rollback.go. * [NOD-828] Fix a comment. * [NOD-828] Fix a comment. * [NOD-828] Remove data length from flat file data format. * [NOD-828] Rearrange TestLevelDBTransactionSanity a bit. * [NOD-828] Add stack traces to all errors that come out of library functions. * [NOD-828] Return errors from rollback(). * [NOD-828] Remove an irrelevant comment. * [NOD-828] Remove redundant whitespace. * [NOD-828] Handle nil in FetchBlock. * [NOD-828] Move the explanation about batches and snapshots to the LevelDBTransaction struct. * Revert "[NOD-828] Make bucket a type alias instead of a struct." This reverts commit 1fd39652 * [NOD-828] Fix revert errors. * Revert "[NOD-828] Remove data length from flat file data format." This reverts commit ef408e32 * [NOD-862] Move Cursor() into the DataAccessor interface. * [NOD-828] Add Delete to DataAccessor. * [NOD-865] Move fee data db operations to dbaccess * [NOD-865] Move reachability data db operations to dbaccess * [NOD-865] Move UTXO diff data db operations to dbaccess * [NOD-865] Move subnetwork data db operations to dbaccess * [NOD-865] Fix createDAGState * [NOD-865] Remove old Get signature with "exists" * [NOD-865] Move multiset db operations to dbaccess * [NOD-865] Use dbaccess transactions where possible * [NOD-865] Remove old Get signature with "exists" * [NOD-881] Recover TestGHOSTDAGErrors * [NOD-865] Create function for db keys * [NOD-865] Change Exists to Has, and use accessor.Has where possible * [NOD-865] Make ClearReachabilityData transactive * [NOD-865] Don't iterate cursors while changing db data * [NOD-865] Rename RegisterSubnetwork -> StoreSubnetwork * [NOD-865] Change bucket from utxodiffs to utxo-diffs * [NOD-865] Rename SubnetworkExists->HasSubnetwork * [NOD-865] Change a comment * [NOD-868] Remove all tests from old database. * [NOD-868] Remove all unused methods from the old database's interfaces. * [NOD-865] Fix tests * [NOD-868] Remove references to DB. * [NOD-865] Fix comment * [NOD-868] Remove the old ffldb besides the interface and errors.go. * [NOD-868] Remove errors.go. * [NOD-868] Remove the old database package. * [NOD-868] Add openDB to DAGSetup to emulate the old dbpath in dag.config. * [NOD-868] Rename database2 to database. * [NOD-868] Use NewTx instead of NoTx where required. * [NOD-868] Fix merge errors. * [NOD-868] Rename dbXXX functions to just xxx. * [NOD-868] Rename putDAGState to saveDAGState. * [NOD-868] Replace comments in initDAGState with logs. * [NOD-868] Explain the openDB parameter in DAGSetup. * [NOD-868] Fixup doc.go and README.md. * [NOD-868] Remove pointless transactions. Co-authored-by: Ori Newman <orinewman1@gmail.com> * [NOD-805] Fix merge errors. * [NOD-805] Fix a comment. * [NOD-805] Don't return virtualTxsAcceptanceData from applyDAGChanges. * [NOD-805] Add missing error handling in TestAcceptanceDataIndexRecover. * [NOD-805] Rename blockDAG to dag in indexers/manager.go. * [NOD-805] Defer cursor.Close() everywhere. * [NOD-805] Rename scanFlatFiles to findCurrentLocation. * [NOD-805] Extract crc32ChecksumLength and dataLengthLength to constants. * [NOD-805] Handle open files properly in rollback.go. * [NOD-805] Remove unnecessary func wrapper. * [NOD-805] Remove unnecessary trimming in initialize. * [NOD-805] Made StoreBlock accept only TxContext. * [NOD-805] Changed the log level of an error message to Error. * [NOD-805] Add a note about holding mutexes over deleteFile. * [NOD-805] Remove a false comment. * [NOD-805] Fix a comment. * [NOD-805] Rename blk to block. * [NOD-805] Extract utxoKey to a separate function. * [NOD-805] Move dbaccess.xxxKey functions to the tops of their respective files. * [NOD-805] Fix grammar in dbaccess/db.go. * [NOD-805] Wrap a failed database corruption recovery error. * [NOD-805] Split lines with WithStack in them. * [NOD-805] Fix the comment over initialize. * [NOD-805] Rename ffdb to flatFileDB and ldb to levelDB. * [NOD-805] Fix a comment. * [NOD-805] Fix a comment. * [NOD-805] Use s.writeCursor instead of cursor. * [NOD-805] Embed file in lockableFile. * [NOD-805] the the -> the * [NOD-805] openDB -> db * [NOD-805] Use TxContext in all flushToDB functions. * [NOD-805] Rename context -> dbContext. * [NOD-805] Reword the comment at the beginning on initDAGState. * [NOD-805] Explain cursor key trimming. * [NOD-805] Remove Error from Cursor. * [NOD-805] Return ErrNotFound from done Cursor Key and Value. * [NOD-805] Add missing error handling. * [NOD-805] Fix a comment. * [NOD-805] Fix a variable name. * [NOD-805] Remove pointless underscore. * [NOD-805] Fix a comment. * [NOD-805] Fix a variable name. Co-authored-by: Mike Zak <feanorr@gmail.com> Co-authored-by: Ori Newman <orinewman1@gmail.com>
137 lines
3.8 KiB
Go
137 lines
3.8 KiB
Go
// Copyright (c) 2015-2017 The btcsuite developers
|
|
// Use of this source code is governed by an ISC
|
|
// license that can be found in the LICENSE file.
|
|
|
|
package blockdag
|
|
|
|
import (
|
|
"github.com/kaspanet/kaspad/dbaccess"
|
|
"sync"
|
|
|
|
"github.com/kaspanet/kaspad/dagconfig"
|
|
"github.com/kaspanet/kaspad/util/daghash"
|
|
)
|
|
|
|
// blockIndex provides facilities for keeping track of an in-memory index of the
|
|
// block DAG.
|
|
type blockIndex struct {
|
|
// The following fields are set when the instance is created and can't
|
|
// be changed afterwards, so there is no need to protect them with a
|
|
// separate mutex.
|
|
dagParams *dagconfig.Params
|
|
|
|
sync.RWMutex
|
|
index map[daghash.Hash]*blockNode
|
|
dirty map[*blockNode]struct{}
|
|
}
|
|
|
|
// newBlockIndex returns a new empty instance of a block index. The index will
|
|
// be dynamically populated as block nodes are loaded from the database and
|
|
// manually added.
|
|
func newBlockIndex(dagParams *dagconfig.Params) *blockIndex {
|
|
return &blockIndex{
|
|
dagParams: dagParams,
|
|
index: make(map[daghash.Hash]*blockNode),
|
|
dirty: make(map[*blockNode]struct{}),
|
|
}
|
|
}
|
|
|
|
// HaveBlock returns whether or not the block index contains the provided hash.
|
|
//
|
|
// This function is safe for concurrent access.
|
|
func (bi *blockIndex) HaveBlock(hash *daghash.Hash) bool {
|
|
bi.RLock()
|
|
defer bi.RUnlock()
|
|
_, hasBlock := bi.index[*hash]
|
|
return hasBlock
|
|
}
|
|
|
|
// LookupNode returns the block node identified by the provided hash. It will
|
|
// return nil if there is no entry for the hash.
|
|
//
|
|
// This function is safe for concurrent access.
|
|
func (bi *blockIndex) LookupNode(hash *daghash.Hash) *blockNode {
|
|
bi.RLock()
|
|
defer bi.RUnlock()
|
|
node := bi.index[*hash]
|
|
return node
|
|
}
|
|
|
|
// AddNode adds the provided node to the block index and marks it as dirty.
|
|
// Duplicate entries are not checked so it is up to caller to avoid adding them.
|
|
//
|
|
// This function is safe for concurrent access.
|
|
func (bi *blockIndex) AddNode(node *blockNode) {
|
|
bi.Lock()
|
|
defer bi.Unlock()
|
|
bi.addNode(node)
|
|
bi.dirty[node] = struct{}{}
|
|
}
|
|
|
|
// addNode adds the provided node to the block index, but does not mark it as
|
|
// dirty. This can be used while initializing the block index.
|
|
//
|
|
// This function is NOT safe for concurrent access.
|
|
func (bi *blockIndex) addNode(node *blockNode) {
|
|
bi.index[*node.hash] = node
|
|
}
|
|
|
|
// NodeStatus provides concurrent-safe access to the status field of a node.
|
|
//
|
|
// This function is safe for concurrent access.
|
|
func (bi *blockIndex) NodeStatus(node *blockNode) blockStatus {
|
|
bi.RLock()
|
|
defer bi.RUnlock()
|
|
status := node.status
|
|
return status
|
|
}
|
|
|
|
// SetStatusFlags flips the provided status flags on the block node to on,
|
|
// regardless of whether they were on or off previously. This does not unset any
|
|
// flags currently on.
|
|
//
|
|
// This function is safe for concurrent access.
|
|
func (bi *blockIndex) SetStatusFlags(node *blockNode, flags blockStatus) {
|
|
bi.Lock()
|
|
defer bi.Unlock()
|
|
node.status |= flags
|
|
bi.dirty[node] = struct{}{}
|
|
}
|
|
|
|
// UnsetStatusFlags flips the provided status flags on the block node to off,
|
|
// regardless of whether they were on or off previously.
|
|
//
|
|
// This function is safe for concurrent access.
|
|
func (bi *blockIndex) UnsetStatusFlags(node *blockNode, flags blockStatus) {
|
|
bi.Lock()
|
|
defer bi.Unlock()
|
|
node.status &^= flags
|
|
bi.dirty[node] = struct{}{}
|
|
}
|
|
|
|
// flushToDB writes all dirty block nodes to the database.
|
|
func (bi *blockIndex) flushToDB(dbContext *dbaccess.TxContext) error {
|
|
bi.Lock()
|
|
defer bi.Unlock()
|
|
if len(bi.dirty) == 0 {
|
|
return nil
|
|
}
|
|
|
|
for node := range bi.dirty {
|
|
serializedBlockNode, err := serializeBlockNode(node)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
key := blockIndexKey(node.hash, node.blueScore)
|
|
err = dbaccess.StoreIndexBlock(dbContext, key, serializedBlockNode)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
}
|
|
return nil
|
|
}
|
|
|
|
func (bi *blockIndex) clearDirtyEntries() {
|
|
bi.dirty = make(map[*blockNode]struct{})
|
|
}
|