Add basic support for archival node (#1370)

* Add archival cli flag

* If --archival was activated - don't delete anything

* Fix tests

* Still change block status to StatusHeaderOnly even in archival nodes
This commit is contained in:
Svarog 2021-01-10 10:25:15 +02:00 committed by GitHub
parent 285ae5cd40
commit 49e0a2a2e7
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
22 changed files with 72 additions and 50 deletions

View File

@ -2,9 +2,10 @@ package app
import (
"fmt"
"github.com/kaspanet/kaspad/domain/utxoindex"
"sync/atomic"
"github.com/kaspanet/kaspad/domain/utxoindex"
infrastructuredatabase "github.com/kaspanet/kaspad/infrastructure/db/database"
"github.com/kaspanet/kaspad/domain"
@ -79,7 +80,7 @@ func (a *ComponentManager) Stop() {
func NewComponentManager(cfg *config.Config, db infrastructuredatabase.Database, interrupt chan<- struct{}) (
*ComponentManager, error) {
domain, err := domain.New(cfg.ActiveNetParams, db)
domain, err := domain.New(cfg.ActiveNetParams, db, cfg.IsArchivalNode)
if err != nil {
return nil, err
}

View File

@ -14,7 +14,7 @@ import (
func TestConsensus_GetBlockInfo(t *testing.T) {
testutils.ForAllNets(t, true, func(t *testing.T, params *dagconfig.Params) {
factory := NewFactory()
consensus, teardown, err := factory.NewTestConsensus(params, "TestConsensus_GetBlockInfo")
consensus, teardown, err := factory.NewTestConsensus(params, false, "TestConsensus_GetBlockInfo")
if err != nil {
t.Fatalf("Error setting up consensus: %+v", err)
}

View File

@ -46,10 +46,11 @@ import (
// Factory instantiates new Consensuses
type Factory interface {
NewConsensus(dagParams *dagconfig.Params, db infrastructuredatabase.Database) (externalapi.Consensus, error)
NewTestConsensus(dagParams *dagconfig.Params, testName string) (
NewConsensus(dagParams *dagconfig.Params, db infrastructuredatabase.Database, isArchivalNode bool) (
externalapi.Consensus, error)
NewTestConsensus(dagParams *dagconfig.Params, isArchivalNode bool, testName string) (
tc testapi.TestConsensus, teardown func(keepDataDir bool), err error)
NewTestConsensusWithDataDir(dagParams *dagconfig.Params, dataDir string) (
NewTestConsensusWithDataDir(dagParams *dagconfig.Params, dataDir string, isArchivalNode bool) (
tc testapi.TestConsensus, teardown func(keepDataDir bool), err error)
}
@ -61,7 +62,9 @@ func NewFactory() Factory {
}
// NewConsensus instantiates a new Consensus
func (f *factory) NewConsensus(dagParams *dagconfig.Params, db infrastructuredatabase.Database) (externalapi.Consensus, error) {
func (f *factory) NewConsensus(dagParams *dagconfig.Params, db infrastructuredatabase.Database, isArchivalNode bool) (
externalapi.Consensus, error) {
dbManager := consensusdatabase.New(db)
pruningWindowSizeForCaches := int(dagParams.PruningDepth())
@ -236,6 +239,7 @@ func (f *factory) NewConsensus(dagParams *dagconfig.Params, db infrastructuredat
blockStore,
blockHeaderStore,
utxoDiffStore,
isArchivalNode,
genesisHash,
dagParams.FinalityDepth(),
dagParams.PruningDepth())
@ -348,7 +352,7 @@ func (f *factory) NewConsensus(dagParams *dagconfig.Params, db infrastructuredat
return c, nil
}
func (f *factory) NewTestConsensus(dagParams *dagconfig.Params, testName string) (
func (f *factory) NewTestConsensus(dagParams *dagconfig.Params, isArchivalNode bool, testName string) (
tc testapi.TestConsensus, teardown func(keepDataDir bool), err error) {
dataDir, err := ioutil.TempDir("", testName)
@ -356,17 +360,17 @@ func (f *factory) NewTestConsensus(dagParams *dagconfig.Params, testName string)
return nil, nil, err
}
return f.NewTestConsensusWithDataDir(dagParams, dataDir)
return f.NewTestConsensusWithDataDir(dagParams, dataDir, isArchivalNode)
}
func (f *factory) NewTestConsensusWithDataDir(dagParams *dagconfig.Params, dataDir string) (
func (f *factory) NewTestConsensusWithDataDir(dagParams *dagconfig.Params, dataDir string, isArchivalNode bool) (
tc testapi.TestConsensus, teardown func(keepDataDir bool), err error) {
db, err := ldb.NewLevelDB(dataDir)
if err != nil {
return nil, nil, err
}
consensusAsInterface, err := f.NewConsensus(dagParams, db)
consensusAsInterface, err := f.NewConsensus(dagParams, db, isArchivalNode)
if err != nil {
return nil, nil, err
}

View File

@ -1,10 +1,11 @@
package consensus
import (
"github.com/kaspanet/kaspad/domain/dagconfig"
"github.com/kaspanet/kaspad/infrastructure/db/database/ldb"
"io/ioutil"
"testing"
"github.com/kaspanet/kaspad/domain/dagconfig"
"github.com/kaspanet/kaspad/infrastructure/db/database/ldb"
)
func TestNewConsensus(t *testing.T) {
@ -22,7 +23,7 @@ func TestNewConsensus(t *testing.T) {
t.Fatalf("error in NewLevelDB: %s", err)
}
_, err = f.NewConsensus(dagParams, db)
_, err = f.NewConsensus(dagParams, db, false)
if err != nil {
t.Fatalf("error in NewConsensus: %+v", err)
}

View File

@ -20,7 +20,7 @@ func TestFinality(t *testing.T) {
params.FinalityDuration = 50 * params.TargetTimePerBlock
factory := NewFactory()
consensus, teardown, err := factory.NewTestConsensus(params, "TestFinality")
consensus, teardown, err := factory.NewTestConsensus(params, false, "TestFinality")
if err != nil {
t.Fatalf("Error setting up consensus: %+v", err)
}
@ -232,12 +232,12 @@ func TestBoundedMergeDepth(t *testing.T) {
}
factory := NewFactory()
consensusBuild, teardownFunc1, err := factory.NewTestConsensus(params, "BoundedMergeTestBuild")
consensusBuild, teardownFunc1, err := factory.NewTestConsensus(params, false, "BoundedMergeTestBuild")
if err != nil {
t.Fatalf("TestBoundedMergeDepth: Error setting up consensus: %+v", err)
}
consensusReal, teardownFunc2, err := factory.NewTestConsensus(params, "BoundedMergeTestReal")
consensusReal, teardownFunc2, err := factory.NewTestConsensus(params, false, "BoundedMergeTestReal")
if err != nil {
t.Fatalf("TestBoundedMergeDepth: Error setting up consensus: %+v", err)
}

View File

@ -1,6 +1,8 @@
package blockprocessor_test
import (
"testing"
"github.com/kaspanet/kaspad/domain/consensus"
"github.com/kaspanet/kaspad/domain/consensus/model/externalapi"
"github.com/kaspanet/kaspad/domain/consensus/ruleerrors"
@ -9,13 +11,12 @@ import (
"github.com/kaspanet/kaspad/domain/consensus/utils/testutils"
"github.com/kaspanet/kaspad/domain/dagconfig"
"github.com/pkg/errors"
"testing"
)
func TestBlockStatus(t *testing.T) {
testutils.ForAllNets(t, true, func(t *testing.T, params *dagconfig.Params) {
factory := consensus.NewFactory()
tc, teardown, err := factory.NewTestConsensus(params, "TestBlockStatus")
tc, teardown, err := factory.NewTestConsensus(params, false, "TestBlockStatus")
if err != nil {
t.Fatalf("Error setting up consensus: %+v", err)
}

View File

@ -1,10 +1,11 @@
package blockvalidator_test
import (
"github.com/kaspanet/kaspad/domain/consensus/utils/blockheader"
"math"
"testing"
"github.com/kaspanet/kaspad/domain/consensus/utils/blockheader"
"github.com/kaspanet/kaspad/domain/consensus"
"github.com/kaspanet/kaspad/domain/consensus/model/externalapi"
"github.com/kaspanet/kaspad/domain/consensus/ruleerrors"
@ -21,7 +22,7 @@ func TestChainedTransactions(t *testing.T) {
factory := consensus.NewFactory()
tc, teardown, err := factory.NewTestConsensus(params, "TestChainedTransactions")
tc, teardown, err := factory.NewTestConsensus(params, false, "TestChainedTransactions")
if err != nil {
t.Fatalf("Error setting up consensus: %+v", err)
}
@ -83,7 +84,7 @@ func TestChainedTransactions(t *testing.T) {
func TestCheckBlockSanity(t *testing.T) {
testutils.ForAllNets(t, true, func(t *testing.T, params *dagconfig.Params) {
factory := consensus.NewFactory()
consensus, teardown, err := factory.NewTestConsensus(params, "TestCheckBlockSanity")
consensus, teardown, err := factory.NewTestConsensus(params, false, "TestCheckBlockSanity")
if err != nil {
t.Fatalf("Error setting up consensus: %+v", err)
}

View File

@ -2,9 +2,10 @@ package blockvalidator_test
import (
"errors"
"github.com/kaspanet/kaspad/domain/consensus/utils/blockheader"
"testing"
"github.com/kaspanet/kaspad/domain/consensus/utils/blockheader"
"github.com/kaspanet/kaspad/domain/consensus"
"github.com/kaspanet/kaspad/domain/consensus/model"
"github.com/kaspanet/kaspad/domain/consensus/model/externalapi"
@ -17,7 +18,7 @@ import (
func TestValidateMedianTime(t *testing.T) {
testutils.ForAllNets(t, true, func(t *testing.T, params *dagconfig.Params) {
factory := consensus.NewFactory()
tc, teardown, err := factory.NewTestConsensus(params, "TestValidateMedianTime")
tc, teardown, err := factory.NewTestConsensus(params, false, "TestValidateMedianTime")
if err != nil {
t.Fatalf("Error setting up consensus: %+v", err)
}
@ -86,7 +87,7 @@ func TestValidateMedianTime(t *testing.T) {
func TestCheckParentsIncest(t *testing.T) {
testutils.ForAllNets(t, true, func(t *testing.T, params *dagconfig.Params) {
factory := consensus.NewFactory()
tc, teardown, err := factory.NewTestConsensus(params, "TestCheckParentsIncest")
tc, teardown, err := factory.NewTestConsensus(params, false, "TestCheckParentsIncest")
if err != nil {
t.Fatalf("Error setting up consensus: %+v", err)
}

View File

@ -19,7 +19,7 @@ func TestUTXOCommitment(t *testing.T) {
params.BlockCoinbaseMaturity = 0
factory := consensus.NewFactory()
consensus, teardown, err := factory.NewTestConsensus(params, "TestUTXOCommitment")
consensus, teardown, err := factory.NewTestConsensus(params, false, "TestUTXOCommitment")
if err != nil {
t.Fatalf("Error setting up consensus: %+v", err)
}
@ -115,7 +115,7 @@ func TestPastUTXOMultiset(t *testing.T) {
testutils.ForAllNets(t, true, func(t *testing.T, params *dagconfig.Params) {
factory := consensus.NewFactory()
consensus, teardown, err := factory.NewTestConsensus(params, "TestUTXOCommitment")
consensus, teardown, err := factory.NewTestConsensus(params, false, "TestUTXOCommitment")
if err != nil {
t.Fatalf("Error setting up consensus: %+v", err)
}

View File

@ -1,18 +1,19 @@
package consensusstatemanager_test
import (
"testing"
"github.com/kaspanet/kaspad/domain/consensus"
"github.com/kaspanet/kaspad/domain/consensus/model"
"github.com/kaspanet/kaspad/domain/consensus/model/externalapi"
"github.com/kaspanet/kaspad/domain/consensus/utils/testutils"
"github.com/kaspanet/kaspad/domain/dagconfig"
"testing"
)
func TestCalculateSelectedParentChainChanges(t *testing.T) {
testutils.ForAllNets(t, true, func(t *testing.T, params *dagconfig.Params) {
factory := consensus.NewFactory()
consensus, teardown, err := factory.NewTestConsensus(params, "TestCalculateSelectedParentChainChanges")
consensus, teardown, err := factory.NewTestConsensus(params, false, "TestCalculateSelectedParentChainChanges")
if err != nil {
t.Fatalf("Error setting up consensus: %+v", err)
}

View File

@ -2,10 +2,11 @@ package consensusstatemanager_test
import (
"errors"
"testing"
"github.com/kaspanet/kaspad/domain/consensus/model"
"github.com/kaspanet/kaspad/domain/consensus/utils/constants"
"github.com/kaspanet/kaspad/domain/consensus/utils/subnetworks"
"testing"
"github.com/kaspanet/kaspad/domain/consensus/ruleerrors"
@ -24,7 +25,7 @@ func TestDoubleSpends(t *testing.T) {
factory := consensus.NewFactory()
consensus, teardown, err := factory.NewTestConsensus(params, "TestUTXOCommitment")
consensus, teardown, err := factory.NewTestConsensus(params, false, "TestUTXOCommitment")
if err != nil {
t.Fatalf("Error setting up consensus: %+v", err)
}
@ -161,7 +162,7 @@ func TestTransactionAcceptance(t *testing.T) {
params.BlockCoinbaseMaturity = 0
factory := consensus.NewFactory()
testConsensus, teardown, err := factory.NewTestConsensus(params, "TestTransactionAcceptance")
testConsensus, teardown, err := factory.NewTestConsensus(params, false, "TestTransactionAcceptance")
if err != nil {
t.Fatalf("Error setting up testConsensus: %+v", err)
}
@ -381,7 +382,7 @@ func TestTransactionAcceptance(t *testing.T) {
func TestResolveBlockStatusSanity(t *testing.T) {
testutils.ForAllNets(t, true, func(t *testing.T, params *dagconfig.Params) {
consensus, teardown, err := consensus.NewFactory().NewTestConsensus(params, "TestResolveBlockStatusSanity")
consensus, teardown, err := consensus.NewFactory().NewTestConsensus(params, false, "TestResolveBlockStatusSanity")
if err != nil {
t.Fatalf("Error setting up consensus: %+v", err)
}

View File

@ -1,6 +1,9 @@
package consensusstatemanager_test
import (
"sort"
"testing"
"github.com/kaspanet/kaspad/domain/consensus"
"github.com/kaspanet/kaspad/domain/consensus/model"
"github.com/kaspanet/kaspad/domain/consensus/model/externalapi"
@ -8,13 +11,11 @@ import (
"github.com/kaspanet/kaspad/domain/consensus/utils/consensushashing"
"github.com/kaspanet/kaspad/domain/consensus/utils/testutils"
"github.com/kaspanet/kaspad/domain/dagconfig"
"sort"
"testing"
)
func TestConsensusStateManager_pickVirtualParents(t *testing.T) {
testutils.ForAllNets(t, true, func(t *testing.T, params *dagconfig.Params) {
tc, teardown, err := consensus.NewFactory().NewTestConsensus(params, "TestConsensusStateManager_pickVirtualParents")
tc, teardown, err := consensus.NewFactory().NewTestConsensus(params, false, "TestConsensusStateManager_pickVirtualParents")
if err != nil {
t.Fatalf("Error setting up tc: %+v", err)
}

View File

@ -12,7 +12,7 @@ import (
func TestIsAncestorOf(t *testing.T) {
testutils.ForAllNets(t, true, func(t *testing.T, params *dagconfig.Params) {
factory := consensus.NewFactory()
tc, tearDown, err := factory.NewTestConsensus(params, "TestIsAncestorOf")
tc, tearDown, err := factory.NewTestConsensus(params, false, "TestIsAncestorOf")
if err != nil {
t.Fatalf("NewTestConsensus: %s", err)
}

View File

@ -311,7 +311,7 @@ func TestBlueBlockWindow(t *testing.T) {
testutils.ForAllNets(t, true, func(t *testing.T, params *dagconfig.Params) {
params.K = 1
factory := consensus.NewFactory()
tc, tearDown, err := factory.NewTestConsensus(params, "TestBlueBlockWindow")
tc, tearDown, err := factory.NewTestConsensus(params, false, "TestBlueBlockWindow")
if err != nil {
t.Fatalf("NewTestConsensus: %s", err)
}

View File

@ -33,7 +33,7 @@ func TestDifficulty(t *testing.T) {
params.DifficultyAdjustmentWindowSize = 264
factory := consensus.NewFactory()
tc, teardown, err := factory.NewTestConsensus(params, "TestDifficulty")
tc, teardown, err := factory.NewTestConsensus(params, false, "TestDifficulty")
if err != nil {
t.Fatalf("Error setting up consensus: %+v", err)
}

View File

@ -13,7 +13,7 @@ import (
func TestPastMedianTime(t *testing.T) {
testutils.ForAllNets(t, true, func(t *testing.T, params *dagconfig.Params) {
factory := consensus.NewFactory()
tc, tearDown, err := factory.NewTestConsensus(params, "TestUpdateReindexRoot")
tc, tearDown, err := factory.NewTestConsensus(params, false, "TestUpdateReindexRoot")
if err != nil {
t.Fatalf("NewTestConsensus: %s", err)
}

View File

@ -67,7 +67,7 @@ func TestPruning(t *testing.T) {
params.MergeSetSizeLimit = test.MergeSetSizeLimit
factory := consensus.NewFactory()
tc, teardown, err := factory.NewTestConsensus(params, "TestPruning")
tc, teardown, err := factory.NewTestConsensus(params, false, "TestPruning")
if err != nil {
t.Fatalf("Error setting up consensus: %+v", err)
}

View File

@ -28,6 +28,7 @@ type pruningManager struct {
blockHeaderStore model.BlockHeaderStore
utxoDiffStore model.UTXODiffStore
isArchivalNode bool
genesisHash *externalapi.DomainHash
finalityInterval uint64
pruningDepth uint64
@ -52,6 +53,7 @@ func New(
blockHeaderStore model.BlockHeaderStore,
utxoDiffStore model.UTXODiffStore,
isArchivalNode bool,
genesisHash *externalapi.DomainHash,
finalityInterval uint64,
pruningDepth uint64,
@ -72,6 +74,7 @@ func New(
blockHeaderStore: blockHeaderStore,
utxoDiffStore: utxoDiffStore,
headerSelectedTipStore: headerSelectedTipStore,
isArchivalNode: isArchivalNode,
genesisHash: genesisHash,
pruningDepth: pruningDepth,
finalityInterval: finalityInterval,
@ -328,12 +331,16 @@ func (pm *pruningManager) deleteBlock(blockHash *externalapi.DomainHash) (alread
return true, nil
}
pm.blockStatusStore.Stage(blockHash, externalapi.StatusHeaderOnly)
if pm.isArchivalNode {
return false, nil
}
pm.multiSetStore.Delete(blockHash)
pm.acceptanceDataStore.Delete(blockHash)
pm.blocksStore.Delete(blockHash)
pm.utxoDiffStore.Delete(blockHash)
pm.blockStatusStore.Stage(blockHash, externalapi.StatusHeaderOnly)
return false, nil
}

View File

@ -13,7 +13,8 @@ func TestAddChildThatPointsDirectlyToTheSelectedParentChainBelowReindexRoot(t *t
reachabilityReindexWindow := uint64(10)
testutils.ForAllNets(t, true, func(t *testing.T, params *dagconfig.Params) {
factory := consensus.NewFactory()
tc, tearDown, err := factory.NewTestConsensus(params, "TestAddChildThatPointsDirectlyToTheSelectedParentChainBelowReindexRoot")
tc, tearDown, err := factory.NewTestConsensus(params, false,
"TestAddChildThatPointsDirectlyToTheSelectedParentChainBelowReindexRoot")
if err != nil {
t.Fatalf("NewTestConsensus: %+v", err)
}
@ -68,7 +69,7 @@ func TestUpdateReindexRoot(t *testing.T) {
reachabilityReindexWindow := uint64(10)
testutils.ForAllNets(t, true, func(t *testing.T, params *dagconfig.Params) {
factory := consensus.NewFactory()
tc, tearDown, err := factory.NewTestConsensus(params, "TestUpdateReindexRoot")
tc, tearDown, err := factory.NewTestConsensus(params, false, "TestUpdateReindexRoot")
if err != nil {
t.Fatalf("NewTestConsensus: %s", err)
}
@ -158,7 +159,7 @@ func TestReindexIntervalsEarlierThanReindexRoot(t *testing.T) {
reachabilityReindexWindow := uint64(10)
testutils.ForAllNets(t, true, func(t *testing.T, params *dagconfig.Params) {
factory := consensus.NewFactory()
tc, tearDown, err := factory.NewTestConsensus(params, "TestUpdateReindexRoot")
tc, tearDown, err := factory.NewTestConsensus(params, false, "TestUpdateReindexRoot")
if err != nil {
t.Fatalf("NewTestConsensus: %+v", err)
}
@ -292,7 +293,7 @@ func TestTipsAfterReindexIntervalsEarlierThanReindexRoot(t *testing.T) {
reachabilityReindexWindow := uint64(10)
testutils.ForAllNets(t, true, func(t *testing.T, params *dagconfig.Params) {
factory := consensus.NewFactory()
tc, tearDown, err := factory.NewTestConsensus(params, "TestUpdateReindexRoot")
tc, tearDown, err := factory.NewTestConsensus(params, false, "TestUpdateReindexRoot")
if err != nil {
t.Fatalf("NewTestConsensus: %s", err)
}

View File

@ -1,9 +1,10 @@
package transactionvalidator_test
import (
"github.com/kaspanet/kaspad/domain/consensus/utils/hashes"
"testing"
"github.com/kaspanet/kaspad/domain/consensus/utils/hashes"
"github.com/kaspanet/kaspad/domain/consensus"
"github.com/kaspanet/kaspad/domain/consensus/model/externalapi"
"github.com/kaspanet/kaspad/domain/consensus/ruleerrors"
@ -25,7 +26,7 @@ type txSubnetworkData struct {
func TestValidateTransactionInIsolation(t *testing.T) {
testutils.ForAllNets(t, true, func(t *testing.T, params *dagconfig.Params) {
factory := consensus.NewFactory()
tc, teardown, err := factory.NewTestConsensus(params, "TestValidateTransactionInIsolation")
tc, teardown, err := factory.NewTestConsensus(params, false, "TestValidateTransactionInIsolation")
if err != nil {
t.Fatalf("Error setting up consensus: %+v", err)
}

View File

@ -28,9 +28,9 @@ func (d domain) MiningManager() miningmanager.MiningManager {
}
// New instantiates a new instance of a Domain object
func New(dagParams *dagconfig.Params, db infrastructuredatabase.Database) (Domain, error) {
func New(dagParams *dagconfig.Params, db infrastructuredatabase.Database, isArchivalNode bool) (Domain, error) {
consensusFactory := consensus.NewFactory()
consensusInstance, err := consensusFactory.NewConsensus(dagParams, db)
consensusInstance, err := consensusFactory.NewConsensus(dagParams, db, isArchivalNode)
if err != nil {
return nil, err
}

View File

@ -121,6 +121,7 @@ type Flags struct {
ResetDatabase bool `long:"reset-db" description:"Reset database before starting node. It's needed when switching between subnetworks."`
MaxUTXOCacheSize uint64 `long:"maxutxocachesize" description:"Max size of loaded UTXO into ram from the disk in bytes"`
UTXOIndex bool `long:"utxoindex" description:"Enable the UTXO index"`
IsArchivalNode bool `long:"archival" description:"Run as an archival node: don't delete old block data when moving the pruning point (Warning: heavy disk usage)'"`
NetworkFlags
ServiceOptions *ServiceOptions
}