From 1cedc720acbeac6056fa4ef432a1cc9214a6d0c2 Mon Sep 17 00:00:00 2001 From: Ori Newman Date: Wed, 2 Mar 2022 12:31:14 +0200 Subject: [PATCH 01/11] patch --- app/protocol/flows/v4/blockrelay/ibd.go | 61 ++++++++++++++++++++----- 1 file changed, 49 insertions(+), 12 deletions(-) diff --git a/app/protocol/flows/v4/blockrelay/ibd.go b/app/protocol/flows/v4/blockrelay/ibd.go index 1c9e1848e..24095631a 100644 --- a/app/protocol/flows/v4/blockrelay/ibd.go +++ b/app/protocol/flows/v4/blockrelay/ibd.go @@ -77,14 +77,35 @@ func (flow *handleIBDFlow) runIBDIfNotRunning(block *externalapi.DomainBlock) er }() highHash := consensushashing.BlockHash(block) - log.Debugf("IBD started with peer %s and highHash %s", flow.peer, highHash) - log.Debugf("Syncing blocks up to %s", highHash) - log.Debugf("Trying to find highest shared chain block with peer %s with high hash %s", flow.peer, highHash) + log.Criticalf("IBD started with peer %s and highHash %s", flow.peer, highHash) + log.Criticalf("Syncing blocks up to %s", highHash) + log.Criticalf("Trying to find highest shared chain block with peer %s with high hash %s", flow.peer, highHash) highestSharedBlockHash, highestSharedBlockFound, err := flow.findHighestSharedBlockHash(highHash) if err != nil { return err } - log.Debugf("Found highest shared chain block %s with peer %s", highestSharedBlockHash, flow.peer) + log.Criticalf("Found highest shared chain block %s with peer %s", highestSharedBlockHash, flow.peer) + checkpoint, err := externalapi.NewDomainHashFromString("05ff0f2e1d201dcaee7c5e567cc2c1d42ca3cce9fefbd3b519dc68b5bb89d0b9") + if err != nil { + return err + } + + info, err := flow.Domain().Consensus().GetBlockInfo(checkpoint) + if err != nil { + return err + } + + if info.Exists { + isInSelectedParentChainOf, err := flow.Domain().Consensus().IsInSelectedParentChainOf(checkpoint, highestSharedBlockHash) + if err != nil { + return err + } + + if !isInSelectedParentChainOf { + log.Criticalf("Stopped IBD because the checkpoint %s is not in the selected chain of %s", checkpoint, highestSharedBlockHash) + return nil + } + } shouldDownloadHeadersProof, shouldSync, err := flow.shouldSyncAndShouldDownloadHeadersProof(block, highestSharedBlockFound) if err != nil { @@ -308,6 +329,7 @@ func (flow *handleIBDFlow) syncPruningPointFutureHeaders(consensus externalapi.C } }) + count := 0 for { select { case ibdBlocksMessage, ok := <-blockHeadersMessageChan: @@ -324,10 +346,26 @@ func (flow *handleIBDFlow) syncPruningPointFutureHeaders(consensus externalapi.C return nil } for _, header := range ibdBlocksMessage.BlockHeaders { - err = flow.processHeader(consensus, header) + added, err := flow.processHeader(consensus, header) if err != nil { return err } + + if added { + count++ + log.Criticalf("LALA %d Accepted header %s DAA score %d blue score %d Arrived at %d ( %s ) timestamp %d ( %s ) diff %d ( %s )", + count, + header.BlockHash(), + header.DAAScore, + header.BlueScore, + time.Now().UnixMilli(), + time.Now(), + header.Timestamp.UnixMilliseconds(), + header.Timestamp, + time.Now().UnixMilli()-header.Timestamp.UnixMilliseconds(), + time.Millisecond*time.Duration(time.Now().UnixMilli()-header.Timestamp.UnixMilliseconds()), + ) + } } lastReceivedHeader := ibdBlocksMessage.BlockHeaders[len(ibdBlocksMessage.BlockHeaders)-1] @@ -365,7 +403,7 @@ func (flow *handleIBDFlow) receiveHeaders() (msgIBDBlock *appmessage.BlockHeader } } -func (flow *handleIBDFlow) processHeader(consensus externalapi.Consensus, msgBlockHeader *appmessage.MsgBlockHeader) error { +func (flow *handleIBDFlow) processHeader(consensus externalapi.Consensus, msgBlockHeader *appmessage.MsgBlockHeader) (bool, error) { header := appmessage.BlockHeaderToDomainBlockHeader(msgBlockHeader) block := &externalapi.DomainBlock{ Header: header, @@ -375,27 +413,26 @@ func (flow *handleIBDFlow) processHeader(consensus externalapi.Consensus, msgBlo blockHash := consensushashing.BlockHash(block) blockInfo, err := consensus.GetBlockInfo(blockHash) if err != nil { - return err + return false, err } if blockInfo.Exists { log.Debugf("Block header %s is already in the DAG. Skipping...", blockHash) - return nil + return false, nil } _, err = consensus.ValidateAndInsertBlock(block, false) if err != nil { if !errors.As(err, &ruleerrors.RuleError{}) { - return errors.Wrapf(err, "failed to process header %s during IBD", blockHash) + return false, errors.Wrapf(err, "failed to process header %s during IBD", blockHash) } if errors.Is(err, ruleerrors.ErrDuplicateBlock) { log.Debugf("Skipping block header %s as it is a duplicate", blockHash) } else { log.Infof("Rejected block header %s from %s during IBD: %s", blockHash, flow.peer, err) - return protocolerrors.Wrapf(true, err, "got invalid block header %s during IBD", blockHash) + return false, protocolerrors.Wrapf(true, err, "got invalid block header %s during IBD", blockHash) } } - - return nil + return true, nil } func (flow *handleIBDFlow) validatePruningPointFutureHeaderTimestamps() error { From ef1a3c0dcedc113a5da707951ce1fe32bf38603b Mon Sep 17 00:00:00 2001 From: Ori Newman Date: Wed, 2 Mar 2022 13:09:34 +0200 Subject: [PATCH 02/11] remove debug log --- app/protocol/flows/v4/blockrelay/ibd.go | 18 +----------------- 1 file changed, 1 insertion(+), 17 deletions(-) diff --git a/app/protocol/flows/v4/blockrelay/ibd.go b/app/protocol/flows/v4/blockrelay/ibd.go index 24095631a..2bab4758a 100644 --- a/app/protocol/flows/v4/blockrelay/ibd.go +++ b/app/protocol/flows/v4/blockrelay/ibd.go @@ -346,26 +346,10 @@ func (flow *handleIBDFlow) syncPruningPointFutureHeaders(consensus externalapi.C return nil } for _, header := range ibdBlocksMessage.BlockHeaders { - added, err := flow.processHeader(consensus, header) + _, err := flow.processHeader(consensus, header) if err != nil { return err } - - if added { - count++ - log.Criticalf("LALA %d Accepted header %s DAA score %d blue score %d Arrived at %d ( %s ) timestamp %d ( %s ) diff %d ( %s )", - count, - header.BlockHash(), - header.DAAScore, - header.BlueScore, - time.Now().UnixMilli(), - time.Now(), - header.Timestamp.UnixMilliseconds(), - header.Timestamp, - time.Now().UnixMilli()-header.Timestamp.UnixMilliseconds(), - time.Millisecond*time.Duration(time.Now().UnixMilli()-header.Timestamp.UnixMilliseconds()), - ) - } } lastReceivedHeader := ibdBlocksMessage.BlockHeaders[len(ibdBlocksMessage.BlockHeaders)-1] From b5eda334885d475a36323222e19032a8fe2f3351 Mon Sep 17 00:00:00 2001 From: Ori Newman Date: Wed, 2 Mar 2022 13:11:32 +0200 Subject: [PATCH 03/11] remove count --- app/protocol/flows/v4/blockrelay/ibd.go | 1 - 1 file changed, 1 deletion(-) diff --git a/app/protocol/flows/v4/blockrelay/ibd.go b/app/protocol/flows/v4/blockrelay/ibd.go index 2bab4758a..ba37a90fd 100644 --- a/app/protocol/flows/v4/blockrelay/ibd.go +++ b/app/protocol/flows/v4/blockrelay/ibd.go @@ -329,7 +329,6 @@ func (flow *handleIBDFlow) syncPruningPointFutureHeaders(consensus externalapi.C } }) - count := 0 for { select { case ibdBlocksMessage, ok := <-blockHeadersMessageChan: From 4d3f504b7366cadab46861f3f7028164026b94e8 Mon Sep 17 00:00:00 2001 From: Ori Newman Date: Wed, 2 Mar 2022 21:17:46 +0200 Subject: [PATCH 04/11] Check checkpoint only if highestSharedBlockFound --- app/protocol/flows/v4/blockrelay/ibd.go | 32 +++++++++++++------------ 1 file changed, 17 insertions(+), 15 deletions(-) diff --git a/app/protocol/flows/v4/blockrelay/ibd.go b/app/protocol/flows/v4/blockrelay/ibd.go index ba37a90fd..b408e671a 100644 --- a/app/protocol/flows/v4/blockrelay/ibd.go +++ b/app/protocol/flows/v4/blockrelay/ibd.go @@ -85,25 +85,27 @@ func (flow *handleIBDFlow) runIBDIfNotRunning(block *externalapi.DomainBlock) er return err } log.Criticalf("Found highest shared chain block %s with peer %s", highestSharedBlockHash, flow.peer) - checkpoint, err := externalapi.NewDomainHashFromString("05ff0f2e1d201dcaee7c5e567cc2c1d42ca3cce9fefbd3b519dc68b5bb89d0b9") - if err != nil { - return err - } - - info, err := flow.Domain().Consensus().GetBlockInfo(checkpoint) - if err != nil { - return err - } - - if info.Exists { - isInSelectedParentChainOf, err := flow.Domain().Consensus().IsInSelectedParentChainOf(checkpoint, highestSharedBlockHash) + if highestSharedBlockFound { + checkpoint, err := externalapi.NewDomainHashFromString("05ff0f2e1d201dcaee7c5e567cc2c1d42ca3cce9fefbd3b519dc68b5bb89d0b9") if err != nil { return err } - if !isInSelectedParentChainOf { - log.Criticalf("Stopped IBD because the checkpoint %s is not in the selected chain of %s", checkpoint, highestSharedBlockHash) - return nil + info, err := flow.Domain().Consensus().GetBlockInfo(checkpoint) + if err != nil { + return err + } + + if info.Exists { + isInSelectedParentChainOf, err := flow.Domain().Consensus().IsInSelectedParentChainOf(checkpoint, highestSharedBlockHash) + if err != nil { + return err + } + + if !isInSelectedParentChainOf { + log.Criticalf("Stopped IBD because the checkpoint %s is not in the selected chain of %s", checkpoint, highestSharedBlockHash) + return nil + } } } From 20f16cf729a0410cc8999128674f05f60b026632 Mon Sep 17 00:00:00 2001 From: msutton Date: Sun, 6 Mar 2022 01:03:44 +0200 Subject: [PATCH 05/11] Update checkpoint to new side-chain --- app/protocol/flows/v4/blockrelay/ibd.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/app/protocol/flows/v4/blockrelay/ibd.go b/app/protocol/flows/v4/blockrelay/ibd.go index b408e671a..0b7abb13d 100644 --- a/app/protocol/flows/v4/blockrelay/ibd.go +++ b/app/protocol/flows/v4/blockrelay/ibd.go @@ -86,7 +86,7 @@ func (flow *handleIBDFlow) runIBDIfNotRunning(block *externalapi.DomainBlock) er } log.Criticalf("Found highest shared chain block %s with peer %s", highestSharedBlockHash, flow.peer) if highestSharedBlockFound { - checkpoint, err := externalapi.NewDomainHashFromString("05ff0f2e1d201dcaee7c5e567cc2c1d42ca3cce9fefbd3b519dc68b5bb89d0b9") + checkpoint, err := externalapi.NewDomainHashFromString("efd297d4ff50c02571268bd47894503d2c8d02a0b2e0efa926fca193b00ec2b6") if err != nil { return err } From cb5e9b55b7dccacb2eed5560f64f58456de3439b Mon Sep 17 00:00:00 2001 From: msutton Date: Mon, 7 Mar 2022 14:54:05 +0200 Subject: [PATCH 06/11] Update checkpoint to yet another side-chain --- app/protocol/flows/v4/blockrelay/ibd.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/app/protocol/flows/v4/blockrelay/ibd.go b/app/protocol/flows/v4/blockrelay/ibd.go index 0b7abb13d..5fb11c4c2 100644 --- a/app/protocol/flows/v4/blockrelay/ibd.go +++ b/app/protocol/flows/v4/blockrelay/ibd.go @@ -86,7 +86,7 @@ func (flow *handleIBDFlow) runIBDIfNotRunning(block *externalapi.DomainBlock) er } log.Criticalf("Found highest shared chain block %s with peer %s", highestSharedBlockHash, flow.peer) if highestSharedBlockFound { - checkpoint, err := externalapi.NewDomainHashFromString("efd297d4ff50c02571268bd47894503d2c8d02a0b2e0efa926fca193b00ec2b6") + checkpoint, err := externalapi.NewDomainHashFromString("cdae6724abde94eebbc5cd2d8aa0861a39e5661526771c027ef2ebce32f76bbe") if err != nil { return err } From 685c049a12d1c841b69d46e1d604bbbf5e9491ef Mon Sep 17 00:00:00 2001 From: msutton Date: Mon, 7 Mar 2022 15:51:59 +0200 Subject: [PATCH 07/11] yet another checkpoint --- app/protocol/flows/v4/blockrelay/ibd.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/app/protocol/flows/v4/blockrelay/ibd.go b/app/protocol/flows/v4/blockrelay/ibd.go index 5fb11c4c2..b428845fa 100644 --- a/app/protocol/flows/v4/blockrelay/ibd.go +++ b/app/protocol/flows/v4/blockrelay/ibd.go @@ -86,7 +86,7 @@ func (flow *handleIBDFlow) runIBDIfNotRunning(block *externalapi.DomainBlock) er } log.Criticalf("Found highest shared chain block %s with peer %s", highestSharedBlockHash, flow.peer) if highestSharedBlockFound { - checkpoint, err := externalapi.NewDomainHashFromString("cdae6724abde94eebbc5cd2d8aa0861a39e5661526771c027ef2ebce32f76bbe") + checkpoint, err := externalapi.NewDomainHashFromString("f4a415f28990806a899a208b77930fa5a58f3a94876c3cbe814e60a7ed22824f") if err != nil { return err } From c903a65defda9a8be0d2ef3c0f174b6f167535a6 Mon Sep 17 00:00:00 2001 From: msutton Date: Tue, 8 Mar 2022 03:51:06 +0200 Subject: [PATCH 08/11] a temp patch for fixing IBD issues for all side-chains --- app/protocol/flows/v4/blockrelay/ibd.go | 26 +++++++++++++------------ 1 file changed, 14 insertions(+), 12 deletions(-) diff --git a/app/protocol/flows/v4/blockrelay/ibd.go b/app/protocol/flows/v4/blockrelay/ibd.go index b428845fa..c722e809e 100644 --- a/app/protocol/flows/v4/blockrelay/ibd.go +++ b/app/protocol/flows/v4/blockrelay/ibd.go @@ -13,7 +13,9 @@ import ( "github.com/kaspanet/kaspad/infrastructure/config" "github.com/kaspanet/kaspad/infrastructure/logger" "github.com/kaspanet/kaspad/infrastructure/network/netadapter/router" + "github.com/kaspanet/kaspad/util/difficulty" "github.com/pkg/errors" + "math/big" "time" ) @@ -85,25 +87,25 @@ func (flow *handleIBDFlow) runIBDIfNotRunning(block *externalapi.DomainBlock) er return err } log.Criticalf("Found highest shared chain block %s with peer %s", highestSharedBlockHash, flow.peer) + if highestSharedBlockFound { - checkpoint, err := externalapi.NewDomainHashFromString("f4a415f28990806a899a208b77930fa5a58f3a94876c3cbe814e60a7ed22824f") + virtualSelectedParent, err := flow.Domain().Consensus().GetVirtualSelectedParent() if err != nil { return err } - - info, err := flow.Domain().Consensus().GetBlockInfo(checkpoint) + virtualSelectedParentHeader, err := flow.Domain().Consensus().GetBlockHeader(virtualSelectedParent) if err != nil { return err } - - if info.Exists { - isInSelectedParentChainOf, err := flow.Domain().Consensus().IsInSelectedParentChainOf(checkpoint, highestSharedBlockHash) - if err != nil { - return err - } - - if !isInSelectedParentChainOf { - log.Criticalf("Stopped IBD because the checkpoint %s is not in the selected chain of %s", checkpoint, highestSharedBlockHash) + if virtualSelectedParentHeader.DAAScore() > block.Header.DAAScore()+2641*3 { + virtualDifficulty := difficulty.CalcWork(virtualSelectedParentHeader.Bits()) + var virtualSub, difficultyMul big.Int + if difficultyMul.Mul(virtualDifficulty, big.NewInt(180)). + Cmp(virtualSub.Sub(virtualSelectedParentHeader.BlueWork(), block.Header.BlueWork())) < 0 { + log.Criticalf("Stopped IBD because it is coming from a deep (%d DAA score depth) "+ + "side-chain which split at %s and has lower blue work (%d, %d)", + virtualSelectedParentHeader.DAAScore()-block.Header.DAAScore(), + highestSharedBlockHash, virtualSelectedParentHeader.BlueWork(), block.Header.BlueWork()) return nil } } From 7c327683d36fa3033c3eb4e9db98f0e71330ed14 Mon Sep 17 00:00:00 2001 From: msutton Date: Tue, 8 Mar 2022 09:12:53 +0200 Subject: [PATCH 09/11] route capacity workaround (for new syncing nodes) --- infrastructure/network/netadapter/router/route.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/infrastructure/network/netadapter/router/route.go b/infrastructure/network/netadapter/router/route.go index afaac1a21..a44d3b7ab 100644 --- a/infrastructure/network/netadapter/router/route.go +++ b/infrastructure/network/netadapter/router/route.go @@ -12,7 +12,7 @@ import ( const ( // DefaultMaxMessages is the default capacity for a route with a capacity defined - DefaultMaxMessages = 100 + DefaultMaxMessages = 1000 ) var ( From 09cebe696088d171902cb06fa8b195bbd2096db6 Mon Sep 17 00:00:00 2001 From: msutton Date: Tue, 8 Mar 2022 09:18:24 +0200 Subject: [PATCH 10/11] Perform side-chain check earlier to avoid IBD start --- app/protocol/flows/v4/blockrelay/ibd.go | 43 ++++++++++++------------- 1 file changed, 20 insertions(+), 23 deletions(-) diff --git a/app/protocol/flows/v4/blockrelay/ibd.go b/app/protocol/flows/v4/blockrelay/ibd.go index c722e809e..901f06230 100644 --- a/app/protocol/flows/v4/blockrelay/ibd.go +++ b/app/protocol/flows/v4/blockrelay/ibd.go @@ -66,6 +66,26 @@ func (flow *handleIBDFlow) start() error { } func (flow *handleIBDFlow) runIBDIfNotRunning(block *externalapi.DomainBlock) error { + // Temp code to avoid IBD from lagging nodes publishing there side-chain + virtualSelectedParent, err := flow.Domain().Consensus().GetVirtualSelectedParent() + if err == nil { + virtualSelectedParentHeader, err := flow.Domain().Consensus().GetBlockHeader(virtualSelectedParent) + if err == nil { + if virtualSelectedParentHeader.DAAScore() > block.Header.DAAScore()+2641 { + virtualDifficulty := difficulty.CalcWork(virtualSelectedParentHeader.Bits()) + var virtualSub, difficultyMul big.Int + if difficultyMul.Mul(virtualDifficulty, big.NewInt(180)). + Cmp(virtualSub.Sub(virtualSelectedParentHeader.BlueWork(), block.Header.BlueWork())) < 0 { + log.Criticalf("Avoiding IBD because it is coming from a deep (%d DAA score depth) "+ + "side-chain which has much lower blue work (%d, %d)", + virtualSelectedParentHeader.DAAScore()-block.Header.DAAScore(), + virtualSelectedParentHeader.BlueWork(), block.Header.BlueWork()) + return nil + } + } + } + } + wasIBDNotRunning := flow.TrySetIBDRunning(flow.peer) if !wasIBDNotRunning { log.Debugf("IBD is already running") @@ -88,29 +108,6 @@ func (flow *handleIBDFlow) runIBDIfNotRunning(block *externalapi.DomainBlock) er } log.Criticalf("Found highest shared chain block %s with peer %s", highestSharedBlockHash, flow.peer) - if highestSharedBlockFound { - virtualSelectedParent, err := flow.Domain().Consensus().GetVirtualSelectedParent() - if err != nil { - return err - } - virtualSelectedParentHeader, err := flow.Domain().Consensus().GetBlockHeader(virtualSelectedParent) - if err != nil { - return err - } - if virtualSelectedParentHeader.DAAScore() > block.Header.DAAScore()+2641*3 { - virtualDifficulty := difficulty.CalcWork(virtualSelectedParentHeader.Bits()) - var virtualSub, difficultyMul big.Int - if difficultyMul.Mul(virtualDifficulty, big.NewInt(180)). - Cmp(virtualSub.Sub(virtualSelectedParentHeader.BlueWork(), block.Header.BlueWork())) < 0 { - log.Criticalf("Stopped IBD because it is coming from a deep (%d DAA score depth) "+ - "side-chain which split at %s and has lower blue work (%d, %d)", - virtualSelectedParentHeader.DAAScore()-block.Header.DAAScore(), - highestSharedBlockHash, virtualSelectedParentHeader.BlueWork(), block.Header.BlueWork()) - return nil - } - } - } - shouldDownloadHeadersProof, shouldSync, err := flow.shouldSyncAndShouldDownloadHeadersProof(block, highestSharedBlockFound) if err != nil { return err From 9df231f810e81efedb76f239a56bfbf78885b82c Mon Sep 17 00:00:00 2001 From: msutton Date: Tue, 8 Mar 2022 09:29:00 +0200 Subject: [PATCH 11/11] added relay hash to the log print --- app/protocol/flows/v4/blockrelay/ibd.go | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/app/protocol/flows/v4/blockrelay/ibd.go b/app/protocol/flows/v4/blockrelay/ibd.go index 901f06230..b7ff20be4 100644 --- a/app/protocol/flows/v4/blockrelay/ibd.go +++ b/app/protocol/flows/v4/blockrelay/ibd.go @@ -66,7 +66,9 @@ func (flow *handleIBDFlow) start() error { } func (flow *handleIBDFlow) runIBDIfNotRunning(block *externalapi.DomainBlock) error { - // Temp code to avoid IBD from lagging nodes publishing there side-chain + highHash := consensushashing.BlockHash(block) + + // Temp code to avoid IBD from lagging nodes publishing their side-chain virtualSelectedParent, err := flow.Domain().Consensus().GetVirtualSelectedParent() if err == nil { virtualSelectedParentHeader, err := flow.Domain().Consensus().GetBlockHeader(virtualSelectedParent) @@ -76,8 +78,9 @@ func (flow *handleIBDFlow) runIBDIfNotRunning(block *externalapi.DomainBlock) er var virtualSub, difficultyMul big.Int if difficultyMul.Mul(virtualDifficulty, big.NewInt(180)). Cmp(virtualSub.Sub(virtualSelectedParentHeader.BlueWork(), block.Header.BlueWork())) < 0 { - log.Criticalf("Avoiding IBD because it is coming from a deep (%d DAA score depth) "+ - "side-chain which has much lower blue work (%d, %d)", + log.Criticalf("Avoiding IBD triggered by relay %s because it is coming from " + + "a deep (%d DAA score depth) side-chain which has much lower blue work (%d, %d)", + highHash, virtualSelectedParentHeader.DAAScore()-block.Header.DAAScore(), virtualSelectedParentHeader.BlueWork(), block.Header.BlueWork()) return nil @@ -98,7 +101,6 @@ func (flow *handleIBDFlow) runIBDIfNotRunning(block *externalapi.DomainBlock) er flow.logIBDFinished(isFinishedSuccessfully) }() - highHash := consensushashing.BlockHash(block) log.Criticalf("IBD started with peer %s and highHash %s", flow.peer, highHash) log.Criticalf("Syncing blocks up to %s", highHash) log.Criticalf("Trying to find highest shared chain block with peer %s with high hash %s", flow.peer, highHash)