added wait for another block to have more reliable tests (#344)

* added wait for another block to have more reliable tests
* Added one network reload in case the address/port was still bound during the last try
test related

* separated Dao test suites (faster output on the CI)
* added 2 seconds wait time to avoid errors of still blocked ports/bindings (network)
* removed parallel flag from test/e2e/asset
* renamed a bunch of things to please the linters
* moved the reset of the libConfing clientctx back to the validator context before waiting for some blocks.
This should prevent the

panic: Log in goroutine after TestE2EMachineTestSuite has completed: [app] PoP broadcast tx failed: node0.info: key not found

on the CI.


Signed-off-by: Jürgen Eckel <juergen@riddleandcode.com>

tests/e2e/asset/cli_test.go

@@ -2,6 +2,7 @@ package asset
 
 import (
 	"testing"
+	"time"
 
 	"github.com/planetmint/planetmint-go/testutil/network"
 
@@ -9,7 +10,7 @@ import (
 )
 
 func TestE2EAssetTestSuite(t *testing.T) {
-	t.Parallel()
+	time.Sleep(2 * time.Second)
 	cfg := network.LoaderDefaultConfig()
 	cfg.NumValidators = 3
 	suite.Run(t, NewE2ETestSuite(cfg))

tests/e2e/dao/basic/cli_test.go

@@ -0,0 +1,17 @@
+package basic
+
+import (
+	"testing"
+	"time"
+
+	"github.com/planetmint/planetmint-go/testutil/network"
+
+	"github.com/stretchr/testify/suite"
+)
+
+func TestE2EDaoTestSuite(t *testing.T) {
+	time.Sleep(2 * time.Second)
+	cfg := network.LoaderDefaultConfig()
+	cfg.NumValidators = 3
+	suite.Run(t, NewE2ETestSuite(cfg))
+}

tests/e2e/dao/basic/suite.go

@@ -1,4 +1,4 @@
-package dao
+package basic
 
 import (
 	"bufio"

tests/e2e/dao/cli_test.go

@@ -1,39 +0,0 @@
-package dao
-
-import (
-	"testing"
-
-	"github.com/planetmint/planetmint-go/testutil/network"
-
-	"github.com/stretchr/testify/suite"
-)
-
-func TestE2EDaoTestSuite(t *testing.T) {
-	cfg := network.LoaderDefaultConfig()
-	cfg.NumValidators = 3
-	suite.Run(t, NewE2ETestSuite(cfg))
-}
-
-func TestPopE2EDaoTestSuite(t *testing.T) {
-	cfg := network.LoaderDefaultConfig()
-	cfg.NumValidators = 3
-	suite.Run(t, NewPopSelectionE2ETestSuite(cfg))
-}
-
-func TestGasConsumptionE2EDaoTestSuite(t *testing.T) {
-	cfg := network.LoaderDefaultConfig()
-	cfg.NumValidators = 3
-	suite.Run(t, NewGasConsumptionE2ETestSuite(cfg))
-}
-
-func TestRestrictedMsgsE2EDaoTestSuite(t *testing.T) {
-	cfg := network.LoaderDefaultConfig()
-	cfg.NumValidators = 3
-	suite.Run(t, NewRestrictedMsgsE2ESuite(cfg))
-}
-
-func TestAssetDistributionE2EDaoTestSuite(t *testing.T) {
-	cfg := network.LoaderDefaultConfig()
-	cfg.NumValidators = 3
-	suite.Run(t, NewAssetDistributionE2ETestSuite(cfg))
-}

tests/e2e/dao/distribution/asset_distribution_suite.go

@@ -1,4 +1,4 @@
-package dao
+package distribution
 
 import (
 	"math"

tests/e2e/dao/distribution/cli_test.go

@@ -0,0 +1,17 @@
+package distribution
+
+import (
+	"testing"
+	"time"
+
+	"github.com/planetmint/planetmint-go/testutil/network"
+
+	"github.com/stretchr/testify/suite"
+)
+
+func TestAssetDistributionE2EDaoTestSuite(t *testing.T) {
+	time.Sleep(2 * time.Second)
+	cfg := network.LoaderDefaultConfig()
+	cfg.NumValidators = 3
+	suite.Run(t, NewAssetDistributionE2ETestSuite(cfg))
+}

tests/e2e/dao/gas/cli_test.go

@@ -0,0 +1,17 @@
+package gas
+
+import (
+	"testing"
+	"time"
+
+	"github.com/planetmint/planetmint-go/testutil/network"
+
+	"github.com/stretchr/testify/suite"
+)
+
+func TestConsumptionE2EDaoTestSuite(t *testing.T) {
+	time.Sleep(2 * time.Second)
+	cfg := network.LoaderDefaultConfig()
+	cfg.NumValidators = 3
+	suite.Run(t, NewConsumptionE2ETestSuite(cfg))
+}

tests/e2e/dao/gas/gas_consumption_suite.go

@@ -1,4 +1,4 @@
-package dao
+package gas
 
 import (
 	"bufio"
@@ -21,7 +21,7 @@ import (
 	"github.com/stretchr/testify/suite"
 )
 
-type GasConsumptionE2ETestSuite struct {
+type ConsumptionE2ETestSuite struct {
 	suite.Suite
 
 	cfg        network.Config
@@ -30,11 +30,11 @@ type GasConsumptionE2ETestSuite struct {
 	feeDenom   string
 }
 
-func NewGasConsumptionE2ETestSuite(cfg network.Config) *GasConsumptionE2ETestSuite {
+func NewConsumptionE2ETestSuite(cfg network.Config) *ConsumptionE2ETestSuite {
-	return &GasConsumptionE2ETestSuite{cfg: cfg}
+	return &ConsumptionE2ETestSuite{cfg: cfg}
 }
 
-func (s *GasConsumptionE2ETestSuite) createValAccount(cfg network.Config) (address sdk.AccAddress, err error) {
+func (s *ConsumptionE2ETestSuite) createValAccount(cfg network.Config) (address sdk.AccAddress, err error) {
 	buf := bufio.NewReader(os.Stdin)
 
 	kb, err := keyring.New(sdk.KeyringServiceName(), keyring.BackendTest, s.T().TempDir(), buf, cfg.Codec, cfg.KeyringOptions...)
@@ -63,7 +63,7 @@ func (s *GasConsumptionE2ETestSuite) createValAccount(cfg network.Config) (addre
 	return addr, nil
 }
 
-func (s *GasConsumptionE2ETestSuite) SetupSuite() {
+func (s *ConsumptionE2ETestSuite) SetupSuite() {
 	s.T().Log("setting up e2e dao gas consumption test suite")
 
 	s.feeDenom = sample.FeeDenom
@@ -93,12 +93,12 @@ func (s *GasConsumptionE2ETestSuite) SetupSuite() {
 	s.Require().NoError(err)
 }
 
-func (s *GasConsumptionE2ETestSuite) TearDownSuite() {
+func (s *ConsumptionE2ETestSuite) TearDownSuite() {
 	util.TerminationWaitGroup.Wait()
 	s.T().Log("tearing down e2e dao gas consumption test suites")
 }
 
-func (s *GasConsumptionE2ETestSuite) TestValidatorConsumption() {
+func (s *ConsumptionE2ETestSuite) TestValidatorConsumption() {
 	val := s.network.Validators[0]
 
 	k, err := val.ClientCtx.Keyring.Key(sample.Name)
@@ -117,7 +117,7 @@ func (s *GasConsumptionE2ETestSuite) TestValidatorConsumption() {
 	s.Require().NoError(err)
 }
 
-func (s *GasConsumptionE2ETestSuite) TestNonValidatorConsumptionOverflow() {
+func (s *ConsumptionE2ETestSuite) TestNonValidatorConsumptionOverflow() {
 	val := s.network.Validators[0]
 
 	k, err := val.ClientCtx.Keyring.Key(sample.Name)
@@ -141,7 +141,7 @@ func (s *GasConsumptionE2ETestSuite) TestNonValidatorConsumptionOverflow() {
 	assert.Contains(s.T(), err.Error(), "out of gas")
 }
 
-func (s *GasConsumptionE2ETestSuite) createMsgs(from sdk.AccAddress, to sdk.AccAddress, n int) (msgs []sdk.Msg) {
+func (s *ConsumptionE2ETestSuite) createMsgs(from sdk.AccAddress, to sdk.AccAddress, n int) (msgs []sdk.Msg) {
 	coins := sdk.NewCoins(sdk.NewInt64Coin(s.feeDenom, 10))
 	for i := 0; i < n; i++ {
 		msg := banktypes.NewMsgSend(from, to, coins)
@@ -150,7 +150,7 @@ func (s *GasConsumptionE2ETestSuite) createMsgs(from sdk.AccAddress, to sdk.AccA
 	return
 }
 
-func (s *GasConsumptionE2ETestSuite) TestNetworkBasedTxGasLimit() {
+func (s *ConsumptionE2ETestSuite) TestNetworkBasedTxGasLimit() {
 	var gasAmountAboveGlobalGasLimit uint64 = 200000000
 	libConfig := lib.GetConfig()
 	libConfig.SetTxGas(gasAmountAboveGlobalGasLimit)

tests/e2e/dao/msgs/cli_test.go

@@ -0,0 +1,17 @@
+package msgs
+
+import (
+	"testing"
+	"time"
+
+	"github.com/planetmint/planetmint-go/testutil/network"
+
+	"github.com/stretchr/testify/suite"
+)
+
+func TestRestrictedMsgsE2EDaoTestSuite(t *testing.T) {
+	time.Sleep(2 * time.Second)
+	cfg := network.LoaderDefaultConfig()
+	cfg.NumValidators = 3
+	suite.Run(t, NewRestrictedMsgsE2ESuite(cfg))
+}

tests/e2e/dao/msgs/restricted_msgs_suite.go

@@ -1,4 +1,4 @@
-package dao
+package msgs
 
 import (
 	sdk "github.com/cosmos/cosmos-sdk/types"

tests/e2e/dao/pop/cli_test.go

@@ -0,0 +1,17 @@
+package pop
+
+import (
+	"testing"
+	"time"
+
+	"github.com/planetmint/planetmint-go/testutil/network"
+
+	"github.com/stretchr/testify/suite"
+)
+
+func TestPopE2EDaoTestSuite(t *testing.T) {
+	time.Sleep(2 * time.Second)
+	cfg := network.LoaderDefaultConfig()
+	cfg.NumValidators = 3
+	suite.Run(t, NewSelectionE2ETestSuite(cfg))
+}

tests/e2e/dao/pop/selection_suite.go

@@ -1,4 +1,4 @@
-package dao
+package pop
 
 import (
 	"bufio"
@@ -42,7 +42,7 @@ var machines = []struct {
 	},
 }
 
-type PopSelectionE2ETestSuite struct {
+type SelectionE2ETestSuite struct {
 	suite.Suite
 
 	cfg                network.Config
@@ -55,13 +55,13 @@ type PopSelectionE2ETestSuite struct {
 	errormsg           string
 }
 
-func NewPopSelectionE2ETestSuite(cfg network.Config) *PopSelectionE2ETestSuite {
+func NewSelectionE2ETestSuite(cfg network.Config) *SelectionE2ETestSuite {
-	testsuite := &PopSelectionE2ETestSuite{cfg: cfg}
+	testsuite := &SelectionE2ETestSuite{cfg: cfg}
 	testsuite.errormsg = "--%s=%s"
 	return testsuite
 }
 
-func (s *PopSelectionE2ETestSuite) SetupSuite() {
+func (s *SelectionE2ETestSuite) SetupSuite() {
 	s.T().Log("setting up e2e dao pop selection test suite")
 
 	s.popEpochs = 10
@@ -86,12 +86,12 @@ func (s *PopSelectionE2ETestSuite) SetupSuite() {
 }
 
 // TearDownSuite clean up after testing
-func (s *PopSelectionE2ETestSuite) TearDownSuite() {
+func (s *SelectionE2ETestSuite) TearDownSuite() {
 	util.TerminationWaitGroup.Wait()
 	s.T().Log("tearing down e2e dao pop selection test suite")
 }
 
-func (s *PopSelectionE2ETestSuite) perpareLocalTest() testutil.BufferWriter {
+func (s *SelectionE2ETestSuite) perpareLocalTest() testutil.BufferWriter {
 	val := s.network.Validators[0]
 
 	latestHeight, err := s.network.LatestHeight()
@@ -123,7 +123,7 @@ type yamlChallenge struct {
 	Finished   bool   `yaml:"finished"`
 }
 
-func (s *PopSelectionE2ETestSuite) sendPoPResult(storedChallenge []byte, success bool) {
+func (s *SelectionE2ETestSuite) sendPoPResult(storedChallenge []byte, success bool) {
 	val := s.network.Validators[0]
 	var wrapper struct {
 		Challenge yamlChallenge `yaml:"challenge"`
@@ -156,14 +156,14 @@ func (s *PopSelectionE2ETestSuite) sendPoPResult(storedChallenge []byte, success
 	s.Require().NoError(err)
 }
 
-func (s *PopSelectionE2ETestSuite) TestPopSelectionNoActors() {
+func (s *SelectionE2ETestSuite) TestPopSelectionNoActors() {
 	out := s.perpareLocalTest()
 
 	assert.NotContains(s.T(), out.String(), machines[0].address)
 	assert.NotContains(s.T(), out.String(), machines[1].address)
 }
 
-func (s *PopSelectionE2ETestSuite) TestPopSelectionOneActors() {
+func (s *SelectionE2ETestSuite) TestPopSelectionOneActors() {
 	err := e2etestutil.AttestMachine(s.network, machines[0].name, machines[0].mnemonic, 0, s.feeDenom)
 	s.Require().NoError(err)
 
@@ -173,7 +173,7 @@ func (s *PopSelectionE2ETestSuite) TestPopSelectionOneActors() {
 	assert.NotContains(s.T(), out.String(), machines[1].address)
 }
 
-func (s *PopSelectionE2ETestSuite) TestPopSelectionTwoActors() {
+func (s *SelectionE2ETestSuite) TestPopSelectionTwoActors() {
 	err := e2etestutil.AttestMachine(s.network, machines[1].name, machines[1].mnemonic, 1, s.feeDenom)
 	s.Require().NoError(err)
 
@@ -184,7 +184,7 @@ func (s *PopSelectionE2ETestSuite) TestPopSelectionTwoActors() {
 	s.sendPoPResult(out.Bytes(), true)
 }
 
-func (s *PopSelectionE2ETestSuite) VerifyTokens(token string) {
+func (s *SelectionE2ETestSuite) VerifyTokens(token string) {
 	val := s.network.Validators[0]
 	// check balance for crddl
 	out, err := clitestutil.ExecTestCLICmd(val.ClientCtx, bank.GetCmdQueryTotalSupply(), []string{
@@ -211,7 +211,7 @@ func (s *PopSelectionE2ETestSuite) VerifyTokens(token string) {
 	assert.Equal(s.T(), "amount: \"11986301368\"\ndenom: "+token+"\n", out.String()) // 2 * 5993150684 = 11986301368
 }
 
-func (s *PopSelectionE2ETestSuite) TestTokenDistribution1() {
+func (s *SelectionE2ETestSuite) TestTokenDistribution1() {
 	out := s.perpareLocalTest()
 
 	assert.Contains(s.T(), out.String(), machines[0].address)
@@ -252,7 +252,7 @@ func (s *PopSelectionE2ETestSuite) TestTokenDistribution1() {
 	s.VerifyTokens(daoGenState.Params.ClaimDenom)
 }
 
-func (s *PopSelectionE2ETestSuite) TestTokenRedeemClaim() {
+func (s *SelectionE2ETestSuite) TestTokenRedeemClaim() {
 	val := s.network.Validators[0]
 
 	k, err := val.ClientCtx.Keyring.Key(machines[0].name)
@@ -297,7 +297,8 @@ func (s *PopSelectionE2ETestSuite) TestTokenRedeemClaim() {
 	s.Require().NoError(err)
 	s.Require().Equal(int(0), int(txResponse.Code))
 
-	// WaitForBlock before query
+	// WaitForBlock before query (2 blocks since 3 validators)
+	s.Require().NoError(s.network.WaitForNextBlock())
 	s.Require().NoError(s.network.WaitForNextBlock())
 
 	// QueryRedeemClaim
@@ -310,7 +311,7 @@ func (s *PopSelectionE2ETestSuite) TestTokenRedeemClaim() {
 	assert.Equal(s.T(), "redeemClaim:\n  amount: \"10000\"\n  beneficiary: liquidAddress\n  confirmed: true\n  creator: plmnt1kp93kns6hs2066d8qw0uz84fw3vlthewt2ck6p\n  id: \"0\"\n  liquidTxHash: \"0000000000000000000000000000000000000000000000000000000000000000\"\n", qOut.String())
 }
 
-func (s *PopSelectionE2ETestSuite) createValAccount(cfg network.Config) (address sdk.AccAddress, err error) {
+func (s *SelectionE2ETestSuite) createValAccount(cfg network.Config) (address sdk.AccAddress, err error) {
 	buf := bufio.NewReader(os.Stdin)
 
 	kb, err := keyring.New(sdk.KeyringServiceName(), keyring.BackendTest, s.T().TempDir(), buf, cfg.Codec, cfg.KeyringOptions...)

tests/e2e/machine/cli_test.go

@@ -2,6 +2,7 @@ package machine
 
 import (
 	"testing"
+	"time"
 
 	"github.com/planetmint/planetmint-go/testutil/network"
 
@@ -9,6 +10,7 @@ import (
 )
 
 func TestE2EMachineTestSuite(t *testing.T) {
+	time.Sleep(2 * time.Second)
 	cfg := network.LoaderDefaultConfig()
 	cfg.NumValidators = 3
 	suite.Run(t, NewE2ETestSuite(cfg))

tests/e2e/machine/suite.go

@@ -178,6 +178,10 @@ func (s *E2ETestSuite) TestMachineAllowanceAttestation() {
 
 	msg3 := machinetypes.NewMsgAttestMachine(addr.String(), &machine)
 	_, err = e2etestutil.BuildSignBroadcastTx(s.T(), addr, msg3)
+
+	// reset clientCtx to validator ctx
+	libConfig.SetClientCtx(val.ClientCtx)
+
 	s.Require().NoError(err)
 
 	// give machine attestation some time to issue the liquid asset
@@ -186,9 +190,6 @@ func (s *E2ETestSuite) TestMachineAllowanceAttestation() {
 	s.Require().NoError(s.network.WaitForNextBlock())
 	s.Require().NoError(s.network.WaitForNextBlock())
 
-	// reset clientCtx to validator ctx
-	libConfig.SetClientCtx(val.ClientCtx)
-
 	args := []string{
 		pubKey,
 	}

testutil/network/loader.go

@@ -1,6 +1,7 @@
 package network
 
 import (
+	"strings"
 	"testing"
 	"time"
 
@@ -45,9 +46,12 @@ func Load(t *testing.T, configs ...Config) *Network {
 	appLogger := util.GetAppLogger()
 	appLogger.SetTestingLogger(t)
 
-	// set the proper root dir for the test environment so that the abci.go logic works
-
 	net, err := New(t, validatorTmpDir, cfg)
+	// this is only done to support multi validator test
+	// race conditions(load/unload) on the CI
+	if err != nil && strings.Contains(err.Error(), "bind: address already in use") {
+		net, err = New(t, validatorTmpDir, cfg)
+	}
 	require.NoError(t, err)
 
 	_, err = net.WaitForHeight(1)