Bd21 account sequence nonce sync

cmd/circle/attestation.go

@@ -38,6 +38,7 @@ func CheckAttestation(cfg config.Config, logger log.Logger, irisLookupId string)
 		logger.Debug("unable to unmarshal response")
 		return nil
 	}
+	logger.Info(fmt.Sprintf("Attestation found for %s%s%s", cfg.Circle.AttestationBaseUrl, "0x", irisLookupId))
 
 	return &response
 }

cmd/ethereum/broadcast.go

@@ -40,7 +40,7 @@ func Broadcast(
 	}
 
 	for attempt := 0; attempt <= cfg.Networks.Destination.Ethereum.BroadcastRetries; attempt++ {
-		logger.Debug(fmt.Sprintf(
+		logger.Info(fmt.Sprintf(
 			"Broadcasting %s message from %d to %d: with source tx hash %s",
 			msg.Type,
 			msg.SourceDomain,

cmd/ethereum/listener.go

@@ -72,6 +72,10 @@ func StartListener(cfg config.Config, logger log.Logger, processingQueue chan *t
 		logger.Info(fmt.Sprintf("New historical msg from source domain %d with tx hash %s", parsedMsg.SourceDomain, parsedMsg.SourceTxHash))
 
 		processingQueue <- parsedMsg
+
+		// It might help to wait a small amount of time between sending messages into the processing queue
+		// so that account sequences / nonces are set correctly
+		// time.Sleep(10 * time.Millisecond)
 	}
 
 	// consume stream
@@ -90,6 +94,10 @@ func StartListener(cfg config.Config, logger log.Logger, processingQueue chan *t
 				logger.Info(fmt.Sprintf("New stream msg from %d with tx hash %s", parsedMsg.SourceDomain, parsedMsg.SourceTxHash))
 
 				processingQueue <- parsedMsg
+
+				// It might help to wait a small amount of time between sending messages into the processing queue
+				// so that account sequences / nonces are set correctly
+				// time.Sleep(10 * time.Millisecond)
 			}
 		}
 	}()

cmd/ethereum/util_test.go

@@ -19,7 +19,7 @@ func init() {
 }
 
 func TestGetEthereumAccountNonce(t *testing.T) {
-	_, err := ethereum.GetEthereumAccountNonce(cfg.Networks.Destination.Ethereum.RPC, cfg.Networks.Minters[0].MinterAddress)
+	_, err := ethereum.GetEthereumAccountNonce(cfg.Networks.Destination.Ethereum.RPC, "0x4996f29b254c77972fff8f25e6f7797b3c9a0eb6")
 	require.Nil(t, err)
 }
 

cmd/noble/broadcast.go

@@ -8,6 +8,7 @@ import (
 	"fmt"
 	"io"
 	"net/http"
+	"regexp"
 	"strconv"
 	"time"
 
@@ -78,16 +79,16 @@ func Broadcast(
 	}
 
 	for attempt := 0; attempt <= cfg.Networks.Destination.Noble.BroadcastRetries; attempt++ {
-		logger.Debug(fmt.Sprintf(
+		logger.Info(fmt.Sprintf(
 			"Broadcasting %s message from %d to %d: with source tx hash %s",
 			msg.Type,
 			msg.SourceDomain,
 			msg.DestDomain,
 			msg.SourceTxHash))
 
-		// TODO Account sequence lock is implemented but gets out of sync with remote.
-		// accountSequence := sequenceMap.Next(cfg.Networks.Destination.Noble.DomainId)
-		accountNumber, accountSequence, err := GetNobleAccountNumberSequence(cfg.Networks.Destination.Noble.API, nobleAddress)
+		accountSequence := sequenceMap.Next(cfg.Networks.Destination.Noble.DomainId)
+		accountNumber, _, err := GetNobleAccountNumberSequence(cfg.Networks.Destination.Noble.API, nobleAddress)
+
 		if err != nil {
 			logger.Error("unable to retrieve account number")
 		}
@@ -134,14 +135,40 @@ func Broadcast(
 			msg.Status = types.Complete
 			return rpcResponse, nil
 		}
+
+		// check tx response code
+		logger.Error(fmt.Sprintf("received non zero : %d - %s", rpcResponse.Code, rpcResponse.Log))
+
+		if err == nil && rpcResponse.Code == 32 {
+			// on account sequence mismatch, extract correct account sequence and retry
+			pattern := `expected (\d+), got (\d+)`
+			re := regexp.MustCompile(pattern)
+			match := re.FindStringSubmatch(rpcResponse.Log)
+
+			var newAccountSequence int64
+			if len(match) == 3 {
+				// Extract the numbers from the match.
+				newAccountSequence, _ = strconv.ParseInt(match[1], 10, 0)
+			} else {
+				// otherwise, just request the account sequence
+				_, newAccountSequence, err = GetNobleAccountNumberSequence(cfg.Networks.Destination.Noble.API, nobleAddress)
+				if err != nil {
+					logger.Error("unable to retrieve account number")
+				}
+			}
+
+			logger.Debug(fmt.Sprintf("error during broadcast: %s", rpcResponse.Log))
+			logger.Debug(fmt.Sprintf("retrying with new account sequence: %d", newAccountSequence))
+			sequenceMap.Put(4, newAccountSequence)
+
+		}
 		if err != nil {
 			logger.Error(fmt.Sprintf("error during broadcast: %s", err.Error()))
 			logger.Info(fmt.Sprintf("Retrying in %d seconds", cfg.Networks.Destination.Noble.BroadcastRetryInterval))
 			time.Sleep(time.Duration(cfg.Networks.Destination.Noble.BroadcastRetryInterval) * time.Second)
 			continue
 		}
-		// check tx response code
-		logger.Error(fmt.Sprintf("received non zero : %d - %s", rpcResponse.Code, rpcResponse.Log))
+
 		logger.Info(fmt.Sprintf("Retrying in %d seconds", cfg.Networks.Destination.Noble.BroadcastRetryInterval))
 		time.Sleep(time.Duration(cfg.Networks.Destination.Noble.BroadcastRetryInterval) * time.Second)
 	}

cmd/noble/listener_test.go

@@ -1,8 +1,9 @@
-package noble
+package noble_test
 
 import (
 	"cosmossdk.io/log"
 	"github.com/rs/zerolog"
+	"github.com/strangelove-ventures/noble-cctp-relayer/cmd/noble"
 	"github.com/strangelove-ventures/noble-cctp-relayer/config"
 	"github.com/strangelove-ventures/noble-cctp-relayer/types"
 	"github.com/stretchr/testify/require"
@@ -26,7 +27,7 @@ func init() {
 func TestStartListener(t *testing.T) {
 	cfg.Networks.Source.Noble.StartBlock = 3273557
 	cfg.Networks.Source.Noble.LookbackPeriod = 0
-	go StartListener(cfg, logger, processingQueue)
+	go noble.StartListener(cfg, logger, processingQueue)
 
 	time.Sleep(20 * time.Second)
 

cmd/process.go

@@ -26,7 +26,7 @@ var startCmd = &cobra.Command{
 // Store represents terminal states
 var State = types.NewStateMap()
 
-// SequenceMap maps the domain -> the equivalent minter address sequence/nonce
+// SequenceMap maps the domain -> the equivalent minter account sequence or nonce
 var sequenceMap = types.NewSequenceMap()
 
 func Start(cmd *cobra.Command, args []string) {
@@ -184,6 +184,8 @@ func filterInvalidDestinationCallers(cfg config.Config, logger log.Logger, msg *
 	if err != nil {
 		result = true
 	}
+
+	//transformedDestinationCaller :=
 	if !bytes.Equal(msg.DestinationCaller, zeroByteArr) &&
 		bech32DestinationCaller != cfg.Networks.Minters[msg.DestDomain].MinterAddress {
 		result = true

integration/config.go

@@ -2,6 +2,7 @@ package integration_testing
 
 import (
 	"cosmossdk.io/log"
+	"github.com/rs/zerolog"
 	"github.com/strangelove-ventures/noble-cctp-relayer/cmd/noble"
 	"github.com/strangelove-ventures/noble-cctp-relayer/config"
 	"github.com/strangelove-ventures/noble-cctp-relayer/types"
@@ -25,7 +26,7 @@ func setupTest() func() {
 	// setup
 	testCfg = Parse("../.ignore/integration.yaml")
 	cfg = config.Parse("../.ignore/testnet.yaml")
-	logger = log.NewLogger(os.Stdout)
+	logger = log.NewLogger(os.Stdout, log.LevelOption(zerolog.DebugLevel))
 
 	_, nextMinterSequence, err := noble.GetNobleAccountNumberSequence(
 		cfg.Networks.Destination.Noble.API,

integration/noble_multi_send_test.go

@@ -0,0 +1,138 @@
+package integration_testing
+
+import (
+	"encoding/hex"
+	"fmt"
+	"github.com/cosmos/cosmos-sdk/crypto/keys/secp256k1"
+	"github.com/cosmos/cosmos-sdk/testutil/testdata"
+	"github.com/cosmos/cosmos-sdk/types/bech32"
+	"github.com/ethereum/go-ethereum/accounts/abi/bind"
+	"github.com/ethereum/go-ethereum/common"
+	"github.com/ethereum/go-ethereum/crypto"
+	"github.com/ethereum/go-ethereum/ethclient"
+	"github.com/strangelove-ventures/noble-cctp-relayer/cmd"
+	eth "github.com/strangelove-ventures/noble-cctp-relayer/cmd/ethereum"
+	"github.com/strangelove-ventures/noble-cctp-relayer/cmd/noble"
+	"github.com/strangelove-ventures/noble-cctp-relayer/types"
+	"github.com/stretchr/testify/require"
+	"math/big"
+	"testing"
+	"time"
+)
+
+// TestNobleMultiSend broadcasts N depositForBurnWithCaller messages on Ethereum, and then tries to receive them all at once on Noble.
+// We require a destination caller in this test so the deployed relayer doesn't pick it up.
+//
+// The point of this test is to verify that the Noble minter's account sequence is synced.
+// A successful test means that all messages went through without retries (which are set to zero).
+// We verify this result by checking the account balance at the end of the test.
+func TestNobleMultiSend(t *testing.T) {
+	setupTest()
+
+	nobleMultiSendCfg := Parse("../.ignore/noble_multi_send.yaml")
+
+	// the caller account functions both as the destination caller and minter
+	var callerPrivKey = nobleMultiSendCfg.Networks.Noble.PrivateKey
+	keyBz, err := hex.DecodeString(callerPrivKey)
+	require.Nil(t, err)
+	privKey := secp256k1.PrivKey{Key: keyBz}
+	caller, err := bech32.ConvertAndEncode("noble", privKey.PubKey().Address())
+	require.Nil(t, err)
+
+	for i, minter := range cfg.Networks.Minters {
+		switch i {
+		case 4:
+			minter.MinterAddress = caller
+			minter.MinterPrivateKey = callerPrivKey
+			cfg.Networks.Minters[4] = minter
+		}
+	}
+
+	_, nextMinterSequence, err := noble.GetNobleAccountNumberSequence(
+		cfg.Networks.Destination.Noble.API,
+		cfg.Networks.Minters[4].MinterAddress)
+
+	require.Nil(t, err)
+
+	sequenceMap = types.NewSequenceMap()
+	sequenceMap.Put(uint32(4), nextMinterSequence)
+
+	// number of depositForBurn txns to send
+	n := 7
+
+	// start up relayer
+	cfg.Networks.Source.Ethereum.StartBlock = getEthereumLatestBlockHeight(t)
+	cfg.Networks.Source.Ethereum.LookbackPeriod = 5
+	cfg.Networks.Destination.Noble.BroadcastRetries = 0 // don't rely on retries to broadcast txns
+
+	fmt.Println(fmt.Sprintf("Building %d Ethereum depositForBurnWithMetadata txns...", n))
+
+	_, _, cosmosAddress := testdata.KeyTestPubAddr()
+	nobleAddress, _ := bech32.ConvertAndEncode("noble", cosmosAddress)
+	fmt.Println("Minting on Noble to https://testnet.mintscan.io/noble-testnet/account/" + nobleAddress)
+
+	// verify original noble usdc amount
+	originalNobleBalance := getNobleBalance(nobleAddress)
+
+	// deposit for burn with metadata
+	client, err := ethclient.Dial(testCfg.Networks.Ethereum.RPC)
+	require.Nil(t, err)
+	defer client.Close()
+
+	privateKey, err := crypto.HexToECDSA(testCfg.Networks.Ethereum.PrivateKey)
+	require.Nil(t, err)
+	auth, err := bind.NewKeyedTransactorWithChainID(privateKey, big.NewInt(5))
+	require.Nil(t, err)
+
+	tokenMessenger, err := cmd.NewTokenMessenger(common.HexToAddress(TokenMessengerAddress), client)
+	require.Nil(t, err)
+
+	mintRecipientPadded := append([]byte{0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0}, cosmosAddress...)
+	require.Nil(t, err)
+
+	_, callerRaw, _ := bech32.DecodeAndConvert(caller)
+	destinationCallerPadded := append([]byte{0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0}, callerRaw...)
+	require.Nil(t, err)
+
+	erc20, err := NewERC20(common.HexToAddress(UsdcAddress), client)
+	_, err = erc20.Approve(auth, common.HexToAddress(TokenMessengerWithMetadataAddress), big.NewInt(99999))
+	require.Nil(t, err)
+	var burnAmount = big.NewInt(1)
+
+	for i := 0; i < n; i++ {
+		tx, err := tokenMessenger.DepositForBurnWithCaller(
+			auth,
+			burnAmount,
+			4,
+			[32]byte(mintRecipientPadded),
+			common.HexToAddress(UsdcAddress),
+			[32]byte(destinationCallerPadded),
+		)
+		if err != nil {
+			logger.Error("Failed to update value: %v", err)
+		}
+
+		time.Sleep(1 * time.Second)
+
+		fmt.Printf("Update pending: https://goerli.etherscan.io/tx/%s\n", tx.Hash().String())
+
+	}
+
+	fmt.Println("Waiting 90 seconds for attestations...")
+	time.Sleep(90 * time.Second)
+
+	fmt.Println("Starting relayer...")
+	processingQueue := make(chan *types.MessageState, 100)
+
+	go eth.StartListener(cfg, logger, processingQueue)
+	go cmd.StartProcessor(cfg, logger, processingQueue, sequenceMap)
+
+	fmt.Println("Checking noble wallet...")
+	for i := 0; i < 250; i++ {
+		if originalNobleBalance+burnAmount.Uint64()*uint64(n) == getNobleBalance(nobleAddress) {
+			fmt.Println(fmt.Sprintf("Successfully minted %d times at https://testnet.mintscan.io/noble-testnet/account/%s", n, nobleAddress))
+			return
+		}
+		time.Sleep(1 * time.Second)
+	}
+}

types/sequence_map.go

@@ -4,7 +4,7 @@ import (
 	"sync"
 )
 
-// SequenceMap holds the account sequence to avoid account sequence mismatch errors
+// SequenceMap holds a minter account's txn count to avoid account sequence mismatch errors
 type SequenceMap struct {
 	mu sync.Mutex
 	// map destination domain -> minter account sequence