From 515b492006747f745e324b1b17fa799dbea73700 Mon Sep 17 00:00:00 2001 From: bd21 Date: Fri, 20 Oct 2023 17:08:53 -0700 Subject: [PATCH 1/4] in progress --- cmd/circle/attestation.go | 1 + cmd/ethereum/broadcast.go | 2 +- cmd/ethereum/listener.go | 9 +++ cmd/noble/broadcast.go | 2 +- cmd/noble/listener_test.go | 5 +- cmd/process.go | 2 +- integration/config.go | 3 +- integration/noble_multi_send_test.go | 103 +++++++++++++++++++++++++++ types/sequence_map.go | 2 +- 9 files changed, 122 insertions(+), 7 deletions(-) create mode 100644 integration/noble_multi_send_test.go diff --git a/cmd/circle/attestation.go b/cmd/circle/attestation.go index 544886a..7436a63 100644 --- a/cmd/circle/attestation.go +++ b/cmd/circle/attestation.go @@ -38,6 +38,7 @@ func CheckAttestation(cfg config.Config, logger log.Logger, irisLookupId string) logger.Debug("unable to unmarshal response") return nil } + logger.Info(fmt.Sprintf("Attestation found for %s%s%s", cfg.Circle.AttestationBaseUrl, "0x", irisLookupId)) return &response } diff --git a/cmd/ethereum/broadcast.go b/cmd/ethereum/broadcast.go index 419f168..98c530b 100644 --- a/cmd/ethereum/broadcast.go +++ b/cmd/ethereum/broadcast.go @@ -40,7 +40,7 @@ func Broadcast( } for attempt := 0; attempt <= cfg.Networks.Destination.Ethereum.BroadcastRetries; attempt++ { - logger.Debug(fmt.Sprintf( + logger.Info(fmt.Sprintf( "Broadcasting %s message from %d to %d: with source tx hash %s", msg.Type, msg.SourceDomain, diff --git a/cmd/ethereum/listener.go b/cmd/ethereum/listener.go index 0e7a958..3d211ec 100644 --- a/cmd/ethereum/listener.go +++ b/cmd/ethereum/listener.go @@ -15,6 +15,7 @@ import ( "github.com/strangelove-ventures/noble-cctp-relayer/types" "math/big" "os" + "time" ) //go:embed abi/MessageTransmitter.json @@ -72,6 +73,10 @@ func StartListener(cfg config.Config, logger log.Logger, processingQueue chan *t logger.Info(fmt.Sprintf("New historical msg from source domain %d with tx hash %s", parsedMsg.SourceDomain, parsedMsg.SourceTxHash)) processingQueue <- parsedMsg + + // It's important to wait a small amount of time between sending messages into the processing queue + // so that nonces are set correctly + time.Sleep(10 * time.Millisecond) } // consume stream @@ -90,6 +95,10 @@ func StartListener(cfg config.Config, logger log.Logger, processingQueue chan *t logger.Info(fmt.Sprintf("New stream msg from %d with tx hash %s", parsedMsg.SourceDomain, parsedMsg.SourceTxHash)) processingQueue <- parsedMsg + + // It's important to wait a small amount of time between sending messages into the processing queue + // so that nonces are set correctly + time.Sleep(10 * time.Millisecond) } } }() diff --git a/cmd/noble/broadcast.go b/cmd/noble/broadcast.go index e52083a..14f7f23 100644 --- a/cmd/noble/broadcast.go +++ b/cmd/noble/broadcast.go @@ -78,7 +78,7 @@ func Broadcast( } for attempt := 0; attempt <= cfg.Networks.Destination.Noble.BroadcastRetries; attempt++ { - logger.Debug(fmt.Sprintf( + logger.Info(fmt.Sprintf( "Broadcasting %s message from %d to %d: with source tx hash %s", msg.Type, msg.SourceDomain, diff --git a/cmd/noble/listener_test.go b/cmd/noble/listener_test.go index 64e3399..97f4171 100644 --- a/cmd/noble/listener_test.go +++ b/cmd/noble/listener_test.go @@ -1,8 +1,9 @@ -package noble +package noble_test import ( "cosmossdk.io/log" "github.com/rs/zerolog" + "github.com/strangelove-ventures/noble-cctp-relayer/cmd/noble" "github.com/strangelove-ventures/noble-cctp-relayer/config" "github.com/strangelove-ventures/noble-cctp-relayer/types" "github.com/stretchr/testify/require" @@ -26,7 +27,7 @@ func init() { func TestStartListener(t *testing.T) { cfg.Networks.Source.Noble.StartBlock = 3273557 cfg.Networks.Source.Noble.LookbackPeriod = 0 - go StartListener(cfg, logger, processingQueue) + go noble.StartListener(cfg, logger, processingQueue) time.Sleep(20 * time.Second) diff --git a/cmd/process.go b/cmd/process.go index 551276a..2b3eaba 100644 --- a/cmd/process.go +++ b/cmd/process.go @@ -26,7 +26,7 @@ var startCmd = &cobra.Command{ // Store represents terminal states var State = types.NewStateMap() -// SequenceMap maps the domain -> the equivalent minter address sequence/nonce +// SequenceMap maps the domain -> the equivalent minter account sequence or nonce var sequenceMap = types.NewSequenceMap() func Start(cmd *cobra.Command, args []string) { diff --git a/integration/config.go b/integration/config.go index 077d6a4..8ddec3d 100644 --- a/integration/config.go +++ b/integration/config.go @@ -2,6 +2,7 @@ package integration_testing import ( "cosmossdk.io/log" + "github.com/rs/zerolog" "github.com/strangelove-ventures/noble-cctp-relayer/cmd/noble" "github.com/strangelove-ventures/noble-cctp-relayer/config" "github.com/strangelove-ventures/noble-cctp-relayer/types" @@ -25,7 +26,7 @@ func setupTest() func() { // setup testCfg = Parse("../.ignore/integration.yaml") cfg = config.Parse("../.ignore/testnet.yaml") - logger = log.NewLogger(os.Stdout) + logger = log.NewLogger(os.Stdout, log.LevelOption(zerolog.DebugLevel)) _, nextMinterSequence, err := noble.GetNobleAccountNumberSequence( cfg.Networks.Destination.Noble.API, diff --git a/integration/noble_multi_send_test.go b/integration/noble_multi_send_test.go new file mode 100644 index 0000000..23862a0 --- /dev/null +++ b/integration/noble_multi_send_test.go @@ -0,0 +1,103 @@ +package integration_testing + +import ( + "fmt" + "github.com/cosmos/cosmos-sdk/testutil/testdata" + "github.com/cosmos/cosmos-sdk/types/bech32" + "github.com/ethereum/go-ethereum/accounts/abi/bind" + "github.com/ethereum/go-ethereum/common" + "github.com/ethereum/go-ethereum/crypto" + "github.com/ethereum/go-ethereum/ethclient" + "github.com/strangelove-ventures/noble-cctp-relayer/cmd" + eth "github.com/strangelove-ventures/noble-cctp-relayer/cmd/ethereum" + "github.com/strangelove-ventures/noble-cctp-relayer/types" + "github.com/stretchr/testify/require" + "math/big" + "testing" + "time" +) + +// TestNobleMultiSend broadcasts N messages on Ethereum, and then tries to receive them all at once on Noble. +// The point of this test is to verify that account sequences on Noble are synced correctly. +// A success means that all messages went through (i.e. no retries are attempted). +func TestNobleMultiSend(t *testing.T) { + setupTest() + + // number of depositForBurn txns to send + n := 5 + + // start up relayer + cfg.Networks.Source.Ethereum.StartBlock = getEthereumLatestBlockHeight(t) + cfg.Networks.Source.Ethereum.LookbackPeriod = 20 + cfg.Networks.Destination.Noble.BroadcastRetries = 0 + + fmt.Println(fmt.Sprintf("Building %d Ethereum depositForBurnWithMetadata txn..."), n) + + _, _, cosmosAddress := testdata.KeyTestPubAddr() + nobleAddress, _ := bech32.ConvertAndEncode("noble", cosmosAddress) + fmt.Println("Minting on Noble to https://testnet.mintscan.io/noble-testnet/account/" + nobleAddress) + + // verify original noble usdc amount + originalNobleBalance := getNobleBalance(nobleAddress) + + // deposit for burn with metadata + client, err := ethclient.Dial(testCfg.Networks.Ethereum.RPC) + require.Nil(t, err) + defer client.Close() + + privateKey, err := crypto.HexToECDSA(testCfg.Networks.Ethereum.PrivateKey) + require.Nil(t, err) + auth, err := bind.NewKeyedTransactorWithChainID(privateKey, big.NewInt(5)) + require.Nil(t, err) + + tokenMessenger, err := cmd.NewTokenMessenger(common.HexToAddress(TokenMessengerAddress), client) + require.Nil(t, err) + + mintRecipientPadded := append([]byte{0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0}, cosmosAddress...) + require.Nil(t, err) + + erc20, err := NewERC20(common.HexToAddress(UsdcAddress), client) + _, err = erc20.Approve(auth, common.HexToAddress(TokenMessengerWithMetadataAddress), big.NewInt(99999)) + require.Nil(t, err) + var burnAmount = big.NewInt(1) + + for i := 0; i < n; i++ { + tx, err := tokenMessenger.DepositForBurn( + auth, + burnAmount, + 4, + [32]byte(mintRecipientPadded), + common.HexToAddress(UsdcAddress), + ) + if err != nil { + logger.Error("Failed to update value: %v", err) + } + + fmt.Printf("Update pending: https://goerli.etherscan.io/tx/%s\n", tx.Hash().String()) + + // todo sleep between burns? + } + + fmt.Println("Waiting 90 seconds for attestations...") + time.Sleep(90 * time.Second) + + fmt.Println("Starting relayer...") + processingQueue := make(chan *types.MessageState, 100) + + go eth.StartListener(cfg, logger, processingQueue) + go cmd.StartProcessor(cfg, logger, processingQueue, sequenceMap) + + fmt.Println("Checking noble wallet...") + for i := 0; i < 250; i++ { + if originalNobleBalance+burnAmount.Uint64()*uint64(n) == getNobleBalance(nobleAddress) { + fmt.Println("Successfully minted at https://testnet.mintscan.io/noble-testnet/account/" + nobleAddress) + return + } + time.Sleep(1 * time.Second) + } + // verify noble balance + require.Equal(t, originalNobleBalance+burnAmount.Uint64()*uint64(n), getNobleBalance(nobleAddress)) + + // check logs + +} diff --git a/types/sequence_map.go b/types/sequence_map.go index 33c7f94..dab63f1 100644 --- a/types/sequence_map.go +++ b/types/sequence_map.go @@ -4,7 +4,7 @@ import ( "sync" ) -// SequenceMap holds the account sequence to avoid account sequence mismatch errors +// SequenceMap holds a minter account's txn count to avoid account sequence mismatch errors type SequenceMap struct { mu sync.Mutex // map destination domain -> minter account sequence From bb7cad5d9f7b75ebd49f546de37e634ca8bc99b1 Mon Sep 17 00:00:00 2001 From: bd21 Date: Wed, 25 Oct 2023 02:45:51 -0700 Subject: [PATCH 2/4] fixed for noble minting --- cmd/ethereum/listener.go | 6 +-- cmd/ethereum/util_test.go | 2 +- cmd/noble/broadcast.go | 37 +++++++++++++--- cmd/process.go | 2 + integration/noble_multi_send_test.go | 65 +++++++++++++++++++++------- 5 files changed, 88 insertions(+), 24 deletions(-) diff --git a/cmd/ethereum/listener.go b/cmd/ethereum/listener.go index 3d211ec..db1d586 100644 --- a/cmd/ethereum/listener.go +++ b/cmd/ethereum/listener.go @@ -96,9 +96,9 @@ func StartListener(cfg config.Config, logger log.Logger, processingQueue chan *t processingQueue <- parsedMsg - // It's important to wait a small amount of time between sending messages into the processing queue - // so that nonces are set correctly - time.Sleep(10 * time.Millisecond) + // It might help to wait a small amount of time between sending messages into the processing queue + // so that account sequences / nonces are set correctly + // time.Sleep(10 * time.Millisecond) } } }() diff --git a/cmd/ethereum/util_test.go b/cmd/ethereum/util_test.go index a3b5981..b39cb2d 100644 --- a/cmd/ethereum/util_test.go +++ b/cmd/ethereum/util_test.go @@ -19,7 +19,7 @@ func init() { } func TestGetEthereumAccountNonce(t *testing.T) { - _, err := ethereum.GetEthereumAccountNonce(cfg.Networks.Destination.Ethereum.RPC, cfg.Networks.Minters[0].MinterAddress) + _, err := ethereum.GetEthereumAccountNonce(cfg.Networks.Destination.Ethereum.RPC, "0x4996f29b254c77972fff8f25e6f7797b3c9a0eb6") require.Nil(t, err) } diff --git a/cmd/noble/broadcast.go b/cmd/noble/broadcast.go index 14f7f23..f8b63b5 100644 --- a/cmd/noble/broadcast.go +++ b/cmd/noble/broadcast.go @@ -8,6 +8,7 @@ import ( "fmt" "io" "net/http" + "regexp" "strconv" "time" @@ -85,9 +86,9 @@ func Broadcast( msg.DestDomain, msg.SourceTxHash)) - // TODO Account sequence lock is implemented but gets out of sync with remote. - // accountSequence := sequenceMap.Next(cfg.Networks.Destination.Noble.DomainId) - accountNumber, accountSequence, err := GetNobleAccountNumberSequence(cfg.Networks.Destination.Noble.API, nobleAddress) + accountSequence := sequenceMap.Next(cfg.Networks.Destination.Noble.DomainId) + accountNumber, _, err := GetNobleAccountNumberSequence(cfg.Networks.Destination.Noble.API, nobleAddress) + if err != nil { logger.Error("unable to retrieve account number") } @@ -134,14 +135,40 @@ func Broadcast( msg.Status = types.Complete return rpcResponse, nil } + + // check tx response code + logger.Error(fmt.Sprintf("received non zero : %d - %s", rpcResponse.Code, rpcResponse.Log)) + + if err == nil && rpcResponse.Code == 32 { + // on account sequence mismatch, extract correct account sequence and retry + pattern := `expected (\d+), got (\d+)` + re := regexp.MustCompile(pattern) + match := re.FindStringSubmatch(rpcResponse.Log) + + var newAccountSequence int64 + if len(match) == 3 { + // Extract the numbers from the match. + newAccountSequence, _ = strconv.ParseInt(match[1], 10, 0) + } else { + // otherwise, just request the account sequence + _, newAccountSequence, err = GetNobleAccountNumberSequence(cfg.Networks.Destination.Noble.API, nobleAddress) + if err != nil { + logger.Error("unable to retrieve account number") + } + } + + logger.Debug(fmt.Sprintf("error during broadcast: %s", rpcResponse.Log)) + logger.Debug(fmt.Sprintf("retrying with new account sequence: %d", newAccountSequence)) + sequenceMap.Put(4, newAccountSequence) + + } if err != nil { logger.Error(fmt.Sprintf("error during broadcast: %s", err.Error())) logger.Info(fmt.Sprintf("Retrying in %d seconds", cfg.Networks.Destination.Noble.BroadcastRetryInterval)) time.Sleep(time.Duration(cfg.Networks.Destination.Noble.BroadcastRetryInterval) * time.Second) continue } - // check tx response code - logger.Error(fmt.Sprintf("received non zero : %d - %s", rpcResponse.Code, rpcResponse.Log)) + logger.Info(fmt.Sprintf("Retrying in %d seconds", cfg.Networks.Destination.Noble.BroadcastRetryInterval)) time.Sleep(time.Duration(cfg.Networks.Destination.Noble.BroadcastRetryInterval) * time.Second) } diff --git a/cmd/process.go b/cmd/process.go index 2b3eaba..da62c94 100644 --- a/cmd/process.go +++ b/cmd/process.go @@ -184,6 +184,8 @@ func filterInvalidDestinationCallers(cfg config.Config, logger log.Logger, msg * if err != nil { result = true } + + //transformedDestinationCaller := if !bytes.Equal(msg.DestinationCaller, zeroByteArr) && bech32DestinationCaller != cfg.Networks.Minters[msg.DestDomain].MinterAddress { result = true diff --git a/integration/noble_multi_send_test.go b/integration/noble_multi_send_test.go index 23862a0..bdb4633 100644 --- a/integration/noble_multi_send_test.go +++ b/integration/noble_multi_send_test.go @@ -1,7 +1,9 @@ package integration_testing import ( + "encoding/hex" "fmt" + "github.com/cosmos/cosmos-sdk/crypto/keys/secp256k1" "github.com/cosmos/cosmos-sdk/testutil/testdata" "github.com/cosmos/cosmos-sdk/types/bech32" "github.com/ethereum/go-ethereum/accounts/abi/bind" @@ -10,6 +12,7 @@ import ( "github.com/ethereum/go-ethereum/ethclient" "github.com/strangelove-ventures/noble-cctp-relayer/cmd" eth "github.com/strangelove-ventures/noble-cctp-relayer/cmd/ethereum" + "github.com/strangelove-ventures/noble-cctp-relayer/cmd/noble" "github.com/strangelove-ventures/noble-cctp-relayer/types" "github.com/stretchr/testify/require" "math/big" @@ -17,21 +20,52 @@ import ( "time" ) -// TestNobleMultiSend broadcasts N messages on Ethereum, and then tries to receive them all at once on Noble. -// The point of this test is to verify that account sequences on Noble are synced correctly. -// A success means that all messages went through (i.e. no retries are attempted). +// TestNobleMultiSend broadcasts N depositForBurnWithCaller messages on Ethereum, and then tries to receive them all at once on Noble. +// We require a destination caller in this test so the deployed relayer doesn't pick it up. +// +// The point of this test is to verify that the Noble minter's account sequence is synced. +// A successful test means that all messages went through without retries (which are set to zero). +// We verify this result by checking the account balance at the end of the test. func TestNobleMultiSend(t *testing.T) { setupTest() + nobleMultiSendCfg := Parse("../.ignore/noble_multi_send.yaml") + + // the caller account functions both as the destination caller and minter + var callerPrivKey = nobleMultiSendCfg.Networks.Noble.PrivateKey + keyBz, err := hex.DecodeString(callerPrivKey) + require.Nil(t, err) + privKey := secp256k1.PrivKey{Key: keyBz} + caller, err := bech32.ConvertAndEncode("noble", privKey.PubKey().Address()) + require.Nil(t, err) + + for i, minter := range cfg.Networks.Minters { + switch i { + case 4: + minter.MinterAddress = caller + minter.MinterPrivateKey = callerPrivKey + cfg.Networks.Minters[4] = minter + } + } + + _, nextMinterSequence, err := noble.GetNobleAccountNumberSequence( + cfg.Networks.Destination.Noble.API, + cfg.Networks.Minters[4].MinterAddress) + + require.Nil(t, err) + + sequenceMap = types.NewSequenceMap() + sequenceMap.Put(uint32(4), nextMinterSequence) + // number of depositForBurn txns to send - n := 5 + n := 7 // start up relayer cfg.Networks.Source.Ethereum.StartBlock = getEthereumLatestBlockHeight(t) - cfg.Networks.Source.Ethereum.LookbackPeriod = 20 - cfg.Networks.Destination.Noble.BroadcastRetries = 0 + cfg.Networks.Source.Ethereum.LookbackPeriod = 5 + cfg.Networks.Destination.Noble.BroadcastRetries = 0 // don't rely on retries to broadcast txns - fmt.Println(fmt.Sprintf("Building %d Ethereum depositForBurnWithMetadata txn..."), n) + fmt.Println(fmt.Sprintf("Building %d Ethereum depositForBurnWithMetadata txns...", n)) _, _, cosmosAddress := testdata.KeyTestPubAddr() nobleAddress, _ := bech32.ConvertAndEncode("noble", cosmosAddress) @@ -56,26 +90,32 @@ func TestNobleMultiSend(t *testing.T) { mintRecipientPadded := append([]byte{0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0}, cosmosAddress...) require.Nil(t, err) + _, callerRaw, _ := bech32.DecodeAndConvert(caller) + destinationCallerPadded := append([]byte{0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0}, callerRaw...) + require.Nil(t, err) + erc20, err := NewERC20(common.HexToAddress(UsdcAddress), client) _, err = erc20.Approve(auth, common.HexToAddress(TokenMessengerWithMetadataAddress), big.NewInt(99999)) require.Nil(t, err) var burnAmount = big.NewInt(1) for i := 0; i < n; i++ { - tx, err := tokenMessenger.DepositForBurn( + tx, err := tokenMessenger.DepositForBurnWithCaller( auth, burnAmount, 4, [32]byte(mintRecipientPadded), common.HexToAddress(UsdcAddress), + [32]byte(destinationCallerPadded), ) if err != nil { logger.Error("Failed to update value: %v", err) } + time.Sleep(1 * time.Second) + fmt.Printf("Update pending: https://goerli.etherscan.io/tx/%s\n", tx.Hash().String()) - // todo sleep between burns? } fmt.Println("Waiting 90 seconds for attestations...") @@ -90,14 +130,9 @@ func TestNobleMultiSend(t *testing.T) { fmt.Println("Checking noble wallet...") for i := 0; i < 250; i++ { if originalNobleBalance+burnAmount.Uint64()*uint64(n) == getNobleBalance(nobleAddress) { - fmt.Println("Successfully minted at https://testnet.mintscan.io/noble-testnet/account/" + nobleAddress) + fmt.Println(fmt.Sprintf("Successfully minted %d times at https://testnet.mintscan.io/noble-testnet/account/%s", n, nobleAddress)) return } time.Sleep(1 * time.Second) } - // verify noble balance - require.Equal(t, originalNobleBalance+burnAmount.Uint64()*uint64(n), getNobleBalance(nobleAddress)) - - // check logs - } From da1fb9e91f754948ef8afe008de037b37adf4cf5 Mon Sep 17 00:00:00 2001 From: bd21 Date: Wed, 25 Oct 2023 02:50:06 -0700 Subject: [PATCH 3/4] delay wasn't needed after all --- cmd/ethereum/listener.go | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/cmd/ethereum/listener.go b/cmd/ethereum/listener.go index db1d586..1771a5f 100644 --- a/cmd/ethereum/listener.go +++ b/cmd/ethereum/listener.go @@ -15,7 +15,6 @@ import ( "github.com/strangelove-ventures/noble-cctp-relayer/types" "math/big" "os" - "time" ) //go:embed abi/MessageTransmitter.json @@ -74,9 +73,9 @@ func StartListener(cfg config.Config, logger log.Logger, processingQueue chan *t processingQueue <- parsedMsg - // It's important to wait a small amount of time between sending messages into the processing queue - // so that nonces are set correctly - time.Sleep(10 * time.Millisecond) + // It might help to wait a small amount of time between sending messages into the processing queue + // so that account sequences / nonces are set correctly + // time.Sleep(10 * time.Millisecond) } // consume stream From 6f5bfbb5fdbe874c80163bb027af1a163041aa42 Mon Sep 17 00:00:00 2001 From: bd21 Date: Wed, 25 Oct 2023 12:24:26 -0700 Subject: [PATCH 4/4] fixed core issue + bug where dest callers for evm destinations weren't being decoded correctly --- cmd/ethereum/broadcast.go | 20 +-- cmd/noble/broadcast.go | 4 +- cmd/noble/listener_test.go | 1 - cmd/process.go | 41 +++-- config/config.go | 1 - integration/eth_multi_send_test.go | 192 +++++++++++++++++++++ integration/noble_burn_to_eth_mint_test.go | 1 - integration/noble_multi_send_test.go | 3 +- 8 files changed, 232 insertions(+), 31 deletions(-) create mode 100644 integration/eth_multi_send_test.go diff --git a/cmd/ethereum/broadcast.go b/cmd/ethereum/broadcast.go index 98c530b..d2dff54 100644 --- a/cmd/ethereum/broadcast.go +++ b/cmd/ethereum/broadcast.go @@ -47,14 +47,8 @@ func Broadcast( msg.DestDomain, msg.SourceTxHash)) - // TODO Account sequence lock is implemented but gets out of sync with remote. - // accountSequence := sequenceMap.Next(cfg.Networks.Destination.Noble.DomainId) - _, err := GetEthereumAccountNonce(cfg.Networks.Destination.Ethereum.RPC, ethereumAddress) - if err != nil { - logger.Error("unable to retrieve ethereum account nonce") - continue - } - //auth.Nonce = big.NewInt(nonce) + nonce := sequenceMap.Next(cfg.Networks.Destination.Ethereum.DomainId) + auth.Nonce = big.NewInt(nonce) // broadcast txn tx, err := messageTransmitter.ReceiveMessage( @@ -66,12 +60,16 @@ func Broadcast( msg.Status = types.Complete return tx, nil } else { - logger.Error(fmt.Sprintf("error during broadcast: %s", err.Error())) if parsedErr, ok := err.(JsonError); ok { if parsedErr.ErrorCode() == 3 && parsedErr.Error() == "execution reverted: Nonce already used" { - msg.Status = types.Failed - return nil, errors.New(fmt.Sprintf("Nonce already used")) + nonce, err = GetEthereumAccountNonce(cfg.Networks.Destination.Ethereum.RPC, ethereumAddress) + if err != nil { + logger.Error("unable to retrieve account number") + } + logger.Debug(fmt.Sprintf("retrying with new nonce: %d", nonce)) + sequenceMap.Put(cfg.Networks.Destination.Ethereum.DomainId, nonce) + } } diff --git a/cmd/noble/broadcast.go b/cmd/noble/broadcast.go index f8b63b5..8625ec3 100644 --- a/cmd/noble/broadcast.go +++ b/cmd/noble/broadcast.go @@ -156,11 +156,9 @@ func Broadcast( logger.Error("unable to retrieve account number") } } - logger.Debug(fmt.Sprintf("error during broadcast: %s", rpcResponse.Log)) logger.Debug(fmt.Sprintf("retrying with new account sequence: %d", newAccountSequence)) - sequenceMap.Put(4, newAccountSequence) - + sequenceMap.Put(cfg.Networks.Destination.Noble.DomainId, newAccountSequence) } if err != nil { logger.Error(fmt.Sprintf("error during broadcast: %s", err.Error())) diff --git a/cmd/noble/listener_test.go b/cmd/noble/listener_test.go index 97f4171..f92616c 100644 --- a/cmd/noble/listener_test.go +++ b/cmd/noble/listener_test.go @@ -26,7 +26,6 @@ func init() { func TestStartListener(t *testing.T) { cfg.Networks.Source.Noble.StartBlock = 3273557 - cfg.Networks.Source.Noble.LookbackPeriod = 0 go noble.StartListener(cfg, logger, processingQueue) time.Sleep(20 * time.Second) diff --git a/cmd/process.go b/cmd/process.go index da62c94..c25cfa5 100644 --- a/cmd/process.go +++ b/cmd/process.go @@ -3,6 +3,7 @@ package cmd import ( "bytes" "cosmossdk.io/log" + "encoding/hex" "fmt" "github.com/spf13/cobra" "github.com/strangelove-ventures/noble-cctp-relayer/cmd/circle" @@ -11,6 +12,7 @@ import ( "github.com/strangelove-ventures/noble-cctp-relayer/config" "github.com/strangelove-ventures/noble-cctp-relayer/types" "os" + "strings" "sync" "time" ) @@ -178,22 +180,35 @@ func filterDisabledCCTPRoutes(cfg config.Config, logger log.Logger, msg *types.M // filterInvalidDestinationCallers returns true if the minter is not the destination caller for the specified domain func filterInvalidDestinationCallers(cfg config.Config, logger log.Logger, msg *types.MessageState) bool { zeroByteArr := make([]byte, 32) - bech32DestinationCaller, err := types.DecodeDestinationCaller(msg.DestinationCaller) - result := false - if err != nil { - result = true - } - //transformedDestinationCaller := - if !bytes.Equal(msg.DestinationCaller, zeroByteArr) && - bech32DestinationCaller != cfg.Networks.Minters[msg.DestDomain].MinterAddress { - result = true - } + switch msg.DestDomain { + case 4: + bech32DestinationCaller, err := types.DecodeDestinationCaller(msg.DestinationCaller) + if err != nil { + result = true + } + if !bytes.Equal(msg.DestinationCaller, zeroByteArr) && + bech32DestinationCaller != cfg.Networks.Minters[msg.DestDomain].MinterAddress { + result = true + } + if result { + logger.Info(fmt.Sprintf("Filtered tx %s because the destination caller %s is specified and it's not the minter %s", + msg.SourceTxHash, msg.DestinationCaller, cfg.Networks.Minters[msg.DestDomain].MinterAddress)) + } - if result { - logger.Info(fmt.Sprintf("Filtered tx %s because the destination caller %s is specified and it's not the minter %s", - msg.SourceTxHash, msg.DestinationCaller, cfg.Networks.Minters[msg.DestDomain].MinterAddress)) + default: // minting to evm + decodedMinter, err := hex.DecodeString(strings.ReplaceAll(cfg.Networks.Minters[0].MinterAddress, "0x", "")) + if err != nil { + return !bytes.Equal(msg.DestinationCaller, zeroByteArr) + } + + decodedMinterPadded := make([]byte, 32) + copy(decodedMinterPadded[12:], decodedMinter) + + if !bytes.Equal(msg.DestinationCaller, zeroByteArr) && !bytes.Equal(msg.DestinationCaller, decodedMinterPadded) { + result = true + } } return result diff --git a/config/config.go b/config/config.go index b0c29b6..e21c0aa 100644 --- a/config/config.go +++ b/config/config.go @@ -23,7 +23,6 @@ type Config struct { RPC string `yaml:"rpc"` RequestQueueSize uint32 `yaml:"request-queue-size"` StartBlock uint64 `yaml:"start-block"` - LookbackPeriod uint64 `yaml:"lookback-period"` Workers uint32 `yaml:"workers"` Enabled bool `yaml:"enabled"` } `yaml:"noble"` diff --git a/integration/eth_multi_send_test.go b/integration/eth_multi_send_test.go new file mode 100644 index 0000000..6049281 --- /dev/null +++ b/integration/eth_multi_send_test.go @@ -0,0 +1,192 @@ +package integration_testing + +import ( + "context" + "cosmossdk.io/math" + "crypto/ecdsa" + "encoding/hex" + "fmt" + nobletypes "github.com/circlefin/noble-cctp/x/cctp/types" + sdkClient "github.com/cosmos/cosmos-sdk/client" + clientTx "github.com/cosmos/cosmos-sdk/client/tx" + "github.com/cosmos/cosmos-sdk/codec" + codectypes "github.com/cosmos/cosmos-sdk/codec/types" + "github.com/cosmos/cosmos-sdk/crypto/keys/secp256k1" + "github.com/cosmos/cosmos-sdk/types/bech32" + "github.com/cosmos/cosmos-sdk/types/tx/signing" + xauthsigning "github.com/cosmos/cosmos-sdk/x/auth/signing" + xauthtx "github.com/cosmos/cosmos-sdk/x/auth/tx" + "github.com/ethereum/go-ethereum/common" + "github.com/ethereum/go-ethereum/crypto" + "github.com/ethereum/go-ethereum/ethclient" + "github.com/strangelove-ventures/noble-cctp-relayer/cmd" + eth "github.com/strangelove-ventures/noble-cctp-relayer/cmd/ethereum" + "github.com/strangelove-ventures/noble-cctp-relayer/cmd/noble" + "github.com/strangelove-ventures/noble-cctp-relayer/types" + "github.com/stretchr/testify/require" + "testing" + "time" +) + +// TestEthereumMultiSend broadcasts N depositForBurnWithCaller messages on Noble, and then tries to receive them all at once on Ethereum. +// We require a destination caller in this test so the deployed relayer doesn't pick it up. +// +// The point of this test is to verify that the Ethereum minter's account sequence is synced. +// A successful test means that all messages went through without retries (which are set to zero). +// We verify this result by checking the account balance at the end of the test. +func TestEthereumMultiSend(t *testing.T) { + setupTest() + + // number of depositForBurn txns to send + n := 10 + + ethMultiSendCfg := Parse("../.ignore/eth_multi_send.yaml") + + cfg.Networks.Source.Noble.StartBlock = getNobleLatestBlockHeight() + cfg.Networks.Destination.Ethereum.BroadcastRetries = 0 // don't rely on retries to broadcast txns + + // the caller account functions both as the destination caller and minter + callerPrivKey := ethMultiSendCfg.Networks.Ethereum.PrivateKey + privateKeyBytes := common.FromHex(callerPrivKey) + privateKey, err := crypto.ToECDSA(privateKeyBytes) + require.Nil(t, err) + pubKey := privateKey.Public() + publicKeyECDSA, ok := pubKey.(*ecdsa.PublicKey) + require.True(t, ok) + caller := crypto.PubkeyToAddress(*publicKeyECDSA).String() + + for i, minter := range cfg.Networks.Minters { + switch i { + case 0: + minter.MinterAddress = caller + minter.MinterPrivateKey = callerPrivKey + cfg.Networks.Minters[0] = minter + } + } + + nonce, err := eth.GetEthereumAccountNonce(cfg.Networks.Destination.Ethereum.RPC, caller) + require.Nil(t, err) + + sequenceMap = types.NewSequenceMap() + sequenceMap.Put(uint32(0), nonce) + + fmt.Println(fmt.Sprintf("Building %d Noble depositForBurnWithCaller txns...", n)) + + ethMintRecipient := "0x971c54a6Eb782fAccD00bc3Ed5E934Cc5bD8e3Ef" + fmt.Println("Minting on Ethereum to https://goerli.etherscan.io/address/" + ethMintRecipient) + + // verify original eth usdc amount + client, err := ethclient.Dial(testCfg.Networks.Ethereum.RPC) + require.Nil(t, err) + defer client.Close() + originalEthBalance := getEthBalance(client, ethMintRecipient) + fmt.Println(fmt.Sprintf("original usdc balance: %d", originalEthBalance)) + + // deposit for burn with caller + interfaceRegistry := codectypes.NewInterfaceRegistry() + nobletypes.RegisterInterfaces(interfaceRegistry) + cdc := codec.NewProtoCodec(interfaceRegistry) + + sdkContext := sdkClient.Context{ + TxConfig: xauthtx.NewTxConfig(cdc, xauthtx.DefaultSignModes), + } + txBuilder := sdkContext.TxConfig.NewTxBuilder() + + // get priv key + keyBz, _ := hex.DecodeString(testCfg.Networks.Noble.PrivateKey) + privKey := secp256k1.PrivKey{Key: keyBz} + nobleAddress, err := bech32.ConvertAndEncode("noble", privKey.PubKey().Address()) + require.Nil(t, err) + + mintRecipient := make([]byte, 32) + copy(mintRecipient[12:], common.FromHex(ethMintRecipient)) + + destinationCaller := make([]byte, 32) + copy(destinationCaller[12:], common.FromHex(caller[2:])) + + var burnAmount = math.NewInt(1) + + // deposit for burn on noble + accountNumber, accountSequence, err := GetNobleAccountNumberSequence(cfg.Networks.Destination.Noble.API, nobleAddress) + require.Nil(t, err) + + for i := 0; i < n; i++ { + burnMsg := nobletypes.NewMsgDepositForBurnWithCaller( + nobleAddress, + burnAmount, + uint32(0), + mintRecipient, + "uusdc", + destinationCaller, + ) + + err = txBuilder.SetMsgs(burnMsg) + require.Nil(t, err) + + txBuilder.SetGasLimit(300000) + + // sign + broadcast txn + rpcClient, err := NewRPCClient(testCfg.Networks.Noble.RPC, 10*time.Second) + require.Nil(t, err) + + sigV2 := signing.SignatureV2{ + PubKey: privKey.PubKey(), + Data: &signing.SingleSignatureData{ + SignMode: sdkContext.TxConfig.SignModeHandler().DefaultMode(), + Signature: nil, + }, + Sequence: uint64(accountSequence), + } + + signerData := xauthsigning.SignerData{ + ChainID: cfg.Networks.Destination.Noble.ChainId, + AccountNumber: uint64(accountNumber), + Sequence: uint64(accountSequence), + } + + txBuilder.SetSignatures(sigV2) + sigV2, err = clientTx.SignWithPrivKey( + sdkContext.TxConfig.SignModeHandler().DefaultMode(), + signerData, + txBuilder, + &privKey, + sdkContext.TxConfig, + uint64(accountSequence), + ) + + err = txBuilder.SetSignatures(sigV2) + require.Nil(t, err) + + // Generated Protobuf-encoded bytes. + txBytes, err := sdkContext.TxConfig.TxEncoder()(txBuilder.GetTx()) + require.Nil(t, err) + + rpcResponse, err := rpcClient.BroadcastTxSync(context.Background(), txBytes) + require.Nil(t, err) + fmt.Printf("Update pending: https://testnet.mintscan.io/noble-testnet/txs/%s\n", rpcResponse.Hash.String()) + + accountSequence++ + } + + fmt.Println("Waiting 60 seconds for attestations...") + time.Sleep(60 * time.Second) + + fmt.Println("Starting relayer...") + + processingQueue := make(chan *types.MessageState, 100) + + go noble.StartListener(cfg, logger, processingQueue) + go cmd.StartProcessor(cfg, logger, processingQueue, sequenceMap) + + fmt.Println("Checking eth wallet...") + for i := 0; i < 60; i++ { + if originalEthBalance+burnAmount.Uint64()*uint64(n) == getEthBalance(client, ethMintRecipient) { + fmt.Println(fmt.Sprintf("New eth balance: %d", getEthBalance(client, ethMintRecipient))) + fmt.Println(fmt.Sprintf("Successfully minted %d times at https://goerli.etherscan.io/address/%s", n, ethMintRecipient)) + return + } + time.Sleep(1 * time.Second) + } + + require.Equal(t, originalEthBalance+burnAmount.Uint64()*uint64(n), getEthBalance(client, ethMintRecipient)) +} diff --git a/integration/noble_burn_to_eth_mint_test.go b/integration/noble_burn_to_eth_mint_test.go index da37a26..9e96944 100644 --- a/integration/noble_burn_to_eth_mint_test.go +++ b/integration/noble_burn_to_eth_mint_test.go @@ -45,7 +45,6 @@ func TestNobleBurnToEthMint(t *testing.T) { // start up relayer cfg.Networks.Source.Noble.StartBlock = getNobleLatestBlockHeight() - cfg.Networks.Source.Noble.LookbackPeriod = 0 fmt.Println("Starting relayer...") processingQueue := make(chan *types.MessageState, 10) diff --git a/integration/noble_multi_send_test.go b/integration/noble_multi_send_test.go index bdb4633..0b1bbb6 100644 --- a/integration/noble_multi_send_test.go +++ b/integration/noble_multi_send_test.go @@ -65,7 +65,7 @@ func TestNobleMultiSend(t *testing.T) { cfg.Networks.Source.Ethereum.LookbackPeriod = 5 cfg.Networks.Destination.Noble.BroadcastRetries = 0 // don't rely on retries to broadcast txns - fmt.Println(fmt.Sprintf("Building %d Ethereum depositForBurnWithMetadata txns...", n)) + fmt.Println(fmt.Sprintf("Building %d Ethereum depositForBurnWithCaller txns...", n)) _, _, cosmosAddress := testdata.KeyTestPubAddr() nobleAddress, _ := bech32.ConvertAndEncode("noble", cosmosAddress) @@ -135,4 +135,5 @@ func TestNobleMultiSend(t *testing.T) { } time.Sleep(1 * time.Second) } + require.Equal(t, originalNobleBalance+burnAmount.Uint64()*uint64(n), getNobleBalance(nobleAddress)) }