diff --git a/cmd/circle/attestation.go b/cmd/circle/attestation.go index 544886a..7436a63 100644 --- a/cmd/circle/attestation.go +++ b/cmd/circle/attestation.go @@ -38,6 +38,7 @@ func CheckAttestation(cfg config.Config, logger log.Logger, irisLookupId string) logger.Debug("unable to unmarshal response") return nil } + logger.Info(fmt.Sprintf("Attestation found for %s%s%s", cfg.Circle.AttestationBaseUrl, "0x", irisLookupId)) return &response } diff --git a/cmd/ethereum/broadcast.go b/cmd/ethereum/broadcast.go index 419f168..98c530b 100644 --- a/cmd/ethereum/broadcast.go +++ b/cmd/ethereum/broadcast.go @@ -40,7 +40,7 @@ func Broadcast( } for attempt := 0; attempt <= cfg.Networks.Destination.Ethereum.BroadcastRetries; attempt++ { - logger.Debug(fmt.Sprintf( + logger.Info(fmt.Sprintf( "Broadcasting %s message from %d to %d: with source tx hash %s", msg.Type, msg.SourceDomain, diff --git a/cmd/ethereum/listener.go b/cmd/ethereum/listener.go index 0e7a958..1771a5f 100644 --- a/cmd/ethereum/listener.go +++ b/cmd/ethereum/listener.go @@ -72,6 +72,10 @@ func StartListener(cfg config.Config, logger log.Logger, processingQueue chan *t logger.Info(fmt.Sprintf("New historical msg from source domain %d with tx hash %s", parsedMsg.SourceDomain, parsedMsg.SourceTxHash)) processingQueue <- parsedMsg + + // It might help to wait a small amount of time between sending messages into the processing queue + // so that account sequences / nonces are set correctly + // time.Sleep(10 * time.Millisecond) } // consume stream @@ -90,6 +94,10 @@ func StartListener(cfg config.Config, logger log.Logger, processingQueue chan *t logger.Info(fmt.Sprintf("New stream msg from %d with tx hash %s", parsedMsg.SourceDomain, parsedMsg.SourceTxHash)) processingQueue <- parsedMsg + + // It might help to wait a small amount of time between sending messages into the processing queue + // so that account sequences / nonces are set correctly + // time.Sleep(10 * time.Millisecond) } } }() diff --git a/cmd/ethereum/util_test.go b/cmd/ethereum/util_test.go index a3b5981..b39cb2d 100644 --- a/cmd/ethereum/util_test.go +++ b/cmd/ethereum/util_test.go @@ -19,7 +19,7 @@ func init() { } func TestGetEthereumAccountNonce(t *testing.T) { - _, err := ethereum.GetEthereumAccountNonce(cfg.Networks.Destination.Ethereum.RPC, cfg.Networks.Minters[0].MinterAddress) + _, err := ethereum.GetEthereumAccountNonce(cfg.Networks.Destination.Ethereum.RPC, "0x4996f29b254c77972fff8f25e6f7797b3c9a0eb6") require.Nil(t, err) } diff --git a/cmd/noble/broadcast.go b/cmd/noble/broadcast.go index e52083a..f8b63b5 100644 --- a/cmd/noble/broadcast.go +++ b/cmd/noble/broadcast.go @@ -8,6 +8,7 @@ import ( "fmt" "io" "net/http" + "regexp" "strconv" "time" @@ -78,16 +79,16 @@ func Broadcast( } for attempt := 0; attempt <= cfg.Networks.Destination.Noble.BroadcastRetries; attempt++ { - logger.Debug(fmt.Sprintf( + logger.Info(fmt.Sprintf( "Broadcasting %s message from %d to %d: with source tx hash %s", msg.Type, msg.SourceDomain, msg.DestDomain, msg.SourceTxHash)) - // TODO Account sequence lock is implemented but gets out of sync with remote. - // accountSequence := sequenceMap.Next(cfg.Networks.Destination.Noble.DomainId) - accountNumber, accountSequence, err := GetNobleAccountNumberSequence(cfg.Networks.Destination.Noble.API, nobleAddress) + accountSequence := sequenceMap.Next(cfg.Networks.Destination.Noble.DomainId) + accountNumber, _, err := GetNobleAccountNumberSequence(cfg.Networks.Destination.Noble.API, nobleAddress) + if err != nil { logger.Error("unable to retrieve account number") } @@ -134,14 +135,40 @@ func Broadcast( msg.Status = types.Complete return rpcResponse, nil } + + // check tx response code + logger.Error(fmt.Sprintf("received non zero : %d - %s", rpcResponse.Code, rpcResponse.Log)) + + if err == nil && rpcResponse.Code == 32 { + // on account sequence mismatch, extract correct account sequence and retry + pattern := `expected (\d+), got (\d+)` + re := regexp.MustCompile(pattern) + match := re.FindStringSubmatch(rpcResponse.Log) + + var newAccountSequence int64 + if len(match) == 3 { + // Extract the numbers from the match. + newAccountSequence, _ = strconv.ParseInt(match[1], 10, 0) + } else { + // otherwise, just request the account sequence + _, newAccountSequence, err = GetNobleAccountNumberSequence(cfg.Networks.Destination.Noble.API, nobleAddress) + if err != nil { + logger.Error("unable to retrieve account number") + } + } + + logger.Debug(fmt.Sprintf("error during broadcast: %s", rpcResponse.Log)) + logger.Debug(fmt.Sprintf("retrying with new account sequence: %d", newAccountSequence)) + sequenceMap.Put(4, newAccountSequence) + + } if err != nil { logger.Error(fmt.Sprintf("error during broadcast: %s", err.Error())) logger.Info(fmt.Sprintf("Retrying in %d seconds", cfg.Networks.Destination.Noble.BroadcastRetryInterval)) time.Sleep(time.Duration(cfg.Networks.Destination.Noble.BroadcastRetryInterval) * time.Second) continue } - // check tx response code - logger.Error(fmt.Sprintf("received non zero : %d - %s", rpcResponse.Code, rpcResponse.Log)) + logger.Info(fmt.Sprintf("Retrying in %d seconds", cfg.Networks.Destination.Noble.BroadcastRetryInterval)) time.Sleep(time.Duration(cfg.Networks.Destination.Noble.BroadcastRetryInterval) * time.Second) } diff --git a/cmd/noble/listener_test.go b/cmd/noble/listener_test.go index 64e3399..97f4171 100644 --- a/cmd/noble/listener_test.go +++ b/cmd/noble/listener_test.go @@ -1,8 +1,9 @@ -package noble +package noble_test import ( "cosmossdk.io/log" "github.com/rs/zerolog" + "github.com/strangelove-ventures/noble-cctp-relayer/cmd/noble" "github.com/strangelove-ventures/noble-cctp-relayer/config" "github.com/strangelove-ventures/noble-cctp-relayer/types" "github.com/stretchr/testify/require" @@ -26,7 +27,7 @@ func init() { func TestStartListener(t *testing.T) { cfg.Networks.Source.Noble.StartBlock = 3273557 cfg.Networks.Source.Noble.LookbackPeriod = 0 - go StartListener(cfg, logger, processingQueue) + go noble.StartListener(cfg, logger, processingQueue) time.Sleep(20 * time.Second) diff --git a/cmd/process.go b/cmd/process.go index 551276a..da62c94 100644 --- a/cmd/process.go +++ b/cmd/process.go @@ -26,7 +26,7 @@ var startCmd = &cobra.Command{ // Store represents terminal states var State = types.NewStateMap() -// SequenceMap maps the domain -> the equivalent minter address sequence/nonce +// SequenceMap maps the domain -> the equivalent minter account sequence or nonce var sequenceMap = types.NewSequenceMap() func Start(cmd *cobra.Command, args []string) { @@ -184,6 +184,8 @@ func filterInvalidDestinationCallers(cfg config.Config, logger log.Logger, msg * if err != nil { result = true } + + //transformedDestinationCaller := if !bytes.Equal(msg.DestinationCaller, zeroByteArr) && bech32DestinationCaller != cfg.Networks.Minters[msg.DestDomain].MinterAddress { result = true diff --git a/integration/config.go b/integration/config.go index 077d6a4..8ddec3d 100644 --- a/integration/config.go +++ b/integration/config.go @@ -2,6 +2,7 @@ package integration_testing import ( "cosmossdk.io/log" + "github.com/rs/zerolog" "github.com/strangelove-ventures/noble-cctp-relayer/cmd/noble" "github.com/strangelove-ventures/noble-cctp-relayer/config" "github.com/strangelove-ventures/noble-cctp-relayer/types" @@ -25,7 +26,7 @@ func setupTest() func() { // setup testCfg = Parse("../.ignore/integration.yaml") cfg = config.Parse("../.ignore/testnet.yaml") - logger = log.NewLogger(os.Stdout) + logger = log.NewLogger(os.Stdout, log.LevelOption(zerolog.DebugLevel)) _, nextMinterSequence, err := noble.GetNobleAccountNumberSequence( cfg.Networks.Destination.Noble.API, diff --git a/integration/noble_multi_send_test.go b/integration/noble_multi_send_test.go new file mode 100644 index 0000000..bdb4633 --- /dev/null +++ b/integration/noble_multi_send_test.go @@ -0,0 +1,138 @@ +package integration_testing + +import ( + "encoding/hex" + "fmt" + "github.com/cosmos/cosmos-sdk/crypto/keys/secp256k1" + "github.com/cosmos/cosmos-sdk/testutil/testdata" + "github.com/cosmos/cosmos-sdk/types/bech32" + "github.com/ethereum/go-ethereum/accounts/abi/bind" + "github.com/ethereum/go-ethereum/common" + "github.com/ethereum/go-ethereum/crypto" + "github.com/ethereum/go-ethereum/ethclient" + "github.com/strangelove-ventures/noble-cctp-relayer/cmd" + eth "github.com/strangelove-ventures/noble-cctp-relayer/cmd/ethereum" + "github.com/strangelove-ventures/noble-cctp-relayer/cmd/noble" + "github.com/strangelove-ventures/noble-cctp-relayer/types" + "github.com/stretchr/testify/require" + "math/big" + "testing" + "time" +) + +// TestNobleMultiSend broadcasts N depositForBurnWithCaller messages on Ethereum, and then tries to receive them all at once on Noble. +// We require a destination caller in this test so the deployed relayer doesn't pick it up. +// +// The point of this test is to verify that the Noble minter's account sequence is synced. +// A successful test means that all messages went through without retries (which are set to zero). +// We verify this result by checking the account balance at the end of the test. +func TestNobleMultiSend(t *testing.T) { + setupTest() + + nobleMultiSendCfg := Parse("../.ignore/noble_multi_send.yaml") + + // the caller account functions both as the destination caller and minter + var callerPrivKey = nobleMultiSendCfg.Networks.Noble.PrivateKey + keyBz, err := hex.DecodeString(callerPrivKey) + require.Nil(t, err) + privKey := secp256k1.PrivKey{Key: keyBz} + caller, err := bech32.ConvertAndEncode("noble", privKey.PubKey().Address()) + require.Nil(t, err) + + for i, minter := range cfg.Networks.Minters { + switch i { + case 4: + minter.MinterAddress = caller + minter.MinterPrivateKey = callerPrivKey + cfg.Networks.Minters[4] = minter + } + } + + _, nextMinterSequence, err := noble.GetNobleAccountNumberSequence( + cfg.Networks.Destination.Noble.API, + cfg.Networks.Minters[4].MinterAddress) + + require.Nil(t, err) + + sequenceMap = types.NewSequenceMap() + sequenceMap.Put(uint32(4), nextMinterSequence) + + // number of depositForBurn txns to send + n := 7 + + // start up relayer + cfg.Networks.Source.Ethereum.StartBlock = getEthereumLatestBlockHeight(t) + cfg.Networks.Source.Ethereum.LookbackPeriod = 5 + cfg.Networks.Destination.Noble.BroadcastRetries = 0 // don't rely on retries to broadcast txns + + fmt.Println(fmt.Sprintf("Building %d Ethereum depositForBurnWithMetadata txns...", n)) + + _, _, cosmosAddress := testdata.KeyTestPubAddr() + nobleAddress, _ := bech32.ConvertAndEncode("noble", cosmosAddress) + fmt.Println("Minting on Noble to https://testnet.mintscan.io/noble-testnet/account/" + nobleAddress) + + // verify original noble usdc amount + originalNobleBalance := getNobleBalance(nobleAddress) + + // deposit for burn with metadata + client, err := ethclient.Dial(testCfg.Networks.Ethereum.RPC) + require.Nil(t, err) + defer client.Close() + + privateKey, err := crypto.HexToECDSA(testCfg.Networks.Ethereum.PrivateKey) + require.Nil(t, err) + auth, err := bind.NewKeyedTransactorWithChainID(privateKey, big.NewInt(5)) + require.Nil(t, err) + + tokenMessenger, err := cmd.NewTokenMessenger(common.HexToAddress(TokenMessengerAddress), client) + require.Nil(t, err) + + mintRecipientPadded := append([]byte{0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0}, cosmosAddress...) + require.Nil(t, err) + + _, callerRaw, _ := bech32.DecodeAndConvert(caller) + destinationCallerPadded := append([]byte{0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0}, callerRaw...) + require.Nil(t, err) + + erc20, err := NewERC20(common.HexToAddress(UsdcAddress), client) + _, err = erc20.Approve(auth, common.HexToAddress(TokenMessengerWithMetadataAddress), big.NewInt(99999)) + require.Nil(t, err) + var burnAmount = big.NewInt(1) + + for i := 0; i < n; i++ { + tx, err := tokenMessenger.DepositForBurnWithCaller( + auth, + burnAmount, + 4, + [32]byte(mintRecipientPadded), + common.HexToAddress(UsdcAddress), + [32]byte(destinationCallerPadded), + ) + if err != nil { + logger.Error("Failed to update value: %v", err) + } + + time.Sleep(1 * time.Second) + + fmt.Printf("Update pending: https://goerli.etherscan.io/tx/%s\n", tx.Hash().String()) + + } + + fmt.Println("Waiting 90 seconds for attestations...") + time.Sleep(90 * time.Second) + + fmt.Println("Starting relayer...") + processingQueue := make(chan *types.MessageState, 100) + + go eth.StartListener(cfg, logger, processingQueue) + go cmd.StartProcessor(cfg, logger, processingQueue, sequenceMap) + + fmt.Println("Checking noble wallet...") + for i := 0; i < 250; i++ { + if originalNobleBalance+burnAmount.Uint64()*uint64(n) == getNobleBalance(nobleAddress) { + fmt.Println(fmt.Sprintf("Successfully minted %d times at https://testnet.mintscan.io/noble-testnet/account/%s", n, nobleAddress)) + return + } + time.Sleep(1 * time.Second) + } +} diff --git a/types/sequence_map.go b/types/sequence_map.go index 33c7f94..dab63f1 100644 --- a/types/sequence_map.go +++ b/types/sequence_map.go @@ -4,7 +4,7 @@ import ( "sync" ) -// SequenceMap holds the account sequence to avoid account sequence mismatch errors +// SequenceMap holds a minter account's txn count to avoid account sequence mismatch errors type SequenceMap struct { mu sync.Mutex // map destination domain -> minter account sequence