From 113804588cab0a4e8da42228dc4e1e134ded58ec Mon Sep 17 00:00:00 2001 From: Richard Ramos Date: Thu, 19 Sep 2024 16:16:53 -0400 Subject: [PATCH] refactor_: extract storenode cycle to go-waku api --- eth-node/bridge/geth/waku.go | 62 ++- eth-node/bridge/geth/wakuv2.go | 109 ++-- eth-node/types/mailserver.go | 54 -- eth-node/types/topic.go | 4 + eth-node/types/waku.go | 57 +- go.mod | 2 +- go.sum | 4 +- protocol/messenger.go | 81 +-- protocol/messenger_communities.go | 27 +- protocol/messenger_mailserver.go | 412 +++----------- protocol/messenger_mailserver_cycle.go | 521 +++--------------- ..._mailserver_processMailserverBatch_test.go | 167 ------ .../messenger_store_node_request_manager.go | 15 +- ...enger_store_node_request_manager_config.go | 8 +- protocol/messenger_storenode_comunity_test.go | 4 +- protocol/messenger_storenode_request_test.go | 16 +- protocol/messenger_testing_utils.go | 2 +- protocol/storenodes/storenodes.go | 12 +- protocol/transport/transport.go | 144 ++--- services/ext/api.go | 4 - .../waku-org/go-waku/logging/logging.go | 8 + .../go-waku/waku/v2/api/history/cycle.go | 485 ++++++++++++++++ .../go-waku/waku/v2/api/history/emitters.go | 49 ++ .../go-waku/waku/v2/api/history/history.go | 281 ++++++++++ .../go-waku/waku/v2/api/history/sort.go | 32 ++ .../waku/v2/api/publish/message_check.go | 15 +- .../waku/v2/api/publish/message_sender.go | 7 - .../go-waku/waku/v2/node/wakunode2.go | 2 +- .../go-waku/waku/v2/node/wakuoptions.go | 14 + .../waku/v2/protocol/peer_exchange/client.go | 2 - .../go-waku/waku/v2/protocol/relay/config.go | 5 + .../go-waku/waku/v2/protocol/store/client.go | 45 +- .../go-waku/waku/v2/protocol/store/options.go | 9 + .../waku/v2/protocol/store/pb/validation.go | 6 +- .../go-waku/waku/v2/protocol/store/result.go | 4 +- vendor/modules.txt | 3 +- wakuv2/history_processor_wrapper.go | 24 + wakuv2/waku.go | 81 +-- 38 files changed, 1437 insertions(+), 1340 deletions(-) delete mode 100644 protocol/messenger_mailserver_processMailserverBatch_test.go create mode 100644 vendor/github.com/waku-org/go-waku/waku/v2/api/history/cycle.go create mode 100644 vendor/github.com/waku-org/go-waku/waku/v2/api/history/emitters.go create mode 100644 vendor/github.com/waku-org/go-waku/waku/v2/api/history/history.go create mode 100644 vendor/github.com/waku-org/go-waku/waku/v2/api/history/sort.go create mode 100644 wakuv2/history_processor_wrapper.go diff --git a/eth-node/bridge/geth/waku.go b/eth-node/bridge/geth/waku.go index 83b87272475..c4e654b20f8 100644 --- a/eth-node/bridge/geth/waku.go +++ b/eth-node/bridge/geth/waku.go @@ -9,6 +9,8 @@ import ( "github.com/libp2p/go-libp2p/core/peer" "github.com/multiformats/go-multiaddr" + "github.com/waku-org/go-waku/waku/v2/api/history" + "github.com/ethereum/go-ethereum/common" "github.com/ethereum/go-ethereum/p2p/enode" "github.com/status-im/status-go/connection" @@ -272,10 +274,6 @@ func (w *GethWakuWrapper) MarkP2PMessageAsProcessed(hash common.Hash) { w.waku.MarkP2PMessageAsProcessed(hash) } -func (w *GethWakuWrapper) RequestStoreMessages(ctx context.Context, peerID peer.ID, r types.MessagesRequest, processEnvelopes bool) (types.StoreRequestCursor, int, error) { - return nil, 0, errors.New("not implemented") -} - func (w *GethWakuWrapper) ConnectionChanged(_ connection.State) {} func (w *GethWakuWrapper) ClearEnvelopesCache() { @@ -312,13 +310,59 @@ func (w *wakuFilterWrapper) ID() string { func (w *GethWakuWrapper) ConfirmMessageDelivered(hashes []common.Hash) { } -func (w *GethWakuWrapper) SetStorePeerID(peerID peer.ID) { +func (w *GethWakuWrapper) PeerID() peer.ID { + panic("not available in WakuV1") } -func (w *GethWakuWrapper) PeerID() peer.ID { - panic("not implemented") +func (w *GethWakuWrapper) GetActiveStorenode() peer.ID { + panic("not available in WakuV1") +} + +func (w *GethWakuWrapper) OnStorenodeAvailableOneShot() <-chan struct{} { + panic("not available in WakuV1") +} + +func (w *GethWakuWrapper) OnStorenodeChanged() <-chan peer.ID { + panic("not available in WakuV1") +} + +func (w *GethWakuWrapper) OnStorenodeNotWorking() <-chan struct{} { + panic("not available in WakuV1") +} + +func (w *GethWakuWrapper) OnStorenodeAvailable() <-chan peer.ID { + panic("not available in WakuV1") +} + +func (w *GethWakuWrapper) WaitForAvailableStoreNode(timeout time.Duration) bool { + panic("not available in WakuV1") +} + +func (w *GethWakuWrapper) SetStorenodeConfigProvider(c history.StorenodeConfigProvider) { + panic("not available in WakuV1") +} + +func (w *GethWakuWrapper) ProcessMailserverBatch( + ctx context.Context, + batch types.MailserverBatch, + storenodeID peer.ID, + pageLimit uint64, + shouldProcessNextPage func(int) (bool, uint64), + processEnvelopes bool, +) error { + return errors.New("not available in WakuV1") +} + +func (w *GethWakuWrapper) IsStorenodeAvailable(peerID peer.ID) bool { + panic("not available in WakuV1") + +} + +func (w *GethWakuWrapper) PerformStorenodeTask(peerID peer.ID, fn func(peerID peer.ID) error) error { + panic("not available in WakuV1") + } -func (w *GethWakuWrapper) PingPeer(context.Context, peer.ID) (time.Duration, error) { - return 0, errors.New("not available in WakuV1") +func (w *GethWakuWrapper) DisconnectActiveStorenode(ctx context.Context, backoff time.Duration, shouldCycle bool) { + panic("not available in WakuV1") } diff --git a/eth-node/bridge/geth/wakuv2.go b/eth-node/bridge/geth/wakuv2.go index 750790be4af..b6bedcb0885 100644 --- a/eth-node/bridge/geth/wakuv2.go +++ b/eth-node/bridge/geth/wakuv2.go @@ -9,6 +9,7 @@ import ( "github.com/multiformats/go-multiaddr" "google.golang.org/protobuf/proto" + "github.com/waku-org/go-waku/waku/v2/api/history" "github.com/waku-org/go-waku/waku/v2/protocol" "github.com/waku-org/go-waku/waku/v2/protocol/store" @@ -173,39 +174,6 @@ func (w *gethWakuV2Wrapper) createFilterWrapper(id string, keyAsym *ecdsa.Privat }, id), nil } -func (w *gethWakuV2Wrapper) RequestStoreMessages(ctx context.Context, peerID peer.ID, r types.MessagesRequest, processEnvelopes bool) (types.StoreRequestCursor, int, error) { - options := []store.RequestOption{ - store.WithPaging(false, uint64(r.Limit)), - } - - var cursor []byte - if r.StoreCursor != nil { - cursor = r.StoreCursor - } - - contentTopics := []string{} - for _, topic := range r.ContentTopics { - contentTopics = append(contentTopics, wakucommon.BytesToTopic(topic).ContentTopic()) - } - - query := store.FilterCriteria{ - TimeStart: proto.Int64(int64(r.From) * int64(time.Second)), - TimeEnd: proto.Int64(int64(r.To) * int64(time.Second)), - ContentFilter: protocol.NewContentFilter(w.waku.GetPubsubTopic(r.PubsubTopic), contentTopics...), - } - - pbCursor, envelopesCount, err := w.waku.Query(ctx, peerID, query, cursor, options, processEnvelopes) - if err != nil { - return nil, 0, err - } - - if pbCursor != nil { - return pbCursor, envelopesCount, nil - } - - return nil, envelopesCount, nil -} - func (w *gethWakuV2Wrapper) StartDiscV5() error { return w.waku.StartDiscV5() } @@ -286,7 +254,7 @@ func (w *gethWakuV2Wrapper) SubscribeToConnStatusChanges() (*types.ConnStatusSub func (w *gethWakuV2Wrapper) SetCriteriaForMissingMessageVerification(peerID peer.ID, pubsubTopic string, contentTopics []types.TopicType) error { var cTopics []string for _, ct := range contentTopics { - cTopics = append(cTopics, wakucommon.TopicType(ct).ContentTopic()) + cTopics = append(cTopics, wakucommon.BytesToTopic(ct.Bytes()).ContentTopic()) } pubsubTopic = w.waku.GetPubsubTopic(pubsubTopic) w.waku.SetTopicsToVerifyForMissingMessages(peerID, pubsubTopic, cTopics) @@ -335,14 +303,75 @@ func (w *gethWakuV2Wrapper) ConfirmMessageDelivered(hashes []common.Hash) { w.waku.ConfirmMessageDelivered(hashes) } -func (w *gethWakuV2Wrapper) SetStorePeerID(peerID peer.ID) { - w.waku.SetStorePeerID(peerID) -} - func (w *gethWakuV2Wrapper) PeerID() peer.ID { return w.waku.PeerID() } -func (w *gethWakuV2Wrapper) PingPeer(ctx context.Context, peerID peer.ID) (time.Duration, error) { - return w.waku.PingPeer(ctx, peerID) +func (w *gethWakuV2Wrapper) GetActiveStorenode() peer.ID { + return w.waku.StorenodeCycle.GetActiveStorenode() +} + +func (w *gethWakuV2Wrapper) OnStorenodeAvailableOneShot() <-chan struct{} { + return w.waku.StorenodeCycle.StorenodeAvailableOneshotEmitter.Subscribe() +} + +func (w *gethWakuV2Wrapper) OnStorenodeChanged() <-chan peer.ID { + return w.waku.StorenodeCycle.StorenodeChangedEmitter.Subscribe() +} + +func (w *gethWakuV2Wrapper) OnStorenodeNotWorking() <-chan struct{} { + return w.waku.StorenodeCycle.StorenodeNotWorkingEmitter.Subscribe() +} + +func (w *gethWakuV2Wrapper) OnStorenodeAvailable() <-chan peer.ID { + return w.waku.StorenodeCycle.StorenodeAvailableEmitter.Subscribe() +} + +func (w *gethWakuV2Wrapper) WaitForAvailableStoreNode(timeout time.Duration) bool { + return w.waku.StorenodeCycle.WaitForAvailableStoreNode(context.TODO(), timeout) +} + +func (w *gethWakuV2Wrapper) SetStorenodeConfigProvider(c history.StorenodeConfigProvider) { + w.waku.StorenodeCycle.SetStorenodeConfigProvider(c) +} + +func (w *gethWakuV2Wrapper) ProcessMailserverBatch( + ctx context.Context, + batch types.MailserverBatch, + storenodeID peer.ID, + pageLimit uint64, + shouldProcessNextPage func(int) (bool, uint64), + processEnvelopes bool, +) error { + pubsubTopic := w.waku.GetPubsubTopic(batch.PubsubTopic) + contentTopics := []string{} + for _, topic := range batch.Topics { + contentTopics = append(contentTopics, wakucommon.BytesToTopic(topic.Bytes()).ContentTopic()) + } + + criteria := store.FilterCriteria{ + TimeStart: proto.Int64(int64(batch.From) * int64(time.Second)), + TimeEnd: proto.Int64(int64(batch.To) * int64(time.Second)), + ContentFilter: protocol.NewContentFilter(pubsubTopic, contentTopics...), + } + + return w.waku.HistoryRetriever.Query(ctx, criteria, storenodeID, pageLimit, shouldProcessNextPage, processEnvelopes) +} + +func (w *gethWakuV2Wrapper) IsStorenodeAvailable(peerID peer.ID) bool { + return w.waku.StorenodeCycle.IsStorenodeAvailable(peerID) +} + +func (w *gethWakuV2Wrapper) PerformStorenodeTask(peerID peer.ID, fn func(peerID peer.ID) error) error { + return w.waku.StorenodeCycle.PerformStorenodeTask(peerID, fn) +} + +func (w *gethWakuV2Wrapper) DisconnectActiveStorenode(ctx context.Context, backoff time.Duration, shouldCycle bool) { + w.waku.StorenodeCycle.Lock() + defer w.waku.StorenodeCycle.Unlock() + + w.waku.StorenodeCycle.DisconnectActiveStorenode(backoff) + if shouldCycle { + w.waku.StorenodeCycle.Cycle(ctx) + } } diff --git a/eth-node/types/mailserver.go b/eth-node/types/mailserver.go index 2ae7877ceac..dd24824867d 100644 --- a/eth-node/types/mailserver.go +++ b/eth-node/types/mailserver.go @@ -1,59 +1,5 @@ package types -import ( - "time" -) - -const ( - // MaxLimitInMessagesRequest represents the maximum number of messages - // that can be requested from the mailserver - MaxLimitInMessagesRequest = 1000 -) - -// MessagesRequest contains details of a request of historic messages. -type MessagesRequest struct { - // ID of the request. The current implementation requires ID to be 32-byte array, - // however, it's not enforced for future implementation. - ID []byte `json:"id"` - // From is a lower bound of time range. - From uint32 `json:"from"` - // To is a upper bound of time range. - To uint32 `json:"to"` - // Limit determines the number of messages sent by the mail server - // for the current paginated request. - Limit uint32 `json:"limit"` - // Cursor is used as starting point for paginated requests. - Cursor []byte `json:"cursor"` - // StoreCursor is used as starting point for WAKUV2 paginatedRequests - StoreCursor StoreRequestCursor `json:"storeCursor"` - // Bloom is a filter to match requested messages. - Bloom []byte `json:"bloom"` - // PubsubTopic is the gossipsub topic on which the message was broadcasted - PubsubTopic string `json:"pubsubTopic"` - // ContentTopics is a list of topics. A returned message should - // belong to one of the topics from the list. - ContentTopics [][]byte `json:"contentTopics"` -} - -type StoreRequestCursor []byte - -// SetDefaults sets the From and To defaults -func (r *MessagesRequest) SetDefaults(now time.Time) { - // set From and To defaults - if r.To == 0 { - r.To = uint32(now.UTC().Unix()) - } - - if r.From == 0 { - oneDay := uint32(86400) // -24 hours - if r.To < oneDay { - r.From = 0 - } else { - r.From = r.To - oneDay - } - } -} - // MailServerResponse is the response payload sent by the mailserver. type MailServerResponse struct { LastEnvelopeHash Hash diff --git a/eth-node/types/topic.go b/eth-node/types/topic.go index faeeb809ec7..c8fbdeca71e 100644 --- a/eth-node/types/topic.go +++ b/eth-node/types/topic.go @@ -34,6 +34,10 @@ func (t TopicType) String() string { return EncodeHex(t[:]) } +func (t TopicType) Bytes() []byte { + return TopicTypeToByteArray(t) +} + // MarshalText returns the hex representation of t. func (t TopicType) MarshalText() ([]byte, error) { return HexBytes(t[:]).MarshalText() diff --git a/eth-node/types/waku.go b/eth-node/types/waku.go index 1900505f34b..0d9c9cbabc8 100644 --- a/eth-node/types/waku.go +++ b/eth-node/types/waku.go @@ -12,6 +12,8 @@ import ( "github.com/multiformats/go-multiaddr" "github.com/pborman/uuid" + "github.com/waku-org/go-waku/waku/v2/api/history" + "github.com/ethereum/go-ethereum/common" "github.com/ethereum/go-ethereum/p2p/enode" "github.com/status-im/status-go/connection" @@ -176,9 +178,6 @@ type Waku interface { Unsubscribe(ctx context.Context, id string) error UnsubscribeMany(ids []string) error - // RequestStoreMessages uses the WAKU2-STORE protocol to request historic messages - RequestStoreMessages(ctx context.Context, peerID peer.ID, request MessagesRequest, processEnvelopes bool) (StoreRequestCursor, int, error) - // ProcessingP2PMessages indicates whether there are in-flight p2p messages ProcessingP2PMessages() bool @@ -194,12 +193,54 @@ type Waku interface { // ConfirmMessageDelivered updates a message has been delivered in waku ConfirmMessageDelivered(hash []common.Hash) - // SetStorePeerID updates the peer id of store node - SetStorePeerID(peerID peer.ID) - // PeerID returns node's PeerID PeerID() peer.ID - // PingPeer returns the reply time - PingPeer(ctx context.Context, peerID peer.ID) (time.Duration, error) + // GetActiveStorenode returns the peer ID of the currently active storenode. It will be empty if no storenode is active + GetActiveStorenode() peer.ID + + // OnStorenodeAvailableOneShot returns a channel that will be triggered only once when a storenode becomes available + OnStorenodeAvailableOneShot() <-chan struct{} + + // OnStorenodeChanged is triggered when a new storenode is promoted to become the active storenode or when the active storenode is removed + OnStorenodeChanged() <-chan peer.ID + + // OnStorenodeNotWorking is triggered when the last active storenode fails to return results consistently + OnStorenodeNotWorking() <-chan struct{} + + // OnStorenodeAvailable is triggered when there is a new active storenode selected + OnStorenodeAvailable() <-chan peer.ID + + // WaitForAvailableStoreNode will wait for a storenode to be available until `timeout` happens + WaitForAvailableStoreNode(timeout time.Duration) bool + + // SetStorenodeConfigProvider will set the configuration provider for the storenode cycle + SetStorenodeConfigProvider(c history.StorenodeConfigProvider) + + // ProcessMailserverBatch will receive a criteria and storenode and execute a query + ProcessMailserverBatch( + ctx context.Context, + batch MailserverBatch, + storenodeID peer.ID, + pageLimit uint64, + shouldProcessNextPage func(int) (bool, uint64), + processEnvelopes bool, + ) error + + // IsStorenodeAvailable is used to determine whether a storenode is available or not + IsStorenodeAvailable(peerID peer.ID) bool + + PerformStorenodeTask(peerID peer.ID, fn func(peerID peer.ID) error) error + + // DisconnectActiveStorenode will trigger a disconnection of the active storenode, and potentially execute a cycling so a new storenode is promoted + DisconnectActiveStorenode(ctx context.Context, backoff time.Duration, shouldCycle bool) +} + +type MailserverBatch struct { + From uint32 + To uint32 + Cursor string + PubsubTopic string + Topics []TopicType + ChatIDs []string } diff --git a/go.mod b/go.mod index f3cdbb5fa76..b88129a40cd 100644 --- a/go.mod +++ b/go.mod @@ -95,7 +95,7 @@ require ( github.com/schollz/peerdiscovery v1.7.0 github.com/siphiuel/lc-proxy-wrapper v0.0.0-20230516150924-246507cee8c7 github.com/urfave/cli/v2 v2.27.2 - github.com/waku-org/go-waku v0.8.1-0.20240904143057-f9e7895202da + github.com/waku-org/go-waku v0.8.1-0.20240919183940-d9361e5302c3 github.com/wk8/go-ordered-map/v2 v2.1.7 github.com/yeqown/go-qrcode/v2 v2.2.1 github.com/yeqown/go-qrcode/writer/standard v1.2.1 diff --git a/go.sum b/go.sum index 31adf513a3b..54ab4575c42 100644 --- a/go.sum +++ b/go.sum @@ -2137,8 +2137,8 @@ github.com/waku-org/go-libp2p-pubsub v0.0.0-20240703191659-2cbb09eac9b5 h1:9u16E github.com/waku-org/go-libp2p-pubsub v0.0.0-20240703191659-2cbb09eac9b5/go.mod h1:QEb+hEV9WL9wCiUAnpY29FZR6W3zK8qYlaml8R4q6gQ= github.com/waku-org/go-libp2p-rendezvous v0.0.0-20240110193335-a67d1cc760a0 h1:R4YYx2QamhBRl/moIxkDCNW+OP7AHbyWLBygDc/xIMo= github.com/waku-org/go-libp2p-rendezvous v0.0.0-20240110193335-a67d1cc760a0/go.mod h1:EhZP9fee0DYjKH/IOQvoNSy1tSHp2iZadsHGphcAJgY= -github.com/waku-org/go-waku v0.8.1-0.20240904143057-f9e7895202da h1:bkAJVlJL4Ba83frABWjI9p9MeLGmEHuD/QcjYu3HNbQ= -github.com/waku-org/go-waku v0.8.1-0.20240904143057-f9e7895202da/go.mod h1:VNbVmh5UYg3vIvhGV4hCw8QEykq3RScDACo2Y2dIFfg= +github.com/waku-org/go-waku v0.8.1-0.20240919183940-d9361e5302c3 h1:DTD7SQec+ffC74JB9FrGQ7i7Nria6o1ZwpmE8mZdORI= +github.com/waku-org/go-waku v0.8.1-0.20240919183940-d9361e5302c3/go.mod h1:VNbVmh5UYg3vIvhGV4hCw8QEykq3RScDACo2Y2dIFfg= github.com/waku-org/go-zerokit-rln v0.1.14-0.20240102145250-fa738c0bdf59 h1:jisj+OCI6QydLtFq3Pyhu49wl9ytPN7oAHjMfepHDrA= github.com/waku-org/go-zerokit-rln v0.1.14-0.20240102145250-fa738c0bdf59/go.mod h1:1PdBdPzyTaKt3VnpAHk3zj+r9dXPFOr3IHZP9nFle6E= github.com/waku-org/go-zerokit-rln-apple v0.0.0-20230916172309-ee0ee61dde2b h1:KgZVhsLkxsj5gb/FfndSCQu6VYwALrCOgYI3poR95yE= diff --git a/protocol/messenger.go b/protocol/messenger.go index c543a7d9ebd..34c50eb60ee 100644 --- a/protocol/messenger.go +++ b/protocol/messenger.go @@ -138,7 +138,6 @@ type Messenger struct { allInstallations *installationMap modifiedInstallations *stringBoolMap installationID string - mailserverCycle mailserverCycle communityStorenodes *storenodes.CommunityStorenodes database *sql.DB multiAccounts *multiaccounts.Database @@ -171,7 +170,6 @@ type Messenger struct { // TODO(samyoul) Determine if/how the remaining usage of this mutex can be removed mutex sync.Mutex - mailPeersMutex sync.RWMutex handleMessagesMutex sync.Mutex handleImportMessagesMutex sync.Mutex @@ -198,50 +196,6 @@ type Messenger struct { mvdsStatusChangeEvent chan datasyncnode.PeerStatusChangeEvent } -type connStatus int - -const ( - disconnected connStatus = iota + 1 - connected -) - -type peerStatus struct { - status connStatus - canConnectAfter time.Time - lastConnectionAttempt time.Time - mailserver mailserversDB.Mailserver -} -type mailserverCycle struct { - sync.RWMutex - allMailservers []mailserversDB.Mailserver - activeMailserver *mailserversDB.Mailserver - peers map[string]peerStatus - availabilitySubscriptions *availabilitySubscriptions -} - -type availabilitySubscriptions struct { - sync.Mutex - subscriptions []chan struct{} -} - -func (s *availabilitySubscriptions) Subscribe() <-chan struct{} { - s.Lock() - defer s.Unlock() - c := make(chan struct{}) - s.subscriptions = append(s.subscriptions, c) - return c -} - -func (s *availabilitySubscriptions) EmitMailserverAvailable() { - s.Lock() - defer s.Unlock() - - for _, subs := range s.subscriptions { - close(subs) - } - s.subscriptions = nil -} - type EnvelopeEventsInterceptor struct { EnvelopeEventsHandler transport.EnvelopeEventsHandler Messenger *Messenger @@ -623,19 +577,15 @@ func NewMessenger( peerStore: peerStore, mvdsStatusChangeEvent: make(chan datasyncnode.PeerStatusChangeEvent, 5), verificationDatabase: verification.NewPersistence(database), - mailserverCycle: mailserverCycle{ - peers: make(map[string]peerStatus), - availabilitySubscriptions: &availabilitySubscriptions{}, - }, - mailserversDatabase: c.mailserversDatabase, - communityStorenodes: storenodes.NewCommunityStorenodes(storenodes.NewDB(database), logger), - account: c.account, - quit: make(chan struct{}), - ctx: ctx, - cancel: cancel, - importingCommunities: make(map[string]bool), - importingChannels: make(map[string]bool), - importRateLimiter: rate.NewLimiter(rate.Every(importSlowRate), 1), + mailserversDatabase: c.mailserversDatabase, + communityStorenodes: storenodes.NewCommunityStorenodes(storenodes.NewDB(database), logger), + account: c.account, + quit: make(chan struct{}), + ctx: ctx, + cancel: cancel, + importingCommunities: make(map[string]bool), + importingChannels: make(map[string]bool), + importRateLimiter: rate.NewLimiter(rate.Every(importSlowRate), 1), importDelayer: struct { wait chan struct{} once sync.Once @@ -882,22 +832,26 @@ func (m *Messenger) Start() (*MessengerResponse, error) { } response := &MessengerResponse{} - mailservers, err := m.allMailservers() + storenodes, err := m.AllMailservers() if err != nil { return nil, err } - response.Mailservers = mailservers - err = m.StartMailserverCycle(mailservers) + err = m.setupStorenodes(storenodes) if err != nil { return nil, err } + response.Mailservers = storenodes + + m.transport.SetStorenodeConfigProvider(m) + if err := m.communityStorenodes.ReloadFromDB(); err != nil { return nil, err } go m.checkForMissingMessagesLoop() + go m.checkForStorenodeCycleSignals() controlledCommunities, err := m.communitiesManager.Controlled() if err != nil { @@ -905,9 +859,8 @@ func (m *Messenger) Start() (*MessengerResponse, error) { } if m.archiveManager.IsReady() { - available := m.mailserverCycle.availabilitySubscriptions.Subscribe() go func() { - <-available + <-m.transport.OnStorenodeAvailableOneShot() m.InitHistoryArchiveTasks(controlledCommunities) }() } diff --git a/protocol/messenger_communities.go b/protocol/messenger_communities.go index 8049f24ed04..064885a280d 100644 --- a/protocol/messenger_communities.go +++ b/protocol/messenger_communities.go @@ -16,6 +16,7 @@ import ( "github.com/golang/protobuf/proto" "github.com/google/uuid" + "github.com/libp2p/go-libp2p/core/peer" gethcommon "github.com/ethereum/go-ethereum/common" "github.com/ethereum/go-ethereum/common/hexutil" @@ -39,6 +40,7 @@ import ( "github.com/status-im/status-go/protocol/encryption" "github.com/status-im/status-go/protocol/protobuf" "github.com/status-im/status-go/protocol/requests" + "github.com/status-im/status-go/protocol/storenodes" "github.com/status-im/status-go/protocol/transport" v1protocol "github.com/status-im/status-go/protocol/v1" localnotifications "github.com/status-im/status-go/services/local-notifications" @@ -3962,8 +3964,8 @@ func (m *Messenger) InitHistoryArchiveTasks(communities []*communities.Community } // Request possibly missed waku messages for community - ms := m.getActiveMailserver(c.ID().String()) - _, err = m.syncFiltersFrom(*ms, filters, uint32(latestWakuMessageTimestamp)) + ms := m.getCommunityMailserver(c.ID().String()) + _, err = m.syncFiltersFrom(ms, filters, uint32(latestWakuMessageTimestamp)) if err != nil { m.logger.Error("failed to request missing messages", zap.Error(err)) continue @@ -5142,3 +5144,24 @@ func (m *Messenger) startRequestMissingCommunityChannelsHRKeysLoop() { } }() } + +// getCommunityMailserver returns the active mailserver if a communityID is present then it'll return the mailserver +// for that community if it has a mailserver setup otherwise it'll return the global mailserver +func (m *Messenger) getCommunityMailserver(communityID ...string) peer.ID { + if len(communityID) == 0 || communityID[0] == "" { + return m.transport.GetActiveStorenode() + } + + ms, err := m.communityStorenodes.GetStorenodeByCommunityID(communityID[0]) + if err != nil { + if !errors.Is(err, storenodes.ErrNotFound) { + m.logger.Error("getting storenode for community, using global", zap.String("communityID", communityID[0]), zap.Error(err)) + } + // if we don't find a specific mailserver for the community, we just use the regular mailserverCycle's one + return m.transport.GetActiveStorenode() + } + + peerID, _ := ms.PeerID() + + return peerID +} diff --git a/protocol/messenger_mailserver.go b/protocol/messenger_mailserver.go index 76c734adabd..cd189fff8c1 100644 --- a/protocol/messenger_mailserver.go +++ b/protocol/messenger_mailserver.go @@ -1,11 +1,9 @@ package protocol import ( - "context" "fmt" "math" "sort" - "sync" "time" "github.com/libp2p/go-libp2p/core/peer" @@ -28,22 +26,17 @@ const ( // tolerance is how many seconds of potentially out-of-order messages we want to fetch tolerance uint32 = 60 - mailserverRequestTimeout = 30 * time.Second - mailserverMaxTries uint = 2 - mailserverMaxFailedRequests uint = 2 - oneDayDuration = 24 * time.Hour oneMonthDuration = 31 * oneDayDuration -) -// maxTopicsPerRequest sets the batch size to limit the number of topics per store query -var maxTopicsPerRequest int = 10 + backoffByUserAction = 0 * time.Second +) var ErrNoFiltersForChat = errors.New("no filter registered for given chat") func (m *Messenger) shouldSync() (bool, error) { // TODO (pablo) support community store node as well - if m.mailserverCycle.activeMailserver == nil || !m.Online() { + if m.transport.GetActiveStorenode() == "" || !m.Online() { return false, nil } @@ -68,9 +61,9 @@ func (m *Messenger) scheduleSyncChat(chat *Chat) (bool, error) { } go func() { - ms := m.getActiveMailserver(chat.CommunityID) - _, err = m.performMailserverRequest(ms, func(mailServer mailservers.Mailserver) (*MessengerResponse, error) { - response, err := m.syncChatWithFilters(mailServer, chat.ID) + peerID := m.getCommunityMailserver(chat.CommunityID) + _, err = m.performMailserverRequest(peerID, func() (*MessengerResponse, error) { + response, err := m.syncChatWithFilters(peerID, chat.ID) if err != nil { m.logger.Error("failed to sync chat", zap.Error(err)) @@ -89,65 +82,42 @@ func (m *Messenger) scheduleSyncChat(chat *Chat) (bool, error) { return true, nil } -func (m *Messenger) connectToNewMailserverAndWait() error { - // Handle pinned mailservers - m.logger.Info("disconnecting mailserver") - pinnedMailserver, err := m.getPinnedMailserver() - if err != nil { - m.logger.Error("could not obtain the pinned mailserver", zap.Error(err)) - return err - } - // If pinned mailserver is not nil, no need to disconnect and wait for it to be available - if pinnedMailserver == nil { - m.disconnectActiveMailserver(graylistBackoff) - } +func (m *Messenger) performMailserverRequest(peerID peer.ID, task func() (*MessengerResponse, error)) (*MessengerResponse, error) { - return m.findNewMailserver() -} - -func (m *Messenger) performMailserverRequest(ms *mailservers.Mailserver, fn func(mailServer mailservers.Mailserver) (*MessengerResponse, error)) (*MessengerResponse, error) { - if ms == nil { - return nil, errors.New("mailserver not available") - } - - m.mailserverCycle.RLock() - defer m.mailserverCycle.RUnlock() - var tries uint = 0 - for tries < mailserverMaxTries { - if !m.communityStorenodes.IsCommunityStoreNode(ms.ID) && !m.isMailserverAvailable(ms.ID) { - return nil, errors.New("storenode not available") - } - m.logger.Info("trying performing mailserver requests", zap.Uint("try", tries), zap.String("mailserverID", ms.ID)) - - // Peform request - response, err := fn(*ms) // pass by value because we don't want the fn to modify the mailserver - if err == nil { - // Reset failed requests - m.logger.Debug("mailserver request performed successfully", - zap.String("mailserverID", ms.ID)) - ms.FailedRequests = 0 - return response, nil - } + responseCh := make(chan *MessengerResponse) + errCh := make(chan error) - m.logger.Error("failed to perform mailserver request", - zap.String("mailserverID", ms.ID), - zap.Uint("tries", tries), - zap.Error(err), - ) + go func() { + err := m.transport.PerformStorenodeTask(peerID, func(peerID peer.ID) error { + r, err := task() + if err != nil { + return err + } - tries++ - // Increment failed requests - ms.FailedRequests++ + select { + case responseCh <- r: + default: + // + } - // Change mailserver - if ms.FailedRequests >= mailserverMaxFailedRequests { - return nil, errors.New("too many failed requests") + return nil + }) + if err != nil { + errCh <- err } - // Wait a couple of second not to spam - time.Sleep(2 * time.Second) + }() + select { + case err := <-errCh: + return nil, err + case r := <-responseCh: + if r != nil { + return r, nil + } + return nil, errors.New("no response available") + case <-m.ctx.Done(): + return nil, m.ctx.Err() } - return nil, errors.New("failed to perform mailserver request") } func (m *Messenger) scheduleSyncFilters(filters []*transport.Filter) (bool, error) { @@ -165,9 +135,9 @@ func (m *Messenger) scheduleSyncFilters(filters []*transport.Filter) (bool, erro // split filters by community store node so we can request the filters to the correct mailserver filtersByMs := m.SplitFiltersByStoreNode(filters) for communityID, filtersForMs := range filtersByMs { - ms := m.getActiveMailserver(communityID) - _, err := m.performMailserverRequest(ms, func(ms mailservers.Mailserver) (*MessengerResponse, error) { - response, err := m.syncFilters(ms, filtersForMs) + peerID := m.getCommunityMailserver(communityID) + _, err := m.performMailserverRequest(peerID, func() (*MessengerResponse, error) { + response, err := m.syncFilters(peerID, filtersForMs) if err != nil { m.logger.Error("failed to sync filter", zap.Error(err)) @@ -247,13 +217,13 @@ func (m *Messenger) topicsForChat(chatID string) (string, []types.TopicType, err return filters[0].PubsubTopic, contentTopics, nil } -func (m *Messenger) syncChatWithFilters(ms mailservers.Mailserver, chatID string) (*MessengerResponse, error) { +func (m *Messenger) syncChatWithFilters(peerID peer.ID, chatID string) (*MessengerResponse, error) { filters, err := m.filtersForChat(chatID) if err != nil { return nil, err } - return m.syncFilters(ms, filters) + return m.syncFilters(peerID, filters) } func (m *Messenger) syncBackup() error { @@ -272,9 +242,9 @@ func (m *Messenger) syncBackup() error { from, to := m.calculateMailserverTimeBounds(oneMonthDuration) - batch := MailserverBatch{From: from, To: to, Topics: []types.TopicType{filter.ContentTopic}} - ms := m.getActiveMailserver(filter.ChatID) - err = m.processMailserverBatch(*ms, batch) + batch := types.MailserverBatch{From: from, To: to, Topics: []types.TopicType{filter.ContentTopic}} + ms := m.getCommunityMailserver(filter.ChatID) + err = m.processMailserverBatch(ms, batch) if err != nil { return err } @@ -369,10 +339,10 @@ func (m *Messenger) RequestAllHistoricMessages(forceFetchingBackup, withRetries filtersByMs := m.SplitFiltersByStoreNode(filters) allResponses := &MessengerResponse{} for communityID, filtersForMs := range filtersByMs { - ms := m.getActiveMailserver(communityID) + peerID := m.getCommunityMailserver(communityID) if withRetries { - response, err := m.performMailserverRequest(ms, func(ms mailservers.Mailserver) (*MessengerResponse, error) { - return m.syncFilters(ms, filtersForMs) + response, err := m.performMailserverRequest(peerID, func() (*MessengerResponse, error) { + return m.syncFilters(peerID, filtersForMs) }) if err != nil { return nil, err @@ -383,7 +353,7 @@ func (m *Messenger) RequestAllHistoricMessages(forceFetchingBackup, withRetries } continue } - response, err := m.syncFilters(*ms, filtersForMs) + response, err := m.syncFilters(peerID, filtersForMs) if err != nil { return nil, err } @@ -401,7 +371,7 @@ func (m *Messenger) checkForMissingMessagesLoop() { t := time.NewTicker(missingMessageCheckPeriod) defer t.Stop() - mailserverAvailableSignal := m.mailserverCycle.availabilitySubscriptions.Subscribe() + mailserverAvailableSignal := m.transport.OnStorenodeAvailable() for { select { @@ -410,7 +380,6 @@ func (m *Messenger) checkForMissingMessagesLoop() { // Wait for mailserver available, also triggered on mailserver change case <-mailserverAvailableSignal: - mailserverAvailableSignal = m.mailserverCycle.availabilitySubscriptions.Subscribe() case <-t.C: @@ -419,16 +388,11 @@ func (m *Messenger) checkForMissingMessagesLoop() { filters := m.transport.Filters() filtersByMs := m.SplitFiltersByStoreNode(filters) for communityID, filtersForMs := range filtersByMs { - ms := m.getActiveMailserver(communityID) - if ms == nil { + peerID := m.getCommunityMailserver(communityID) + if peerID == "" { continue } - peerID, err := ms.PeerID() - if err != nil { - m.logger.Error("could not obtain the peerID") - return - } m.transport.SetCriteriaForMissingMessageVerification(peerID, filtersForMs) } } @@ -438,7 +402,7 @@ func getPrioritizedBatches() []int { return []int{1, 5, 10} } -func (m *Messenger) syncFiltersFrom(ms mailservers.Mailserver, filters []*transport.Filter, lastRequest uint32) (*MessengerResponse, error) { +func (m *Messenger) syncFiltersFrom(peerID peer.ID, filters []*transport.Filter, lastRequest uint32) (*MessengerResponse, error) { canSync, err := m.canSyncWithStoreNodes() if err != nil { return nil, err @@ -458,7 +422,7 @@ func (m *Messenger) syncFiltersFrom(ms mailservers.Mailserver, filters []*transp topicsData[fmt.Sprintf("%s-%s", topic.PubsubTopic, topic.ContentTopic)] = topic } - batches := make(map[string]map[int]MailserverBatch) + batches := make(map[string]map[int]types.MailserverBatch) to := m.calculateMailserverTo() var syncedTopics []mailservers.MailserverTopic @@ -496,7 +460,7 @@ func (m *Messenger) syncFiltersFrom(ms mailservers.Mailserver, filters []*transp for pubsubTopic, contentTopics := range contentTopicsPerPubsubTopic { if _, ok := batches[pubsubTopic]; !ok { - batches[pubsubTopic] = make(map[int]MailserverBatch) + batches[pubsubTopic] = make(map[int]types.MailserverBatch) } for _, filter := range contentTopics { @@ -555,7 +519,7 @@ func (m *Messenger) syncFiltersFrom(ms mailservers.Mailserver, filters []*transp return nil, err } } - batch = MailserverBatch{From: from, To: to} + batch = types.MailserverBatch{From: from, To: to} } batch.ChatIDs = append(batch.ChatIDs, chatID) @@ -573,7 +537,7 @@ func (m *Messenger) syncFiltersFrom(ms mailservers.Mailserver, filters []*transp m.config.messengerSignalsHandler.HistoryRequestStarted(len(batches)) } - var batches24h []MailserverBatch + var batches24h []types.MailserverBatch for pubsubTopic := range batches { batchKeys := make([]int, 0, len(batches[pubsubTopic])) for k := range batches[pubsubTopic] { @@ -588,7 +552,7 @@ func (m *Messenger) syncFiltersFrom(ms mailservers.Mailserver, filters []*transp for _, k := range keysToIterate { batch := batches[pubsubTopic][k] - dayBatch := MailserverBatch{ + dayBatch := types.MailserverBatch{ To: batch.To, Cursor: batch.Cursor, PubsubTopic: batch.PubsubTopic, @@ -618,7 +582,7 @@ func (m *Messenger) syncFiltersFrom(ms mailservers.Mailserver, filters []*transp } for _, batch := range batches24h { - err := m.processMailserverBatch(ms, batch) + err := m.processMailserverBatch(peerID, batch) if err != nil { m.logger.Error("error syncing topics", zap.Error(err)) return nil, err @@ -676,8 +640,8 @@ func (m *Messenger) syncFiltersFrom(ms mailservers.Mailserver, filters []*transp return response, nil } -func (m *Messenger) syncFilters(ms mailservers.Mailserver, filters []*transport.Filter) (*MessengerResponse, error) { - return m.syncFiltersFrom(ms, filters, 0) +func (m *Messenger) syncFilters(peerID peer.ID, filters []*transport.Filter) (*MessengerResponse, error) { + return m.syncFiltersFrom(peerID, filters, 0) } func (m *Messenger) calculateGapForChat(chat *Chat, from uint32) (*common.Message, error) { @@ -716,184 +680,6 @@ func (m *Messenger) calculateGapForChat(chat *Chat, from uint32) (*common.Messag return message, m.persistence.SaveMessages([]*common.Message{message}) } -type work struct { - pubsubTopic string - contentTopics []types.TopicType - cursor types.StoreRequestCursor - limit uint32 -} - -type messageRequester interface { - SendMessagesRequestForTopics( - ctx context.Context, - peerID peer.ID, - from, to uint32, - previousStoreCursor types.StoreRequestCursor, - pubsubTopic string, - contentTopics []types.TopicType, - limit uint32, - waitForResponse bool, - processEnvelopes bool, - ) (cursor types.StoreRequestCursor, envelopesCount int, err error) -} - -func processMailserverBatch( - ctx context.Context, - messageRequester messageRequester, - batch MailserverBatch, - storenodeID peer.ID, - logger *zap.Logger, - pageLimit uint32, - shouldProcessNextPage func(int) (bool, uint32), - processEnvelopes bool, -) error { - - var topicStrings []string - for _, t := range batch.Topics { - topicStrings = append(topicStrings, t.String()) - } - logger = logger.With(zap.Any("chatIDs", batch.ChatIDs), - zap.String("fromString", time.Unix(int64(batch.From), 0).Format(time.RFC3339)), - zap.String("toString", time.Unix(int64(batch.To), 0).Format(time.RFC3339)), - zap.Any("topic", topicStrings), - zap.Int64("from", int64(batch.From)), - zap.Int64("to", int64(batch.To))) - - logger.Info("syncing topic") - - wg := sync.WaitGroup{} - workWg := sync.WaitGroup{} - workCh := make(chan work, 1000) // each batch item is split in 10 topics bunch and sent to this channel - workCompleteCh := make(chan struct{}) // once all batch items are processed, this channel is triggered - semaphore := make(chan int, 3) // limit the number of concurrent queries - errCh := make(chan error) - - ctx, cancel := context.WithCancel(ctx) - defer cancel() - - // Producer - wg.Add(1) - go func() { - defer func() { - logger.Debug("mailserver batch producer complete") - wg.Done() - }() - - allWorks := int(math.Ceil(float64(len(batch.Topics)) / float64(maxTopicsPerRequest))) - workWg.Add(allWorks) - - for i := 0; i < len(batch.Topics); i += maxTopicsPerRequest { - j := i + maxTopicsPerRequest - if j > len(batch.Topics) { - j = len(batch.Topics) - } - - select { - case <-ctx.Done(): - logger.Debug("processBatch producer - context done") - return - default: - logger.Debug("processBatch producer - creating work") - workCh <- work{ - pubsubTopic: batch.PubsubTopic, - contentTopics: batch.Topics[i:j], - limit: pageLimit, - } - time.Sleep(50 * time.Millisecond) - } - } - - go func() { - workWg.Wait() - workCompleteCh <- struct{}{} - }() - - logger.Debug("processBatch producer complete") - }() - - var result error - -loop: - for { - select { - case <-ctx.Done(): - logger.Debug("processBatch cleanup - context done") - result = ctx.Err() - if errors.Is(result, context.Canceled) { - result = nil - } - break loop - case w, ok := <-workCh: - if !ok { - continue - } - - logger.Debug("processBatch - received work") - semaphore <- 1 - go func(w work) { // Consumer - defer func() { - workWg.Done() - <-semaphore - }() - - queryCtx, queryCancel := context.WithTimeout(ctx, mailserverRequestTimeout) - cursor, envelopesCount, err := messageRequester.SendMessagesRequestForTopics(queryCtx, storenodeID, batch.From, batch.To, w.cursor, w.pubsubTopic, w.contentTopics, w.limit, true, processEnvelopes) - queryCancel() - - if err != nil { - logger.Debug("failed to send request", zap.Error(err)) - errCh <- err - return - } - - processNextPage := true - nextPageLimit := pageLimit - - if shouldProcessNextPage != nil { - processNextPage, nextPageLimit = shouldProcessNextPage(envelopesCount) - } - - if !processNextPage { - return - } - - // Check the cursor after calling `shouldProcessNextPage`. - // The app might use process the fetched envelopes in the callback for own needs. - if cursor == nil { - return - } - - logger.Debug("processBatch producer - creating work (cursor)") - - workWg.Add(1) - workCh <- work{ - pubsubTopic: w.pubsubTopic, - contentTopics: w.contentTopics, - cursor: cursor, - limit: nextPageLimit, - } - }(w) - case err := <-errCh: - logger.Debug("processBatch - received error", zap.Error(err)) - cancel() // Kill go routines - return err - case <-workCompleteCh: - logger.Debug("processBatch - all jobs complete") - cancel() // Kill go routines - } - } - - wg.Wait() - - // NOTE(camellos): Disabling for now, not critical and I'd rather take a bit more time - // to test it - //logger.Info("waiting until message processed") - //m.waitUntilP2PMessagesProcessed() - - logger.Info("synced topic", zap.NamedError("hasError", result)) - return result -} - func (m *Messenger) canSyncWithStoreNodes() (bool, error) { if m.featureFlags.StoreNodesDisabled { return false, nil @@ -909,7 +695,7 @@ func (m *Messenger) DisableStoreNodes() { m.featureFlags.StoreNodesDisabled = true } -func (m *Messenger) processMailserverBatch(ms mailservers.Mailserver, batch MailserverBatch) error { +func (m *Messenger) processMailserverBatch(peerID peer.ID, batch types.MailserverBatch) error { canSync, err := m.canSyncWithStoreNodes() if err != nil { return err @@ -918,15 +704,10 @@ func (m *Messenger) processMailserverBatch(ms mailservers.Mailserver, batch Mail return nil } - mailserverID, err := ms.PeerID() - if err != nil { - return err - } - logger := m.logger.With(zap.String("mailserverID", ms.ID)) - return processMailserverBatch(m.ctx, m.transport, batch, mailserverID, logger, defaultStoreNodeRequestPageSize, nil, false) + return m.transport.ProcessMailserverBatch(m.ctx, batch, peerID, defaultStoreNodeRequestPageSize, nil, false) } -func (m *Messenger) processMailserverBatchWithOptions(ms mailservers.Mailserver, batch MailserverBatch, pageLimit uint32, shouldProcessNextPage func(int) (bool, uint32), processEnvelopes bool) error { +func (m *Messenger) processMailserverBatchWithOptions(peerID peer.ID, batch types.MailserverBatch, pageLimit uint64, shouldProcessNextPage func(int) (bool, uint64), processEnvelopes bool) error { canSync, err := m.canSyncWithStoreNodes() if err != nil { return err @@ -935,21 +716,7 @@ func (m *Messenger) processMailserverBatchWithOptions(ms mailservers.Mailserver, return nil } - mailserverID, err := ms.PeerID() - if err != nil { - return err - } - logger := m.logger.With(zap.String("mailserverID", ms.ID)) - return processMailserverBatch(m.ctx, m.transport, batch, mailserverID, logger, pageLimit, shouldProcessNextPage, processEnvelopes) -} - -type MailserverBatch struct { - From uint32 - To uint32 - Cursor string - PubsubTopic string - Topics []types.TopicType - ChatIDs []string + return m.transport.ProcessMailserverBatch(m.ctx, batch, peerID, pageLimit, shouldProcessNextPage, processEnvelopes) } func (m *Messenger) SyncChatFromSyncedFrom(chatID string) (uint32, error) { @@ -958,9 +725,9 @@ func (m *Messenger) SyncChatFromSyncedFrom(chatID string) (uint32, error) { return 0, ErrChatNotFound } - ms := m.getActiveMailserver(chat.CommunityID) + peerID := m.getCommunityMailserver(chat.CommunityID) var from uint32 - _, err := m.performMailserverRequest(ms, func(ms mailservers.Mailserver) (*MessengerResponse, error) { + _, err := m.performMailserverRequest(peerID, func() (*MessengerResponse, error) { canSync, err := m.canSyncWithStoreNodes() if err != nil { return nil, err @@ -979,7 +746,7 @@ func (m *Messenger) SyncChatFromSyncedFrom(chatID string) (uint32, error) { return nil, err } - batch := MailserverBatch{ + batch := types.MailserverBatch{ ChatIDs: []string{chatID}, To: chat.SyncedFrom, From: chat.SyncedFrom - defaultSyncPeriod, @@ -990,7 +757,7 @@ func (m *Messenger) SyncChatFromSyncedFrom(chatID string) (uint32, error) { m.config.messengerSignalsHandler.HistoryRequestStarted(1) } - err = m.processMailserverBatch(ms, batch) + err = m.processMailserverBatch(peerID, batch) if err != nil { return nil, err } @@ -1047,7 +814,7 @@ func (m *Messenger) FillGaps(chatID string, messageIDs []string) error { } } - batch := MailserverBatch{ + batch := types.MailserverBatch{ ChatIDs: []string{chatID}, To: highestTo, From: lowestFrom, @@ -1059,8 +826,8 @@ func (m *Messenger) FillGaps(chatID string, messageIDs []string) error { m.config.messengerSignalsHandler.HistoryRequestStarted(1) } - ms := m.getActiveMailserver(chat.CommunityID) - err = m.processMailserverBatch(*ms, batch) + peerID := m.getCommunityMailserver(chat.CommunityID) + err = m.processMailserverBatch(peerID, batch) if err != nil { return err } @@ -1072,39 +839,18 @@ func (m *Messenger) FillGaps(chatID string, messageIDs []string) error { return m.persistence.DeleteMessages(messageIDs) } -func (m *Messenger) waitUntilP2PMessagesProcessed() { // nolint: unused - - ticker := time.NewTicker(50 * time.Millisecond) - - for { //nolint: gosimple - select { - case <-ticker.C: - if !m.transport.ProcessingP2PMessages() { - ticker.Stop() - return - } - } - } -} - func (m *Messenger) LoadFilters(filters []*transport.Filter) ([]*transport.Filter, error) { return m.transport.LoadFilters(filters) } func (m *Messenger) ToggleUseMailservers(value bool) error { - m.mailserverCycle.Lock() - defer m.mailserverCycle.Unlock() - err := m.settings.SetUseMailservers(value) if err != nil { return err } - m.disconnectActiveMailserver(backoffByUserAction) - if value { - m.cycleMailservers() - return nil - } + m.transport.DisconnectActiveStorenode(m.ctx, backoffByUserAction, value) + return nil } @@ -1114,8 +860,8 @@ func (m *Messenger) SetPinnedMailservers(mailservers map[string]string) error { return err } - m.disconnectActiveMailserver(backoffByUserAction) - m.cycleMailservers() + m.transport.DisconnectActiveStorenode(m.ctx, backoffByUserAction, true) + return nil } @@ -1147,8 +893,8 @@ func (m *Messenger) fetchMessages(chatID string, duration time.Duration) (uint32 return 0, ErrChatNotFound } - ms := m.getActiveMailserver(chat.CommunityID) - _, err := m.performMailserverRequest(ms, func(ms mailservers.Mailserver) (*MessengerResponse, error) { + peerID := m.getCommunityMailserver(chat.CommunityID) + _, err := m.performMailserverRequest(peerID, func() (*MessengerResponse, error) { canSync, err := m.canSyncWithStoreNodes() if err != nil { return nil, err @@ -1157,13 +903,13 @@ func (m *Messenger) fetchMessages(chatID string, duration time.Duration) (uint32 return nil, nil } - m.logger.Debug("fetching messages", zap.String("chatID", chatID), zap.String("mailserver", ms.Name)) + m.logger.Debug("fetching messages", zap.String("chatID", chatID), zap.Stringer("storenodeID", peerID)) pubsubTopic, topics, err := m.topicsForChat(chatID) if err != nil { return nil, nil } - batch := MailserverBatch{ + batch := types.MailserverBatch{ ChatIDs: []string{chatID}, From: from, To: to, @@ -1174,7 +920,7 @@ func (m *Messenger) fetchMessages(chatID string, duration time.Duration) (uint32 m.config.messengerSignalsHandler.HistoryRequestStarted(1) } - err = m.processMailserverBatch(ms, batch) + err = m.processMailserverBatch(peerID, batch) if err != nil { return nil, err } diff --git a/protocol/messenger_mailserver_cycle.go b/protocol/messenger_mailserver_cycle.go index 636d0eab036..a416306f121 100644 --- a/protocol/messenger_mailserver_cycle.go +++ b/protocol/messenger_mailserver_cycle.go @@ -1,176 +1,17 @@ package protocol import ( - "context" - "crypto/rand" - "math" - "math/big" - "net" - "runtime" - "sort" - "sync" - "time" - - "github.com/pkg/errors" + "github.com/libp2p/go-libp2p/core/peer" "go.uber.org/zap" "github.com/waku-org/go-waku/waku/v2/utils" "github.com/status-im/status-go/params" - "github.com/status-im/status-go/protocol/storenodes" "github.com/status-im/status-go/services/mailservers" "github.com/status-im/status-go/signal" ) -const defaultBackoff = 10 * time.Second -const graylistBackoff = 3 * time.Minute -const backoffByUserAction = 0 -const isAndroidEmulator = runtime.GOOS == "android" && runtime.GOARCH == "amd64" -const findNearestMailServer = !isAndroidEmulator -const overrideDNS = runtime.GOOS == "android" || runtime.GOOS == "ios" -const bootstrapDNS = "8.8.8.8:53" - -type byRTTMsAndCanConnectBefore []SortedMailserver - -func (s byRTTMsAndCanConnectBefore) Len() int { - return len(s) -} - -func (s byRTTMsAndCanConnectBefore) Swap(i, j int) { - s[i], s[j] = s[j], s[i] -} - -func (s byRTTMsAndCanConnectBefore) Less(i, j int) bool { - // Slightly inaccurate as time sensitive sorting, but it does not matter so much - now := time.Now() - if s[i].CanConnectAfter.Before(now) && s[j].CanConnectAfter.Before(now) { - return s[i].RTT < s[j].RTT - } - return s[i].CanConnectAfter.Before(s[j].CanConnectAfter) -} - -func (m *Messenger) StartMailserverCycle(mailservers []mailservers.Mailserver) error { - if m.transport.WakuVersion() != 2 { - m.logger.Warn("not starting mailserver cycle: requires wakuv2") - return nil - } - - m.mailserverCycle.allMailservers = mailservers - - if len(mailservers) == 0 { - m.logger.Warn("not starting mailserver cycle: empty mailservers list") - return nil - } - - for _, storenode := range mailservers { - - peerInfo, err := storenode.PeerInfo() - if err != nil { - return err - } - - for _, addr := range utils.EncapsulatePeerID(peerInfo.ID, peerInfo.Addrs...) { - _, err := m.transport.AddStorePeer(addr) - if err != nil { - return err - } - } - } - go m.verifyStorenodeStatus() - - m.logger.Debug("starting mailserver cycle", - zap.Uint("WakuVersion", m.transport.WakuVersion()), - zap.Any("mailservers", mailservers), - ) - - return nil -} - -func (m *Messenger) DisconnectActiveMailserver() { - m.mailserverCycle.Lock() - defer m.mailserverCycle.Unlock() - m.disconnectActiveMailserver(graylistBackoff) -} - -func (m *Messenger) disconnectMailserver(backoffDuration time.Duration) error { - if m.mailserverCycle.activeMailserver == nil { - m.logger.Info("no active mailserver") - return nil - } - m.logger.Info("disconnecting active mailserver", zap.String("nodeID", m.mailserverCycle.activeMailserver.ID)) - m.mailPeersMutex.Lock() - pInfo, ok := m.mailserverCycle.peers[m.mailserverCycle.activeMailserver.ID] - if ok { - pInfo.status = disconnected - - pInfo.canConnectAfter = time.Now().Add(backoffDuration) - m.mailserverCycle.peers[m.mailserverCycle.activeMailserver.ID] = pInfo - } else { - m.mailserverCycle.peers[m.mailserverCycle.activeMailserver.ID] = peerStatus{ - status: disconnected, - mailserver: *m.mailserverCycle.activeMailserver, - canConnectAfter: time.Now().Add(backoffDuration), - } - } - m.mailPeersMutex.Unlock() - - m.mailserverCycle.activeMailserver = nil - return nil -} - -func (m *Messenger) disconnectActiveMailserver(backoffDuration time.Duration) { - err := m.disconnectMailserver(backoffDuration) - if err != nil { - m.logger.Error("failed to disconnect mailserver", zap.Error(err)) - } - signal.SendMailserverChanged(nil) -} - -func (m *Messenger) cycleMailservers() { - m.logger.Info("Automatically switching mailserver") - - if m.mailserverCycle.activeMailserver != nil { - m.disconnectActiveMailserver(graylistBackoff) - } - - useMailserver, err := m.settings.CanUseMailservers() - if err != nil { - m.logger.Error("failed to get use mailservers", zap.Error(err)) - return - } - - if !useMailserver { - m.logger.Info("Skipping mailserver search due to useMailserver being false") - return - } - - err = m.findNewMailserver() - if err != nil { - m.logger.Error("Error getting new mailserver", zap.Error(err)) - } -} - -func poolSize(fleetSize int) int { - return int(math.Ceil(float64(fleetSize) / 4)) -} - -func (m *Messenger) getFleet() (string, error) { - var fleet string - dbFleet, err := m.settings.GetFleet() - if err != nil { - return "", err - } - if dbFleet != "" { - fleet = dbFleet - } else if m.config.clusterConfig.Fleet != "" { - fleet = m.config.clusterConfig.Fleet - } else { - fleet = params.FleetStatusProd - } - return fleet, nil -} - -func (m *Messenger) allMailservers() ([]mailservers.Mailserver, error) { +func (m *Messenger) AllMailservers() ([]mailservers.Mailserver, error) { // Get configured fleet fleet, err := m.getFleet() if err != nil { @@ -197,216 +38,38 @@ func (m *Messenger) allMailservers() ([]mailservers.Mailserver, error) { return allMailservers, nil } -type SortedMailserver struct { - Mailserver mailservers.Mailserver - RTT time.Duration - CanConnectAfter time.Time -} - -func (m *Messenger) getAvailableMailserversSortedByRTT(allMailservers []mailservers.Mailserver) []mailservers.Mailserver { - // TODO: this can be replaced by peer selector once code is moved to go-waku api - availableMailservers := make(map[string]time.Duration) - availableMailserversMutex := sync.Mutex{} - availableMailserversWg := sync.WaitGroup{} - for _, mailserver := range allMailservers { - availableMailserversWg.Add(1) - go func(mailserver mailservers.Mailserver) { - defer availableMailserversWg.Done() - - peerID, err := mailserver.PeerID() - if err != nil { - return - } - - ctx, cancel := context.WithTimeout(m.ctx, 4*time.Second) - defer cancel() +func (m *Messenger) setupStorenodes(storenodes []mailservers.Mailserver) error { + for _, storenode := range storenodes { - rtt, err := m.transport.PingPeer(ctx, peerID) - if err == nil { // pinging mailservers might fail, but we don't care - availableMailserversMutex.Lock() - availableMailservers[mailserver.ID] = rtt - availableMailserversMutex.Unlock() - } - }(mailserver) - } - availableMailserversWg.Wait() - - if len(availableMailservers) == 0 { - m.logger.Warn("No mailservers available") // Do nothing... - return nil - } - - mailserversByID := make(map[string]mailservers.Mailserver) - for idx := range allMailservers { - mailserversByID[allMailservers[idx].ID] = allMailservers[idx] - } - var sortedMailservers []SortedMailserver - for mailserverID, rtt := range availableMailservers { - ms := mailserversByID[mailserverID] - sortedMailserver := SortedMailserver{ - Mailserver: ms, - RTT: rtt, - } - m.mailPeersMutex.Lock() - pInfo, ok := m.mailserverCycle.peers[ms.ID] - m.mailPeersMutex.Unlock() - if ok { - if time.Now().Before(pInfo.canConnectAfter) { - continue // We can't connect to this node yet - } - } - sortedMailservers = append(sortedMailservers, sortedMailserver) - } - sort.Sort(byRTTMsAndCanConnectBefore(sortedMailservers)) - - result := make([]mailservers.Mailserver, len(sortedMailservers)) - for i, s := range sortedMailservers { - result[i] = s.Mailserver - } - - return result -} - -func (m *Messenger) findNewMailserver() error { - // we have to override DNS manually because of https://github.com/status-im/status-mobile/issues/19581 - if overrideDNS { - var dialer net.Dialer - net.DefaultResolver = &net.Resolver{ - PreferGo: false, - Dial: func(context context.Context, _, _ string) (net.Conn, error) { - conn, err := dialer.DialContext(context, "udp", bootstrapDNS) - if err != nil { - return nil, err - } - return conn, nil - }, - } - } - - pinnedMailserver, err := m.getPinnedMailserver() - if err != nil { - m.logger.Error("Could not obtain the pinned mailserver", zap.Error(err)) - return err - } - if pinnedMailserver != nil { - return m.connectToMailserver(*pinnedMailserver) - } - - m.logger.Info("Finding a new mailserver...") - - allMailservers := m.mailserverCycle.allMailservers - - // TODO: remove this check once sockets are stable on x86_64 emulators - if findNearestMailServer { - allMailservers = m.getAvailableMailserversSortedByRTT(allMailservers) - } - - // Picks a random mailserver amongs the ones with the lowest latency - // The pool size is 1/4 of the mailservers were pinged successfully - pSize := poolSize(len(allMailservers) - 1) - if pSize <= 0 { - pSize = len(allMailservers) - if pSize <= 0 { - m.logger.Warn("No storenodes available") // Do nothing... - return nil - } - } - - r, err := rand.Int(rand.Reader, big.NewInt(int64(pSize))) - if err != nil { - return err - } - - ms := allMailservers[r.Int64()] - return m.connectToMailserver(ms) -} - -func (m *Messenger) mailserverStatus(mailserverID string) connStatus { - m.mailPeersMutex.RLock() - defer m.mailPeersMutex.RUnlock() - peer, ok := m.mailserverCycle.peers[mailserverID] - if !ok { - return disconnected - } - return peer.status -} - -func (m *Messenger) connectToMailserver(ms mailservers.Mailserver) error { - - m.logger.Info("connecting to mailserver", zap.String("mailserverID", ms.ID)) - - m.mailserverCycle.activeMailserver = &ms - signal.SendMailserverChanged(m.mailserverCycle.activeMailserver) - - mailserverStatus := m.mailserverStatus(ms.ID) - if mailserverStatus != connected { - m.mailPeersMutex.Lock() - m.mailserverCycle.peers[ms.ID] = peerStatus{ - status: connected, - lastConnectionAttempt: time.Now(), - canConnectAfter: time.Now().Add(defaultBackoff), - mailserver: ms, - } - m.mailPeersMutex.Unlock() - - m.mailserverCycle.activeMailserver.FailedRequests = 0 - peerID, err := m.mailserverCycle.activeMailserver.PeerID() + peerInfo, err := storenode.PeerInfo() if err != nil { - m.logger.Error("could not decode the peer id of mailserver", zap.Error(err)) return err } - m.logger.Info("mailserver available", zap.String("mailserverID", m.mailserverCycle.activeMailserver.ID)) - m.mailserverCycle.availabilitySubscriptions.EmitMailserverAvailable() - signal.SendMailserverAvailable(m.mailserverCycle.activeMailserver) - - m.transport.SetStorePeerID(peerID) - - // Query mailserver - m.asyncRequestAllHistoricMessages() + for _, addr := range utils.EncapsulatePeerID(peerInfo.ID, peerInfo.Addrs...) { + _, err := m.transport.AddStorePeer(addr) + if err != nil { + return err + } + } } return nil } -// getActiveMailserver returns the active mailserver if a communityID is present then it'll return the mailserver -// for that community if it has a mailserver setup otherwise it'll return the global mailserver -func (m *Messenger) getActiveMailserver(communityID ...string) *mailservers.Mailserver { - if len(communityID) == 0 || communityID[0] == "" { - return m.mailserverCycle.activeMailserver - } - ms, err := m.communityStorenodes.GetStorenodeByCommunityID(communityID[0]) +func (m *Messenger) getFleet() (string, error) { + var fleet string + dbFleet, err := m.settings.GetFleet() if err != nil { - if !errors.Is(err, storenodes.ErrNotFound) { - m.logger.Error("getting storenode for community, using global", zap.String("communityID", communityID[0]), zap.Error(err)) - } - // if we don't find a specific mailserver for the community, we just use the regular mailserverCycle's one - return m.mailserverCycle.activeMailserver - } - return &ms -} - -func (m *Messenger) getActiveMailserverID(communityID ...string) string { - ms := m.getActiveMailserver(communityID...) - if ms == nil { - return "" + return "", err } - return ms.ID -} - -func (m *Messenger) isMailserverAvailable(mailserverID string) bool { - return m.mailserverStatus(mailserverID) == connected -} - -func (m *Messenger) penalizeMailserver(id string) { - m.mailPeersMutex.Lock() - defer m.mailPeersMutex.Unlock() - pInfo, ok := m.mailserverCycle.peers[id] - if !ok { - pInfo.status = disconnected + if dbFleet != "" { + fleet = dbFleet + } else if m.config.clusterConfig.Fleet != "" { + fleet = m.config.clusterConfig.Fleet + } else { + fleet = params.FleetStatusProd } - - pInfo.canConnectAfter = time.Now().Add(graylistBackoff) - m.mailserverCycle.peers[id] = pInfo + return fleet, nil } func (m *Messenger) asyncRequestAllHistoricMessages() { @@ -424,125 +87,113 @@ func (m *Messenger) asyncRequestAllHistoricMessages() { }() } -func (m *Messenger) verifyStorenodeStatus() { - ticker := time.NewTicker(1 * time.Second) - defer ticker.Stop() - - for { - select { - case <-ticker.C: - err := m.disconnectStorenodeIfRequired() - if err != nil { - m.logger.Error("failed to handle mailserver cycle event", zap.Error(err)) - continue - } - - case <-m.quit: - return - } - } -} - -func (m *Messenger) getPinnedMailserver() (*mailservers.Mailserver, error) { +func (m *Messenger) GetPinnedStorenode() (peer.ID, error) { fleet, err := m.getFleet() if err != nil { - return nil, err + return "", err } pinnedMailservers, err := m.settings.GetPinnedMailservers() if err != nil { - return nil, err + return "", err } pinnedMailserver, ok := pinnedMailservers[fleet] if !ok { - return nil, nil + return "", nil } fleetMailservers := mailservers.DefaultMailservers() for _, c := range fleetMailservers { if c.Fleet == fleet && c.ID == pinnedMailserver { - return &c, nil + return c.PeerID() } } if m.mailserversDatabase != nil { customMailservers, err := m.mailserversDatabase.Mailservers() if err != nil { - return nil, err + return "", err } for _, c := range customMailservers { if c.Fleet == fleet && c.ID == pinnedMailserver { - return &c, nil + return c.PeerID() } } } - return nil, nil + return "", nil } -func (m *Messenger) disconnectStorenodeIfRequired() error { - m.logger.Debug("wakuV2 storenode status verification") +func (m *Messenger) UseStorenodes() (bool, error) { + return m.settings.CanUseMailservers() +} - if m.mailserverCycle.activeMailserver == nil { - // No active storenode, find a new one - m.cycleMailservers() - return nil +func (m *Messenger) Storenodes() ([]peer.ID, error) { + mailservers, err := m.AllMailservers() + if err != nil { + return nil, err } - // Check whether we want to disconnect the active storenode - if m.mailserverCycle.activeMailserver.FailedRequests >= mailserverMaxFailedRequests { - m.penalizeMailserver(m.mailserverCycle.activeMailserver.ID) - signal.SendMailserverNotWorking() - m.logger.Info("too many failed requests", zap.String("storenode", m.mailserverCycle.activeMailserver.ID)) - m.mailserverCycle.activeMailserver.FailedRequests = 0 - return m.connectToNewMailserverAndWait() + var result []peer.ID + for _, m := range mailservers { + peerID, err := m.PeerID() + if err != nil { + return nil, err + } + result = append(result, peerID) } - return nil + return result, nil } -func (m *Messenger) waitForAvailableStoreNode(timeout time.Duration) bool { - // Add 1 second to timeout, because the mailserver cycle has 1 second ticker, which doesn't tick on start. - // This can be improved after merging https://github.com/status-im/status-go/pull/4380. - // NOTE: https://stackoverflow.com/questions/32705582/how-to-get-time-tick-to-tick-immediately - timeout += time.Second +func (m *Messenger) checkForStorenodeCycleSignals() { + changed := m.transport.OnStorenodeChanged() + notWorking := m.transport.OnStorenodeNotWorking() + available := m.transport.OnStorenodeAvailable() - finish := make(chan struct{}) - cancel := make(chan struct{}) + allMailservers, err := m.AllMailservers() + if err != nil { + m.logger.Error("Could not retrieve mailserver list", zap.Error(err)) + return + } - wg := sync.WaitGroup{} - wg.Add(1) + mailserverMap := make(map[peer.ID]mailservers.Mailserver) + for _, ms := range allMailservers { + peerID, err := ms.PeerID() + if err != nil { + m.logger.Error("could not retrieve peerID", zap.Error(err)) + return + } + mailserverMap[peerID] = ms + } - go func() { - defer func() { - wg.Done() - }() - for !m.isMailserverAvailable(m.getActiveMailserverID()) { - select { - case <-m.mailserverCycle.availabilitySubscriptions.Subscribe(): - case <-cancel: - return + for { + select { + case <-m.ctx.Done(): + return + case <-notWorking: + signal.SendMailserverNotWorking() + + case activeMailserver := <-changed: + if activeMailserver != "" { + ms, ok := mailserverMap[activeMailserver] + if ok { + signal.SendMailserverChanged(&ms) + } + } else { + signal.SendMailserverChanged(nil) + } + case activeMailserver := <-available: + if activeMailserver != "" { + ms, ok := mailserverMap[activeMailserver] + if ok { + signal.SendMailserverAvailable(&ms) + } + m.asyncRequestAllHistoricMessages() } } - }() - - go func() { - defer func() { - close(finish) - }() - wg.Wait() - }() - - select { - case <-finish: - case <-time.After(timeout): - close(cancel) - case <-m.ctx.Done(): - close(cancel) } - - return m.isMailserverAvailable(m.getActiveMailserverID()) } diff --git a/protocol/messenger_mailserver_processMailserverBatch_test.go b/protocol/messenger_mailserver_processMailserverBatch_test.go deleted file mode 100644 index 71ea759a409..00000000000 --- a/protocol/messenger_mailserver_processMailserverBatch_test.go +++ /dev/null @@ -1,167 +0,0 @@ -package protocol - -import ( - "context" - "crypto/rand" - "encoding/hex" - "errors" - "math/big" - "testing" - "time" - - "github.com/google/uuid" - "github.com/libp2p/go-libp2p/core/peer" - "github.com/stretchr/testify/require" - - "github.com/status-im/status-go/eth-node/types" - "github.com/status-im/status-go/protocol/tt" -) - -type queryResponse struct { - topics []types.TopicType - err error // Indicates if this response will simulate an error returned by SendMessagesRequestForTopics - cursor []byte -} - -type mockTransport struct { - queryResponses map[string]queryResponse -} - -func newMockTransport() *mockTransport { - return &mockTransport{ - queryResponses: make(map[string]queryResponse), - } -} - -func getInitialResponseKey(topics []types.TopicType) string { - return hex.EncodeToString(append([]byte("start"), topics[0][:]...)) -} - -func (t *mockTransport) SendMessagesRequestForTopics( - ctx context.Context, - peerID peer.ID, - from, to uint32, - prevCursor types.StoreRequestCursor, - pubsubTopic string, - contentTopics []types.TopicType, - limit uint32, - waitForResponse bool, - processEnvelopes bool, -) (cursor types.StoreRequestCursor, envelopesCount int, err error) { - var response queryResponse - if prevCursor == nil { - initialResponse := getInitialResponseKey(contentTopics) - response = t.queryResponses[initialResponse] - } else { - response = t.queryResponses[hex.EncodeToString(prevCursor)] - } - return response.cursor, 0, response.err -} - -func (t *mockTransport) Populate(topics []types.TopicType, responses int, includeRandomError bool) error { - if responses <= 0 || len(topics) == 0 { - return errors.New("invalid input parameters") - } - - var topicBatches [][]types.TopicType - - for i := 0; i < len(topics); i += maxTopicsPerRequest { - // Split batch in 10-contentTopic subbatches - j := i + maxTopicsPerRequest - if j > len(topics) { - j = len(topics) - } - topicBatches = append(topicBatches, topics[i:j]) - } - - randomErrIdx, err := rand.Int(rand.Reader, big.NewInt(int64(len(topicBatches)))) - if err != nil { - return err - } - randomErrIdxInt := int(randomErrIdx.Int64()) - - for i, topicBatch := range topicBatches { - // Setup initial response - initialResponseKey := getInitialResponseKey(topicBatch) - t.queryResponses[initialResponseKey] = queryResponse{ - topics: topicBatch, - err: nil, - } - - prevKey := initialResponseKey - for x := 0; x < responses-1; x++ { - newResponseCursor := []byte(uuid.New().String()) - newResponseKey := hex.EncodeToString(newResponseCursor) - - var err error - if includeRandomError && i == randomErrIdxInt && x == responses-2 { // Include an error in last request - err = errors.New("random error") - } - - t.queryResponses[newResponseKey] = queryResponse{ - topics: topicBatch, - err: err, - } - - // Updating prev response cursor to point to the new response - prevResponse := t.queryResponses[prevKey] - prevResponse.cursor = newResponseCursor - t.queryResponses[prevKey] = prevResponse - - prevKey = newResponseKey - } - - } - - return nil -} - -func TestProcessMailserverBatchHappyPath(t *testing.T) { - ctx, cancel := context.WithTimeout(context.TODO(), 5*time.Second) - defer cancel() - - logger := tt.MustCreateTestLogger() - - mailserverID, err := peer.Decode("16Uiu2HAkw3x97MbbZSWHbdF5bob45vcZvPPK4s4Mjyv2mxyB9GS3") - require.NoError(t, err) - topics := []types.TopicType{} - for i := 0; i < 22; i++ { - topics = append(topics, types.BytesToTopic([]byte{0, 0, 0, byte(i)})) - } - - testTransport := newMockTransport() - err = testTransport.Populate(topics, 10, false) - require.NoError(t, err) - - testBatch := MailserverBatch{ - Topics: topics, - } - - err = processMailserverBatch(ctx, testTransport, testBatch, mailserverID, logger, defaultStoreNodeRequestPageSize, nil, false) - require.NoError(t, err) -} - -func TestProcessMailserverBatchFailure(t *testing.T) { - ctx, cancel := context.WithTimeout(context.TODO(), 5*time.Second) - defer cancel() - - logger := tt.MustCreateTestLogger() - - mailserverID, err := peer.Decode("16Uiu2HAkw3x97MbbZSWHbdF5bob45vcZvPPK4s4Mjyv2mxyB9GS3") - require.NoError(t, err) - topics := []types.TopicType{} - for i := 0; i < 5; i++ { - topics = append(topics, types.BytesToTopic([]byte{0, 0, 0, byte(i)})) - } - - testTransport := newMockTransport() - err = testTransport.Populate(topics, 4, true) - require.NoError(t, err) - - testBatch := MailserverBatch{ - Topics: topics, - } - - err = processMailserverBatch(ctx, testTransport, testBatch, mailserverID, logger, defaultStoreNodeRequestPageSize, nil, false) - require.Error(t, err) -} diff --git a/protocol/messenger_store_node_request_manager.go b/protocol/messenger_store_node_request_manager.go index e5a67e1af8b..e0c5b55ea26 100644 --- a/protocol/messenger_store_node_request_manager.go +++ b/protocol/messenger_store_node_request_manager.go @@ -15,7 +15,6 @@ import ( "github.com/status-im/status-go/eth-node/types" "github.com/status-im/status-go/protocol/communities" "github.com/status-im/status-go/protocol/transport" - "github.com/status-im/status-go/services/mailservers" ) const ( @@ -56,7 +55,7 @@ type StoreNodeRequestManager struct { // activeRequestsLock should be locked each time activeRequests is being accessed or changed. activeRequestsLock sync.RWMutex - onPerformingBatch func(MailserverBatch) + onPerformingBatch func(types.MailserverBatch) } func NewStoreNodeRequestManager(m *Messenger) *StoreNodeRequestManager { @@ -372,7 +371,7 @@ func (r *storeNodeRequest) finalize() { } } -func (r *storeNodeRequest) shouldFetchNextPage(envelopesCount int) (bool, uint32) { +func (r *storeNodeRequest) shouldFetchNextPage(envelopesCount int) (bool, uint64) { logger := r.manager.logger.With( zap.Any("requestID", r.requestID), zap.Int("envelopesCount", envelopesCount)) @@ -520,13 +519,13 @@ func (r *storeNodeRequest) routine() { communityID := r.requestID.getCommunityID() if r.requestID.RequestType != storeNodeCommunityRequest || !r.manager.messenger.communityStorenodes.HasStorenodeSetup(communityID) { - if !r.manager.messenger.waitForAvailableStoreNode(storeNodeAvailableTimeout) { + if !r.manager.messenger.transport.WaitForAvailableStoreNode(storeNodeAvailableTimeout) { r.result.err = fmt.Errorf("store node is not available") return } } - storeNode := r.manager.messenger.getActiveMailserver(communityID) + storeNode := r.manager.messenger.getCommunityMailserver(communityID) // Check if community already exists locally and get Clock. if r.requestID.RequestType == storeNodeCommunityRequest { @@ -539,8 +538,8 @@ func (r *storeNodeRequest) routine() { // Start store node request from, to := r.manager.messenger.calculateMailserverTimeBounds(oneMonthDuration) - _, err := r.manager.messenger.performMailserverRequest(storeNode, func(ms mailservers.Mailserver) (*MessengerResponse, error) { - batch := MailserverBatch{ + _, err := r.manager.messenger.performMailserverRequest(storeNode, func() (*MessengerResponse, error) { + batch := types.MailserverBatch{ From: from, To: to, PubsubTopic: r.pubsubTopic, @@ -551,7 +550,7 @@ func (r *storeNodeRequest) routine() { r.manager.onPerformingBatch(batch) } - return nil, r.manager.messenger.processMailserverBatchWithOptions(ms, batch, r.config.InitialPageSize, r.shouldFetchNextPage, true) + return nil, r.manager.messenger.processMailserverBatchWithOptions(storeNode, batch, r.config.InitialPageSize, r.shouldFetchNextPage, true) }) r.result.err = err diff --git a/protocol/messenger_store_node_request_manager_config.go b/protocol/messenger_store_node_request_manager_config.go index cffcc76d70e..56faf835fdb 100644 --- a/protocol/messenger_store_node_request_manager_config.go +++ b/protocol/messenger_store_node_request_manager_config.go @@ -3,8 +3,8 @@ package protocol type StoreNodeRequestConfig struct { WaitForResponse bool StopWhenDataFound bool - InitialPageSize uint32 - FurtherPageSize uint32 + InitialPageSize uint64 + FurtherPageSize uint64 } type StoreNodeRequestOption func(*StoreNodeRequestConfig) @@ -40,13 +40,13 @@ func WithStopWhenDataFound(stopWhenDataFound bool) StoreNodeRequestOption { } } -func WithInitialPageSize(initialPageSize uint32) StoreNodeRequestOption { +func WithInitialPageSize(initialPageSize uint64) StoreNodeRequestOption { return func(c *StoreNodeRequestConfig) { c.InitialPageSize = initialPageSize } } -func WithFurtherPageSize(furtherPageSize uint32) StoreNodeRequestOption { +func WithFurtherPageSize(furtherPageSize uint64) StoreNodeRequestOption { return func(c *StoreNodeRequestConfig) { c.FurtherPageSize = furtherPageSize } diff --git a/protocol/messenger_storenode_comunity_test.go b/protocol/messenger_storenode_comunity_test.go index 746951413fe..51cbf712ff4 100644 --- a/protocol/messenger_storenode_comunity_test.go +++ b/protocol/messenger_storenode_comunity_test.go @@ -351,10 +351,10 @@ func (s *MessengerStoreNodeCommunitySuite) TestToggleUseMailservers() { // Enable use of mailservers err := s.owner.ToggleUseMailservers(true) s.Require().NoError(err) - s.Require().NotNil(s.owner.mailserverCycle.activeMailserver) + s.Require().NotNil(s.owner.transport.GetActiveStorenode()) // Disable use of mailservers err = s.owner.ToggleUseMailservers(false) s.Require().NoError(err) - s.Require().Nil(s.owner.mailserverCycle.activeMailserver) + s.Require().Nil(s.owner.transport.GetActiveStorenode()) } diff --git a/protocol/messenger_storenode_request_test.go b/protocol/messenger_storenode_request_test.go index 543e32fb388..a2e2bf1775d 100644 --- a/protocol/messenger_storenode_request_test.go +++ b/protocol/messenger_storenode_request_test.go @@ -235,7 +235,7 @@ func (s *MessengerStoreNodeRequestSuite) newMessenger(shh types.Waku, logger *za } func (s *MessengerStoreNodeRequestSuite) createCommunity(m *Messenger) *communities.Community { - s.waitForAvailableStoreNode(m) + s.WaitForAvailableStoreNode(m) storeNodeSubscription := s.setupStoreNodeEnvelopesWatcher(nil) @@ -309,7 +309,7 @@ func (s *MessengerStoreNodeRequestSuite) fetchProfile(m *Messenger, contactID st } } -func (s *MessengerStoreNodeRequestSuite) waitForAvailableStoreNode(messenger *Messenger) { +func (s *MessengerStoreNodeRequestSuite) WaitForAvailableStoreNode(messenger *Messenger) { WaitForAvailableStoreNode(&s.Suite, messenger, storeNodeConnectTimeout) } @@ -419,11 +419,11 @@ func (s *MessengerStoreNodeRequestSuite) TestSimultaneousCommunityInfoRequests() community := s.createCommunity(s.owner) storeNodeRequestsCount := 0 - s.bob.storeNodeRequestsManager.onPerformingBatch = func(batch MailserverBatch) { + s.bob.storeNodeRequestsManager.onPerformingBatch = func(batch types.MailserverBatch) { storeNodeRequestsCount++ } - s.waitForAvailableStoreNode(s.bob) + s.WaitForAvailableStoreNode(s.bob) wg := sync.WaitGroup{} @@ -453,7 +453,7 @@ func (s *MessengerStoreNodeRequestSuite) TestRequestNonExistentCommunity() { s.createBob() - s.waitForAvailableStoreNode(s.bob) + s.WaitForAvailableStoreNode(s.bob) fetchedCommunity, err := s.bob.FetchCommunity(&request) s.Require().NoError(err) @@ -722,7 +722,7 @@ func (s *MessengerStoreNodeRequestSuite) TestRequestShardAndCommunityInfo() { s.waitForEnvelopes(storeNodeSubscription, 1) - s.waitForAvailableStoreNode(s.bob) + s.WaitForAvailableStoreNode(s.bob) communityShard := community.CommunityShard() @@ -1192,7 +1192,7 @@ func (s *MessengerStoreNodeRequestSuite) TestFetchingCommunityWithOwnerToken() { s.createOwner() s.createBob() - s.waitForAvailableStoreNode(s.owner) + s.WaitForAvailableStoreNode(s.owner) community := s.createCommunity(s.owner) // owner mints owner token @@ -1225,7 +1225,7 @@ func (s *MessengerStoreNodeRequestSuite) TestFetchingCommunityWithOwnerToken() { s.Require().NoError(err) s.Require().Len(community.TokenPermissions(), 1) - s.waitForAvailableStoreNode(s.bob) + s.WaitForAvailableStoreNode(s.bob) s.fetchCommunity(s.bob, community.CommunityShard(), community) } diff --git a/protocol/messenger_testing_utils.go b/protocol/messenger_testing_utils.go index 906d04346ff..536ffbb0a5f 100644 --- a/protocol/messenger_testing_utils.go +++ b/protocol/messenger_testing_utils.go @@ -362,7 +362,7 @@ func SetIdentityImagesAndWaitForChange(s *suite.Suite, messenger *Messenger, tim } func WaitForAvailableStoreNode(s *suite.Suite, m *Messenger, timeout time.Duration) { - available := m.waitForAvailableStoreNode(timeout) + available := m.transport.WaitForAvailableStoreNode(timeout) s.Require().True(available) } diff --git a/protocol/storenodes/storenodes.go b/protocol/storenodes/storenodes.go index 66a6f4bc35a..4559df80e1c 100644 --- a/protocol/storenodes/storenodes.go +++ b/protocol/storenodes/storenodes.go @@ -6,6 +6,9 @@ import ( "go.uber.org/zap" + "github.com/libp2p/go-libp2p/core/peer" + "github.com/waku-org/go-waku/waku/v2/utils" + "github.com/status-im/status-go/eth-node/types" "github.com/status-im/status-go/services/mailservers" ) @@ -51,14 +54,17 @@ func (m *CommunityStorenodes) GetStorenodeByCommunityID(communityID string) (mai return toMailserver(msData.storenodes[0]), nil } -func (m *CommunityStorenodes) IsCommunityStoreNode(id string) bool { +func (m *CommunityStorenodes) IsCommunityStoreNode(peerID peer.ID) bool { m.storenodesByCommunityIDMutex.RLock() defer m.storenodesByCommunityIDMutex.RUnlock() for _, data := range m.storenodesByCommunityID { for _, snode := range data.storenodes { - if snode.StorenodeID == id { - return true + commStorenodeID, err := utils.GetPeerID(snode.Address) + if err == nil { + if commStorenodeID == peerID { + return true + } } } } diff --git a/protocol/transport/transport.go b/protocol/transport/transport.go index 2d873a1759e..ef7edb03117 100644 --- a/protocol/transport/transport.go +++ b/protocol/transport/transport.go @@ -4,17 +4,17 @@ import ( "context" "crypto/ecdsa" "database/sql" - "encoding/hex" "sync" "time" - "github.com/google/uuid" "github.com/libp2p/go-libp2p/core/peer" "github.com/multiformats/go-multiaddr" "github.com/pkg/errors" "go.uber.org/zap" "golang.org/x/exp/maps" + "github.com/waku-org/go-waku/waku/v2/api/history" + "github.com/ethereum/go-ethereum/common" "github.com/ethereum/go-ethereum/p2p/enode" "github.com/status-im/status-go/connection" @@ -460,87 +460,6 @@ func (t *Transport) Peers() types.PeerStats { return t.waku.Peers() } -func (t *Transport) createMessagesRequest( - ctx context.Context, - peerID peer.ID, - from, to uint32, - previousStoreCursor types.StoreRequestCursor, - pubsubTopic string, - contentTopics []types.TopicType, - limit uint32, - waitForResponse bool, - processEnvelopes bool, -) (storeCursor types.StoreRequestCursor, envelopesCount int, err error) { - r := createMessagesRequest(from, to, nil, previousStoreCursor, pubsubTopic, contentTopics, limit) - - if waitForResponse { - resultCh := make(chan struct { - storeCursor types.StoreRequestCursor - envelopesCount int - err error - }) - - go func() { - storeCursor, envelopesCount, err = t.waku.RequestStoreMessages(ctx, peerID, r, processEnvelopes) - resultCh <- struct { - storeCursor types.StoreRequestCursor - envelopesCount int - err error - }{storeCursor, envelopesCount, err} - }() - - select { - case result := <-resultCh: - return result.storeCursor, result.envelopesCount, result.err - case <-ctx.Done(): - return nil, 0, ctx.Err() - } - } else { - go func() { - _, _, err = t.waku.RequestStoreMessages(ctx, peerID, r, false) - if err != nil { - t.logger.Error("failed to request store messages", zap.Error(err)) - } - }() - } - - return -} - -func (t *Transport) SendMessagesRequestForTopics( - ctx context.Context, - peerID peer.ID, - from, to uint32, - prevCursor types.StoreRequestCursor, - pubsubTopic string, - contentTopics []types.TopicType, - limit uint32, - waitForResponse bool, - processEnvelopes bool, -) (cursor types.StoreRequestCursor, envelopesCount int, err error) { - return t.createMessagesRequest(ctx, peerID, from, to, prevCursor, pubsubTopic, contentTopics, limit, waitForResponse, processEnvelopes) -} - -func createMessagesRequest(from, to uint32, cursor []byte, storeCursor types.StoreRequestCursor, pubsubTopic string, topics []types.TopicType, limit uint32) types.MessagesRequest { - aUUID := uuid.New() - // uuid is 16 bytes, converted to hex it's 32 bytes as expected by types.MessagesRequest - id := []byte(hex.EncodeToString(aUUID[:])) - var topicBytes [][]byte - for idx := range topics { - topicBytes = append(topicBytes, topics[idx][:]) - } - return types.MessagesRequest{ - ID: id, - From: from, - To: to, - Limit: limit, - Cursor: cursor, - PubsubTopic: pubsubTopic, - ContentTopics: topicBytes, - StoreCursor: storeCursor, - } -} - // ConfirmMessagesProcessed marks the messages as processed in the cache so // they won't be passed to the next layer anymore func (t *Transport) ConfirmMessagesProcessed(ids []string, timestamp uint64) error { @@ -631,10 +550,6 @@ func (t *Transport) ConnectionChanged(state connection.State) { t.waku.ConnectionChanged(state) } -func (t *Transport) PingPeer(ctx context.Context, peerID peer.ID) (time.Duration, error) { - return t.waku.PingPeer(ctx, peerID) -} - // Subscribe to a pubsub topic, passing an optional public key if the pubsub topic is protected func (t *Transport) SubscribeToPubsubTopic(topic string, optPublicKey *ecdsa.PublicKey) error { if t.waku.Version() == 2 { @@ -681,10 +596,6 @@ func (t *Transport) ConfirmMessageDelivered(messageID string) { t.waku.ConfirmMessageDelivered(commHashes) } -func (t *Transport) SetStorePeerID(peerID peer.ID) { - t.waku.SetStorePeerID(peerID) -} - func (t *Transport) SetCriteriaForMissingMessageVerification(peerID peer.ID, filters []*Filter) { if t.waku.Version() != 2 { return @@ -717,3 +628,54 @@ func (t *Transport) SetCriteriaForMissingMessageVerification(peerID peer.ID, fil } } } + +func (t *Transport) GetActiveStorenode() peer.ID { + return t.waku.GetActiveStorenode() +} + +func (t *Transport) DisconnectActiveStorenode(ctx context.Context, backoffReason time.Duration, shouldCycle bool) { + t.waku.DisconnectActiveStorenode(ctx, backoffReason, shouldCycle) +} + +func (t *Transport) OnStorenodeAvailableOneShot() <-chan struct{} { + return t.waku.OnStorenodeAvailableOneShot() +} + +func (t *Transport) OnStorenodeChanged() <-chan peer.ID { + return t.waku.OnStorenodeChanged() +} + +func (t *Transport) OnStorenodeNotWorking() <-chan struct{} { + return t.waku.OnStorenodeNotWorking() +} + +func (t *Transport) OnStorenodeAvailable() <-chan peer.ID { + return t.waku.OnStorenodeAvailable() +} + +func (t *Transport) WaitForAvailableStoreNode(timeout time.Duration) bool { + return t.waku.WaitForAvailableStoreNode(timeout) +} + +func (t *Transport) IsStorenodeAvailable(peerID peer.ID) bool { + return t.waku.IsStorenodeAvailable(peerID) +} + +func (t *Transport) PerformStorenodeTask(peerID peer.ID, fn func(peerID peer.ID) error) error { + return t.waku.PerformStorenodeTask(peerID, fn) +} + +func (t *Transport) ProcessMailserverBatch( + ctx context.Context, + batch types.MailserverBatch, + storenodeID peer.ID, + pageLimit uint64, + shouldProcessNextPage func(int) (bool, uint64), + processEnvelopes bool, +) error { + return t.waku.ProcessMailserverBatch(ctx, batch, storenodeID, pageLimit, shouldProcessNextPage, processEnvelopes) +} + +func (t *Transport) SetStorenodeConfigProvider(c history.StorenodeConfigProvider) { + t.waku.SetStorenodeConfigProvider(c) +} diff --git a/services/ext/api.go b/services/ext/api.go index c24c2d0b2b4..8cc6d650f23 100644 --- a/services/ext/api.go +++ b/services/ext/api.go @@ -1402,10 +1402,6 @@ func (api *PublicAPI) RequestAllHistoricMessagesWithRetries(forceFetchingBackup return api.service.messenger.RequestAllHistoricMessages(forceFetchingBackup, true) } -func (api *PublicAPI) DisconnectActiveMailserver() { - api.service.messenger.DisconnectActiveMailserver() -} - // Echo is a method for testing purposes. func (api *PublicAPI) Echo(ctx context.Context, message string) (string, error) { return message, nil diff --git a/vendor/github.com/waku-org/go-waku/logging/logging.go b/vendor/github.com/waku-org/go-waku/logging/logging.go index ac895e9d58a..f8742ac5165 100644 --- a/vendor/github.com/waku-org/go-waku/logging/logging.go +++ b/vendor/github.com/waku-org/go-waku/logging/logging.go @@ -74,6 +74,14 @@ func (t timestamp) String() string { return time.Unix(0, int64(t)).Format(time.RFC3339) } +func Timep(key string, time *int64) zapcore.Field { + if time == nil { + return zap.String(key, "-") + } else { + return Time(key, *time) + } +} + func Epoch(key string, time time.Time) zap.Field { return zap.String(key, fmt.Sprintf("%d", time.UnixNano())) } diff --git a/vendor/github.com/waku-org/go-waku/waku/v2/api/history/cycle.go b/vendor/github.com/waku-org/go-waku/waku/v2/api/history/cycle.go new file mode 100644 index 00000000000..cd760f72dec --- /dev/null +++ b/vendor/github.com/waku-org/go-waku/waku/v2/api/history/cycle.go @@ -0,0 +1,485 @@ +package history + +import ( + "context" + "crypto/rand" + "errors" + "math" + "math/big" + "net" + "runtime" + "sort" + "sync" + "time" + + "github.com/libp2p/go-libp2p/core/host" + "github.com/libp2p/go-libp2p/core/peer" + "github.com/libp2p/go-libp2p/p2p/protocol/ping" + "go.uber.org/zap" +) + +const defaultBackoff = 10 * time.Second +const graylistBackoff = 3 * time.Minute +const storenodeMaxFailedRequests uint = 2 +const isAndroidEmulator = runtime.GOOS == "android" && runtime.GOARCH == "amd64" +const findNearestMailServer = !isAndroidEmulator +const overrideDNS = runtime.GOOS == "android" || runtime.GOOS == "ios" +const bootstrapDNS = "8.8.8.8:53" + +type connStatus int + +const ( + disconnected connStatus = iota + 1 + connected +) + +type peerStatus struct { + status connStatus + canConnectAfter time.Time + lastConnectionAttempt time.Time +} + +type StorenodeConfigProvider interface { + UseStorenodes() (bool, error) + GetPinnedStorenode() (peer.ID, error) + Storenodes() ([]peer.ID, error) +} + +type StorenodeCycle struct { + sync.RWMutex + + logger *zap.Logger + + host host.Host + + storenodeConfigProvider StorenodeConfigProvider + + StorenodeAvailableOneshotEmitter *OneShotEmitter[struct{}] + StorenodeChangedEmitter *Emitter[peer.ID] + StorenodeNotWorkingEmitter *Emitter[struct{}] + StorenodeAvailableEmitter *Emitter[peer.ID] + + failedRequests map[peer.ID]uint + + peersMutex sync.RWMutex + activeStorenode peer.ID + peers map[peer.ID]peerStatus +} + +func NewStorenodeCycle(logger *zap.Logger) *StorenodeCycle { + return &StorenodeCycle{ + StorenodeAvailableOneshotEmitter: NewOneshotEmitter[struct{}](), + StorenodeChangedEmitter: NewEmitter[peer.ID](), + StorenodeNotWorkingEmitter: NewEmitter[struct{}](), + StorenodeAvailableEmitter: NewEmitter[peer.ID](), + logger: logger.Named("storenode-cycle"), + } +} + +func (m *StorenodeCycle) Start(ctx context.Context, h host.Host) { + m.logger.Debug("starting storenode cycle") + m.host = h + m.failedRequests = make(map[peer.ID]uint) + + go m.verifyStorenodeStatus(ctx) +} + +func (m *StorenodeCycle) DisconnectActiveStorenode(backoff time.Duration) { + m.Lock() + defer m.Unlock() + + m.disconnectActiveStorenode(backoff) +} + +func (m *StorenodeCycle) connectToNewStorenodeAndWait(ctx context.Context) error { + // Handle pinned storenodes + m.logger.Info("disconnecting storenode") + pinnedStorenode, err := m.storenodeConfigProvider.GetPinnedStorenode() + if err != nil { + m.logger.Error("could not obtain the pinned storenode", zap.Error(err)) + return err + } + + // If no pinned storenode, no need to disconnect and wait for it to be available + if pinnedStorenode == "" { + m.disconnectActiveStorenode(graylistBackoff) + } + + return m.findNewStorenode(ctx) +} + +func (m *StorenodeCycle) disconnectStorenode(backoffDuration time.Duration) error { + if m.activeStorenode == "" { + m.logger.Info("no active storenode") + return nil + } + + m.logger.Info("disconnecting active storenode", zap.Stringer("peerID", m.activeStorenode)) + + m.peersMutex.Lock() + pInfo, ok := m.peers[m.activeStorenode] + if ok { + pInfo.status = disconnected + pInfo.canConnectAfter = time.Now().Add(backoffDuration) + m.peers[m.activeStorenode] = pInfo + } else { + m.peers[m.activeStorenode] = peerStatus{ + status: disconnected, + canConnectAfter: time.Now().Add(backoffDuration), + } + } + m.peersMutex.Unlock() + + m.activeStorenode = "" + + return nil +} + +func (m *StorenodeCycle) disconnectActiveStorenode(backoffDuration time.Duration) { + err := m.disconnectStorenode(backoffDuration) + if err != nil { + m.logger.Error("failed to disconnect storenode", zap.Error(err)) + } + + m.StorenodeChangedEmitter.Emit("") +} + +func (m *StorenodeCycle) Cycle(ctx context.Context) { + m.logger.Info("Automatically switching storenode") + + if m.activeStorenode != "" { + m.disconnectActiveStorenode(graylistBackoff) + } + + useStorenode, err := m.storenodeConfigProvider.UseStorenodes() + if err != nil { + m.logger.Error("failed to get use storenodes", zap.Error(err)) + return + } + + if !useStorenode { + m.logger.Info("Skipping storenode search due to useStorenode being false") + return + } + + err = m.findNewStorenode(ctx) + if err != nil { + m.logger.Error("Error getting new storenode", zap.Error(err)) + } +} + +func poolSize(fleetSize int) int { + return int(math.Ceil(float64(fleetSize) / 4)) +} + +func (m *StorenodeCycle) getAvailableStorenodesSortedByRTT(ctx context.Context, allStorenodes []peer.ID) []peer.ID { + // TODO: this can be replaced by peer selector once code is moved to go-waku api + availableStorenodes := make(map[peer.ID]time.Duration) + availableStorenodesMutex := sync.Mutex{} + availableStorenodesWg := sync.WaitGroup{} + for _, storenode := range allStorenodes { + availableStorenodesWg.Add(1) + go func(peerID peer.ID) { + defer availableStorenodesWg.Done() + ctx, cancel := context.WithTimeout(ctx, 4*time.Second) + defer cancel() + + rtt, err := m.pingPeer(ctx, peerID) + if err == nil { // pinging storenodes might fail, but we don't care + availableStorenodesMutex.Lock() + availableStorenodes[peerID] = rtt + availableStorenodesMutex.Unlock() + } + }(storenode) + } + availableStorenodesWg.Wait() + + if len(availableStorenodes) == 0 { + m.logger.Warn("No storenodes available") // Do nothing.. + return nil + } + + var sortedStorenodes []SortedStorenode + for storenodeID, rtt := range availableStorenodes { + sortedStorenode := SortedStorenode{ + Storenode: storenodeID, + RTT: rtt, + } + m.peersMutex.Lock() + pInfo, ok := m.peers[storenodeID] + m.peersMutex.Unlock() + if ok && time.Now().Before(pInfo.canConnectAfter) { + continue // We can't connect to this node yet + } + sortedStorenodes = append(sortedStorenodes, sortedStorenode) + } + sort.Sort(byRTTMsAndCanConnectBefore(sortedStorenodes)) + + result := make([]peer.ID, len(sortedStorenodes)) + for i, s := range sortedStorenodes { + result[i] = s.Storenode + } + + return result +} + +func (m *StorenodeCycle) pingPeer(ctx context.Context, peerID peer.ID) (time.Duration, error) { + pingResultCh := ping.Ping(ctx, m.host, peerID) + select { + case <-ctx.Done(): + return 0, ctx.Err() + case r := <-pingResultCh: + if r.Error != nil { + return 0, r.Error + } + return r.RTT, nil + } +} + +func (m *StorenodeCycle) findNewStorenode(ctx context.Context) error { + // we have to override DNS manually because of https://github.com/status-im/status-mobile/issues/19581 + if overrideDNS { + var dialer net.Dialer + net.DefaultResolver = &net.Resolver{ + PreferGo: false, + Dial: func(context context.Context, _, _ string) (net.Conn, error) { + conn, err := dialer.DialContext(context, "udp", bootstrapDNS) + if err != nil { + return nil, err + } + return conn, nil + }, + } + } + + pinnedStorenode, err := m.storenodeConfigProvider.GetPinnedStorenode() + if err != nil { + m.logger.Error("Could not obtain the pinned storenode", zap.Error(err)) + return err + } + + if pinnedStorenode != "" { + return m.connect(pinnedStorenode) + } + + m.logger.Info("Finding a new storenode..") + + allStorenodes, err := m.storenodeConfigProvider.Storenodes() + if err != nil { + return err + } + + // TODO: remove this check once sockets are stable on x86_64 emulators + if findNearestMailServer { + allStorenodes = m.getAvailableStorenodesSortedByRTT(ctx, allStorenodes) + } + + // Picks a random storenode amongs the ones with the lowest latency + // The pool size is 1/4 of the storenodes were pinged successfully + pSize := poolSize(len(allStorenodes) - 1) + if pSize <= 0 { + pSize = len(allStorenodes) + if pSize <= 0 { + m.logger.Warn("No storenodes available") // Do nothing.. + return nil + } + } + + r, err := rand.Int(rand.Reader, big.NewInt(int64(pSize))) + if err != nil { + return err + } + + ms := allStorenodes[r.Int64()] + return m.connect(ms) +} + +func (m *StorenodeCycle) storenodeStatus(peerID peer.ID) connStatus { + m.peersMutex.RLock() + defer m.peersMutex.RUnlock() + + peer, ok := m.peers[peerID] + if !ok { + return disconnected + } + return peer.status +} + +func (m *StorenodeCycle) connect(peerID peer.ID) error { + m.logger.Info("connecting to storenode", zap.Stringer("peerID", peerID)) + + m.activeStorenode = peerID + + m.StorenodeChangedEmitter.Emit(m.activeStorenode) + + storenodeStatus := m.storenodeStatus(peerID) + if storenodeStatus != connected { + m.peersMutex.Lock() + m.peers[peerID] = peerStatus{ + status: connected, + lastConnectionAttempt: time.Now(), + canConnectAfter: time.Now().Add(defaultBackoff), + } + m.peersMutex.Unlock() + + m.failedRequests[peerID] = 0 + m.logger.Info("storenode available", zap.Stringer("peerID", m.activeStorenode)) + + m.StorenodeAvailableOneshotEmitter.Emit(struct{}{}) // Maybe can be refactored away? + m.StorenodeAvailableEmitter.Emit(m.activeStorenode) + } + return nil +} + +func (m *StorenodeCycle) GetActiveStorenode() peer.ID { + m.RLock() + defer m.RUnlock() + + return m.activeStorenode +} + +func (m *StorenodeCycle) IsStorenodeAvailable(peerID peer.ID) bool { + return m.storenodeStatus(peerID) == connected +} + +func (m *StorenodeCycle) penalizeStorenode(id peer.ID) { + m.peersMutex.Lock() + defer m.peersMutex.Unlock() + pInfo, ok := m.peers[id] + if !ok { + pInfo.status = disconnected + } + + pInfo.canConnectAfter = time.Now().Add(graylistBackoff) + m.peers[id] = pInfo +} + +func (m *StorenodeCycle) verifyStorenodeStatus(ctx context.Context) { + ticker := time.NewTicker(1 * time.Second) + defer ticker.Stop() + + for { + select { + case <-ticker.C: + err := m.disconnectStorenodeIfRequired(ctx) + if err != nil { + m.logger.Error("failed to handle storenode cycle event", zap.Error(err)) + continue + } + + case <-ctx.Done(): + return + } + } +} + +func (m *StorenodeCycle) disconnectStorenodeIfRequired(ctx context.Context) error { + m.logger.Debug("wakuV2 storenode status verification") + + if m.activeStorenode == "" { + // No active storenode, find a new one + m.Cycle(ctx) + return nil + } + + // Check whether we want to disconnect the active storenode + if m.failedRequests[m.activeStorenode] >= storenodeMaxFailedRequests { + m.penalizeStorenode(m.activeStorenode) + m.StorenodeNotWorkingEmitter.Emit(struct{}{}) + + m.logger.Info("too many failed requests", zap.Stringer("storenode", m.activeStorenode)) + m.failedRequests[m.activeStorenode] = 0 + return m.connectToNewStorenodeAndWait(ctx) + } + + return nil +} + +func (m *StorenodeCycle) SetStorenodeConfigProvider(provider StorenodeConfigProvider) { + m.storenodeConfigProvider = provider +} + +func (m *StorenodeCycle) WaitForAvailableStoreNode(ctx context.Context, timeout time.Duration) bool { + // Add 1 second to timeout, because the storenode cycle has 1 second ticker, which doesn't tick on start. + // This can be improved after merging https://github.com/status-im/status-go/pull/4380. + // NOTE: https://stackoverflow.com/questions/32705582/how-to-get-time-tick-to-tick-immediately + timeout += time.Second + + ctx, cancel := context.WithTimeout(ctx, timeout) + defer cancel() + + wg := sync.WaitGroup{} + wg.Add(1) + go func() { + defer wg.Done() + for !m.IsStorenodeAvailable(m.activeStorenode) { + select { + case <-m.StorenodeAvailableOneshotEmitter.Subscribe(): + case <-ctx.Done(): + return + } + } + }() + + select { + case <-waitForWaitGroup(&wg): + case <-ctx.Done(): + } + + return m.IsStorenodeAvailable(m.activeStorenode) +} + +func waitForWaitGroup(wg *sync.WaitGroup) <-chan struct{} { + ch := make(chan struct{}) + go func() { + wg.Wait() + close(ch) + }() + return ch +} + +func (m *StorenodeCycle) PerformStorenodeTask(peerID peer.ID, fn func(peerID peer.ID) error) error { + if peerID == "" { + return errors.New("storenode not available") + } + + m.RLock() + defer m.RUnlock() + + var tries uint = 0 + for tries < storenodeMaxFailedRequests { + if m.storenodeStatus(peerID) != connected { + return errors.New("storenode not available") + } + m.logger.Info("trying performing history requests", zap.Uint("try", tries), zap.Stringer("peerID", peerID)) + + // Peform request + err := fn(peerID) + if err == nil { + // Reset failed requests + m.logger.Debug("history request performed successfully", zap.Stringer("peerID", peerID)) + m.failedRequests[peerID] = 0 + return nil + } + + m.logger.Error("failed to perform history request", + zap.Stringer("peerID", peerID), + zap.Uint("tries", tries), + zap.Error(err), + ) + + tries++ + + // Increment failed requests + m.failedRequests[peerID]++ + + // Change storenode + if m.failedRequests[peerID] >= storenodeMaxFailedRequests { + return errors.New("too many failed requests") + } + // Wait a couple of second not to spam + time.Sleep(2 * time.Second) + + } + return errors.New("failed to perform history request") +} diff --git a/vendor/github.com/waku-org/go-waku/waku/v2/api/history/emitters.go b/vendor/github.com/waku-org/go-waku/waku/v2/api/history/emitters.go new file mode 100644 index 00000000000..2aee54763c3 --- /dev/null +++ b/vendor/github.com/waku-org/go-waku/waku/v2/api/history/emitters.go @@ -0,0 +1,49 @@ +package history + +import "sync" + +type Emitter[T any] struct { + sync.Mutex + subscriptions []chan T +} + +func NewEmitter[T any]() *Emitter[T] { + return &Emitter[T]{} +} + +func (s *Emitter[T]) Subscribe() <-chan T { + s.Lock() + defer s.Unlock() + c := make(chan T) + s.subscriptions = append(s.subscriptions, c) + return c +} + +func (s *Emitter[T]) Emit(value T) { + s.Lock() + defer s.Unlock() + + for _, subs := range s.subscriptions { + subs <- value + } + s.subscriptions = nil +} + +type OneShotEmitter[T any] struct { + Emitter[T] +} + +func NewOneshotEmitter[T any]() *OneShotEmitter[T] { + return &OneShotEmitter[T]{} +} + +func (s *OneShotEmitter[T]) Emit(value T) { + s.Lock() + defer s.Unlock() + + for _, subs := range s.subscriptions { + subs <- value + close(subs) + } + s.subscriptions = nil +} diff --git a/vendor/github.com/waku-org/go-waku/waku/v2/api/history/history.go b/vendor/github.com/waku-org/go-waku/waku/v2/api/history/history.go new file mode 100644 index 00000000000..aeb268d9663 --- /dev/null +++ b/vendor/github.com/waku-org/go-waku/waku/v2/api/history/history.go @@ -0,0 +1,281 @@ +package history + +import ( + "context" + "errors" + "math" + "sync" + "time" + + "github.com/ethereum/go-ethereum/common/hexutil" + "github.com/libp2p/go-libp2p/core/peer" + "github.com/waku-org/go-waku/logging" + "github.com/waku-org/go-waku/waku/v2/protocol" + "github.com/waku-org/go-waku/waku/v2/protocol/store" + "go.uber.org/zap" +) + +const maxTopicsPerRequest int = 10 +const mailserverRequestTimeout = 30 * time.Second + +type work struct { + contentFilter protocol.ContentFilter + cursor []byte + limit uint64 +} + +type HistoryRetriever struct { + store *store.WakuStore + logger *zap.Logger + historyProcessor HistoryProcessor +} + +type HistoryProcessor interface { + OnEnvelope(env *protocol.Envelope, processEnvelopes bool) error + OnRequestFailed(requestID []byte, peerID peer.ID, err error) +} + +func NewHistoryRetriever(store *store.WakuStore, historyProcessor HistoryProcessor, logger *zap.Logger) *HistoryRetriever { + return &HistoryRetriever{ + store: store, + logger: logger.Named("history-retriever"), + historyProcessor: historyProcessor, + } +} + +func (hr *HistoryRetriever) Query( + ctx context.Context, + criteria store.FilterCriteria, + storenodeID peer.ID, + pageLimit uint64, + shouldProcessNextPage func(int) (bool, uint64), + processEnvelopes bool, +) error { + logger := hr.logger.With( + logging.Timep("fromString", criteria.TimeStart), + logging.Timep("toString", criteria.TimeEnd), + zap.String("pubsubTopic", criteria.PubsubTopic), + zap.Strings("contentTopics", criteria.ContentTopicsList()), + zap.Int64p("from", criteria.TimeStart), + zap.Int64p("to", criteria.TimeEnd), + ) + + logger.Info("syncing") + + wg := sync.WaitGroup{} + workWg := sync.WaitGroup{} + workCh := make(chan work, 1000) // each batch item is split in 10 topics bunch and sent to this channel + workCompleteCh := make(chan struct{}) // once all batch items are processed, this channel is triggered + semaphore := make(chan struct{}, 3) // limit the number of concurrent queries + errCh := make(chan error) + + ctx, cancel := context.WithCancel(ctx) + defer cancel() + + // TODO: refactor this by extracting the consumer into a separate go routine. + + // Producer + wg.Add(1) + go func() { + defer func() { + logger.Debug("mailserver batch producer complete") + wg.Done() + }() + + contentTopicList := criteria.ContentTopics.ToList() + + // TODO: split into 24h batches + + allWorks := int(math.Ceil(float64(len(contentTopicList)) / float64(maxTopicsPerRequest))) + workWg.Add(allWorks) + + for i := 0; i < len(contentTopicList); i += maxTopicsPerRequest { + j := i + maxTopicsPerRequest + if j > len(contentTopicList) { + j = len(contentTopicList) + } + + select { + case <-ctx.Done(): + logger.Debug("processBatch producer - context done") + return + default: + logger.Debug("processBatch producer - creating work") + workCh <- work{ + contentFilter: protocol.NewContentFilter(criteria.PubsubTopic, contentTopicList[i:j]...), + limit: pageLimit, + } + } + } + + go func() { + workWg.Wait() + workCompleteCh <- struct{}{} + }() + + logger.Debug("processBatch producer complete") + }() + + var result error + +loop: + for { + select { + case <-ctx.Done(): + logger.Debug("processBatch cleanup - context done") + result = ctx.Err() + if errors.Is(result, context.Canceled) { + result = nil + } + break loop + case w, ok := <-workCh: + if !ok { + continue + } + + logger.Debug("processBatch - received work") + + semaphore <- struct{}{} + go func(w work) { // Consumer + defer func() { + workWg.Done() + <-semaphore + }() + + queryCtx, queryCancel := context.WithTimeout(ctx, mailserverRequestTimeout) + cursor, envelopesCount, err := hr.createMessagesRequest(queryCtx, storenodeID, criteria, w.cursor, w.limit, true, processEnvelopes, logger) + queryCancel() + + if err != nil { + logger.Debug("failed to send request", zap.Error(err)) + errCh <- err + return + } + + processNextPage := true + nextPageLimit := pageLimit + if shouldProcessNextPage != nil { + processNextPage, nextPageLimit = shouldProcessNextPage(envelopesCount) + } + + if !processNextPage { + return + } + + // Check the cursor after calling `shouldProcessNextPage`. + // The app might use process the fetched envelopes in the callback for own needs. + if cursor == nil { + return + } + + logger.Debug("processBatch producer - creating work (cursor)") + + workWg.Add(1) + workCh <- work{ + contentFilter: w.contentFilter, + cursor: cursor, + limit: nextPageLimit, + } + }(w) + case err := <-errCh: + logger.Debug("processBatch - received error", zap.Error(err)) + cancel() // Kill go routines + return err + case <-workCompleteCh: + logger.Debug("processBatch - all jobs complete") + cancel() // Kill go routines + } + } + + wg.Wait() + + logger.Info("synced topic", zap.NamedError("hasError", result)) + + return result +} + +func (hr *HistoryRetriever) createMessagesRequest( + ctx context.Context, + peerID peer.ID, + criteria store.FilterCriteria, + cursor []byte, + limit uint64, + waitForResponse bool, + processEnvelopes bool, + logger *zap.Logger, +) (storeCursor []byte, envelopesCount int, err error) { + if waitForResponse { + resultCh := make(chan struct { + storeCursor []byte + envelopesCount int + err error + }) + + go func() { + storeCursor, envelopesCount, err = hr.requestStoreMessages(ctx, peerID, criteria, cursor, limit, processEnvelopes) + resultCh <- struct { + storeCursor []byte + envelopesCount int + err error + }{storeCursor, envelopesCount, err} + }() + + select { + case result := <-resultCh: + return result.storeCursor, result.envelopesCount, result.err + case <-ctx.Done(): + return nil, 0, ctx.Err() + } + } else { + go func() { + _, _, err = hr.requestStoreMessages(ctx, peerID, criteria, cursor, limit, false) + if err != nil { + logger.Error("failed to request store messages", zap.Error(err)) + } + }() + } + + return +} + +func (hr *HistoryRetriever) requestStoreMessages(ctx context.Context, peerID peer.ID, criteria store.FilterCriteria, cursor []byte, limit uint64, processEnvelopes bool) ([]byte, int, error) { + requestID := protocol.GenerateRequestID() + logger := hr.logger.With(zap.String("requestID", hexutil.Encode(requestID)), zap.Stringer("peerID", peerID)) + + opts := []store.RequestOption{ + store.WithPaging(false, limit), + store.WithRequestID(requestID), + store.WithPeer(peerID), + store.WithCursor(cursor)} + + logger.Debug("store.query", + logging.Timep("startTime", criteria.TimeStart), + logging.Timep("endTime", criteria.TimeEnd), + zap.Strings("contentTopics", criteria.ContentTopics.ToList()), + zap.String("pubsubTopic", criteria.PubsubTopic), + zap.String("cursor", hexutil.Encode(cursor)), + ) + + queryStart := time.Now() + result, err := hr.store.Query(ctx, criteria, opts...) + queryDuration := time.Since(queryStart) + if err != nil { + logger.Error("error querying storenode", zap.Error(err)) + + hr.historyProcessor.OnRequestFailed(requestID, peerID, err) + + return nil, 0, err + } + + messages := result.Messages() + envelopesCount := len(messages) + logger.Debug("store.query response", zap.Duration("queryDuration", queryDuration), zap.Int("numMessages", envelopesCount), zap.Bool("hasCursor", result.IsComplete() && result.Cursor() != nil)) + for _, mkv := range messages { + envelope := protocol.NewEnvelope(mkv.Message, mkv.Message.GetTimestamp(), mkv.GetPubsubTopic()) + err := hr.historyProcessor.OnEnvelope(envelope, processEnvelopes) + if err != nil { + return nil, 0, err + } + } + return result.Cursor(), envelopesCount, nil +} diff --git a/vendor/github.com/waku-org/go-waku/waku/v2/api/history/sort.go b/vendor/github.com/waku-org/go-waku/waku/v2/api/history/sort.go new file mode 100644 index 00000000000..22e94c571f9 --- /dev/null +++ b/vendor/github.com/waku-org/go-waku/waku/v2/api/history/sort.go @@ -0,0 +1,32 @@ +package history + +import ( + "time" + + "github.com/libp2p/go-libp2p/core/peer" +) + +type SortedStorenode struct { + Storenode peer.ID + RTT time.Duration + CanConnectAfter time.Time +} + +type byRTTMsAndCanConnectBefore []SortedStorenode + +func (s byRTTMsAndCanConnectBefore) Len() int { + return len(s) +} + +func (s byRTTMsAndCanConnectBefore) Swap(i, j int) { + s[i], s[j] = s[j], s[i] +} + +func (s byRTTMsAndCanConnectBefore) Less(i, j int) bool { + // Slightly inaccurate as time sensitive sorting, but it does not matter so much + now := time.Now() + if s[i].CanConnectAfter.Before(now) && s[j].CanConnectAfter.Before(now) { + return s[i].RTT < s[j].RTT + } + return s[i].CanConnectAfter.Before(s[j].CanConnectAfter) +} diff --git a/vendor/github.com/waku-org/go-waku/waku/v2/api/publish/message_check.go b/vendor/github.com/waku-org/go-waku/waku/v2/api/publish/message_check.go index 67a67c91344..e86199869f2 100644 --- a/vendor/github.com/waku-org/go-waku/waku/v2/api/publish/message_check.go +++ b/vendor/github.com/waku-org/go-waku/waku/v2/api/publish/message_check.go @@ -8,8 +8,8 @@ import ( "github.com/ethereum/go-ethereum/common" "github.com/ethereum/go-ethereum/common/hexutil" - "github.com/libp2p/go-libp2p/core/peer" apicommon "github.com/waku-org/go-waku/waku/v2/api/common" + "github.com/waku-org/go-waku/waku/v2/api/history" "github.com/waku-org/go-waku/waku/v2/protocol" "github.com/waku-org/go-waku/waku/v2/protocol/pb" "github.com/waku-org/go-waku/waku/v2/protocol/store" @@ -28,7 +28,6 @@ type ISentCheck interface { Start() Add(topic string, messageID common.Hash, sentTime uint32) DeleteByMessageIDs(messageIDs []common.Hash) - SetStorePeerID(peerID peer.ID) } // MessageSentCheck tracks the outgoing messages and check against store node @@ -37,11 +36,11 @@ type ISentCheck interface { type MessageSentCheck struct { messageIDs map[string]map[common.Hash]uint32 messageIDsMu sync.RWMutex - storePeerID peer.ID messageStoredChan chan common.Hash messageExpiredChan chan common.Hash ctx context.Context store *store.WakuStore + storenodeCycle *history.StorenodeCycle timesource timesource.Timesource logger *zap.Logger maxHashQueryLength uint64 @@ -52,7 +51,7 @@ type MessageSentCheck struct { } // NewMessageSentCheck creates a new instance of MessageSentCheck with default parameters -func NewMessageSentCheck(ctx context.Context, store *store.WakuStore, timesource timesource.Timesource, msgStoredChan chan common.Hash, msgExpiredChan chan common.Hash, logger *zap.Logger) *MessageSentCheck { +func NewMessageSentCheck(ctx context.Context, store *store.WakuStore, cycle *history.StorenodeCycle, timesource timesource.Timesource, msgStoredChan chan common.Hash, msgExpiredChan chan common.Hash, logger *zap.Logger) *MessageSentCheck { return &MessageSentCheck{ messageIDs: make(map[string]map[common.Hash]uint32), messageIDsMu: sync.RWMutex{}, @@ -60,6 +59,7 @@ func NewMessageSentCheck(ctx context.Context, store *store.WakuStore, timesource messageExpiredChan: msgExpiredChan, ctx: ctx, store: store, + storenodeCycle: cycle, timesource: timesource, logger: logger, maxHashQueryLength: DefaultMaxHashQueryLength, @@ -138,11 +138,6 @@ func (m *MessageSentCheck) DeleteByMessageIDs(messageIDs []common.Hash) { } } -// SetStorePeerID sets the peer id of store node -func (m *MessageSentCheck) SetStorePeerID(peerID peer.ID) { - m.storePeerID = peerID -} - // Start checks if the tracked outgoing messages are stored periodically func (m *MessageSentCheck) Start() { ticker := time.NewTicker(m.hashQueryInterval) @@ -209,7 +204,7 @@ func (m *MessageSentCheck) Start() { } func (m *MessageSentCheck) messageHashBasedQuery(ctx context.Context, hashes []common.Hash, relayTime []uint32, pubsubTopic string) []common.Hash { - selectedPeer := m.storePeerID + selectedPeer := m.storenodeCycle.GetActiveStorenode() if selectedPeer == "" { m.logger.Error("no store peer id available", zap.String("pubsubTopic", pubsubTopic)) return []common.Hash{} diff --git a/vendor/github.com/waku-org/go-waku/waku/v2/api/publish/message_sender.go b/vendor/github.com/waku-org/go-waku/waku/v2/api/publish/message_sender.go index 479d894ad58..c1e9a4ca783 100644 --- a/vendor/github.com/waku-org/go-waku/waku/v2/api/publish/message_sender.go +++ b/vendor/github.com/waku-org/go-waku/waku/v2/api/publish/message_sender.go @@ -6,7 +6,6 @@ import ( "time" "github.com/ethereum/go-ethereum/common" - "github.com/libp2p/go-libp2p/core/peer" "github.com/waku-org/go-waku/waku/v2/protocol" "github.com/waku-org/go-waku/waku/v2/protocol/lightpush" "github.com/waku-org/go-waku/waku/v2/protocol/relay" @@ -162,9 +161,3 @@ func (ms *MessageSender) MessagesDelivered(messageIDs []common.Hash) { ms.messageSentCheck.DeleteByMessageIDs(messageIDs) } } - -func (ms *MessageSender) SetStorePeerID(peerID peer.ID) { - if ms.messageSentCheck != nil { - ms.messageSentCheck.SetStorePeerID(peerID) - } -} diff --git a/vendor/github.com/waku-org/go-waku/waku/v2/node/wakunode2.go b/vendor/github.com/waku-org/go-waku/waku/v2/node/wakunode2.go index f9dc443fc5d..10153fd6d63 100644 --- a/vendor/github.com/waku-org/go-waku/waku/v2/node/wakunode2.go +++ b/vendor/github.com/waku-org/go-waku/waku/v2/node/wakunode2.go @@ -292,7 +292,7 @@ func New(opts ...WakuNodeOption) (*WakuNode, error) { w.filterLightNode = filter.NewWakuFilterLightNode(w.bcaster, w.peermanager, w.timesource, w.opts.onlineChecker, w.opts.prometheusReg, w.log) w.lightPush = lightpush.NewWakuLightPush(w.Relay(), w.peermanager, w.opts.prometheusReg, w.log, w.opts.lightpushOpts...) - w.store = store.NewWakuStore(w.peermanager, w.timesource, w.log) + w.store = store.NewWakuStore(w.peermanager, w.timesource, w.log, w.opts.storeRateLimit) if params.storeFactory != nil { w.storeFactory = params.storeFactory diff --git a/vendor/github.com/waku-org/go-waku/waku/v2/node/wakuoptions.go b/vendor/github.com/waku-org/go-waku/waku/v2/node/wakuoptions.go index 445065de636..112cafe616b 100644 --- a/vendor/github.com/waku-org/go-waku/waku/v2/node/wakuoptions.go +++ b/vendor/github.com/waku-org/go-waku/waku/v2/node/wakuoptions.go @@ -38,6 +38,7 @@ import ( "github.com/waku-org/go-waku/waku/v2/utils" "go.uber.org/zap" "go.uber.org/zap/zapcore" + "golang.org/x/time/rate" ) // Default UserAgent @@ -94,6 +95,8 @@ type WakuNodeParameters struct { enableStore bool messageProvider legacy_store.MessageProvider + storeRateLimit rate.Limit + enableRendezvousPoint bool rendezvousDB *rendezvous.DB @@ -139,6 +142,7 @@ var DefaultWakuNodeOptions = []WakuNodeOption{ WithCircuitRelayParams(2*time.Second, 3*time.Minute), WithPeerStoreCapacity(DefaultMaxPeerStoreCapacity), WithOnlineChecker(onlinechecker.NewDefaultOnlineChecker(true)), + WithWakuStoreRateLimit(8), // Value currently set in status.staging } // MultiAddresses return the list of multiaddresses configured in the node @@ -458,6 +462,16 @@ func WithWakuFilterFullNode(filterOpts ...filter.Option) WakuNodeOption { } } +// WithWakuStoreRateLimit is used to set a default rate limit on which storenodes will +// be sent per peerID to avoid running into a TOO_MANY_REQUESTS (429) error when consuming +// the store protocol from a storenode +func WithWakuStoreRateLimit(value rate.Limit) WakuNodeOption { + return func(params *WakuNodeParameters) error { + params.storeRateLimit = value + return nil + } +} + // WithWakuStore enables the Waku V2 Store protocol and if the messages should // be stored or not in a message provider. func WithWakuStore() WakuNodeOption { diff --git a/vendor/github.com/waku-org/go-waku/waku/v2/protocol/peer_exchange/client.go b/vendor/github.com/waku-org/go-waku/waku/v2/protocol/peer_exchange/client.go index 8075e2de8d0..f901590d583 100644 --- a/vendor/github.com/waku-org/go-waku/waku/v2/protocol/peer_exchange/client.go +++ b/vendor/github.com/waku-org/go-waku/waku/v2/protocol/peer_exchange/client.go @@ -123,13 +123,11 @@ func (wakuPX *WakuPeerExchange) handleResponse(ctx context.Context, response *pb } if params.clusterID != 0 { - wakuPX.log.Debug("clusterID is non zero, filtering by shard") rs, err := wenr.RelaySharding(enrRecord) if err != nil || rs == nil || !rs.Contains(uint16(params.clusterID), uint16(params.shard)) { wakuPX.log.Debug("peer doesn't matches filter", zap.Int("shard", params.shard)) continue } - wakuPX.log.Debug("peer matches filter", zap.Int("shard", params.shard)) } enodeRecord, err := enode.New(enode.ValidSchemes, enrRecord) diff --git a/vendor/github.com/waku-org/go-waku/waku/v2/protocol/relay/config.go b/vendor/github.com/waku-org/go-waku/waku/v2/protocol/relay/config.go index f0f41f80045..bc1ea9334be 100644 --- a/vendor/github.com/waku-org/go-waku/waku/v2/protocol/relay/config.go +++ b/vendor/github.com/waku-org/go-waku/waku/v2/protocol/relay/config.go @@ -13,6 +13,10 @@ import ( var DefaultRelaySubscriptionBufferSize int = 1024 +// trying to match value here https://github.com/vacp2p/nim-libp2p/pull/1077 +// note that nim-libp2p has 2 peer queues 1 for priority and other non-priority, whereas go-libp2p seems to have single peer-queue +var DefaultPeerOutboundQSize int = 1024 + type RelaySubscribeParameters struct { dontConsume bool cacheSize uint @@ -109,6 +113,7 @@ func (w *WakuRelay) defaultPubsubOptions() []pubsub.Option { pubsub.WithSeenMessagesTTL(2 * time.Minute), pubsub.WithPeerScore(w.peerScoreParams, w.peerScoreThresholds), pubsub.WithPeerScoreInspect(w.peerScoreInspector, 6*time.Second), + pubsub.WithPeerOutboundQueueSize(DefaultPeerOutboundQSize), } } diff --git a/vendor/github.com/waku-org/go-waku/waku/v2/protocol/store/client.go b/vendor/github.com/waku-org/go-waku/waku/v2/protocol/store/client.go index 92c47ff4c1d..3398c4bf922 100644 --- a/vendor/github.com/waku-org/go-waku/waku/v2/protocol/store/client.go +++ b/vendor/github.com/waku-org/go-waku/waku/v2/protocol/store/client.go @@ -19,6 +19,7 @@ import ( "github.com/waku-org/go-waku/waku/v2/protocol/store/pb" "github.com/waku-org/go-waku/waku/v2/timesource" "go.uber.org/zap" + "golang.org/x/time/rate" "google.golang.org/protobuf/proto" ) @@ -69,14 +70,19 @@ type WakuStore struct { timesource timesource.Timesource log *zap.Logger pm *peermanager.PeerManager + + defaultRatelimit rate.Limit + rateLimiters map[peer.ID]*rate.Limiter } // NewWakuStore is used to instantiate a StoreV3 client -func NewWakuStore(pm *peermanager.PeerManager, timesource timesource.Timesource, log *zap.Logger) *WakuStore { +func NewWakuStore(pm *peermanager.PeerManager, timesource timesource.Timesource, log *zap.Logger, defaultRatelimit rate.Limit) *WakuStore { s := new(WakuStore) s.log = log.Named("store-client") s.timesource = timesource s.pm = pm + s.defaultRatelimit = defaultRatelimit + s.rateLimiters = make(map[peer.ID]*rate.Limiter) if pm != nil { pm.RegisterWakuProtocol(StoreQueryID_v300, StoreENRField) @@ -171,7 +177,7 @@ func (s *WakuStore) Request(ctx context.Context, criteria Criteria, opts ...Requ return nil, err } - response, err := s.queryFrom(ctx, storeRequest, params.selectedPeer) + response, err := s.queryFrom(ctx, storeRequest, params) if err != nil { return nil, err } @@ -211,7 +217,7 @@ func (s *WakuStore) Exists(ctx context.Context, messageHash wpb.MessageHash, opt return len(result.messages) != 0, nil } -func (s *WakuStore) next(ctx context.Context, r *Result) (*Result, error) { +func (s *WakuStore) next(ctx context.Context, r *Result, opts ...RequestOption) (*Result, error) { if r.IsComplete() { return &Result{ store: s, @@ -223,11 +229,22 @@ func (s *WakuStore) next(ctx context.Context, r *Result) (*Result, error) { }, nil } + params := new(Parameters) + params.selectedPeer = r.PeerID() + optList := DefaultOptions() + optList = append(optList, opts...) + for _, opt := range optList { + err := opt(params) + if err != nil { + return nil, err + } + } + storeRequest := proto.Clone(r.storeRequest).(*pb.StoreQueryRequest) storeRequest.RequestId = hex.EncodeToString(protocol.GenerateRequestID()) storeRequest.PaginationCursor = r.Cursor() - response, err := s.queryFrom(ctx, storeRequest, r.PeerID()) + response, err := s.queryFrom(ctx, storeRequest, params) if err != nil { return nil, err } @@ -245,16 +262,28 @@ func (s *WakuStore) next(ctx context.Context, r *Result) (*Result, error) { } -func (s *WakuStore) queryFrom(ctx context.Context, storeRequest *pb.StoreQueryRequest, selectedPeer peer.ID) (*pb.StoreQueryResponse, error) { - logger := s.log.With(logging.HostID("peer", selectedPeer), zap.String("requestId", hex.EncodeToString([]byte(storeRequest.RequestId)))) +func (s *WakuStore) queryFrom(ctx context.Context, storeRequest *pb.StoreQueryRequest, params *Parameters) (*pb.StoreQueryResponse, error) { + logger := s.log.With(logging.HostID("peer", params.selectedPeer), zap.String("requestId", hex.EncodeToString([]byte(storeRequest.RequestId)))) logger.Debug("sending store request") - stream, err := s.h.NewStream(ctx, selectedPeer, StoreQueryID_v300) + if !params.skipRatelimit { + rateLimiter, ok := s.rateLimiters[params.selectedPeer] + if !ok { + rateLimiter = rate.NewLimiter(s.defaultRatelimit, 1) + s.rateLimiters[params.selectedPeer] = rateLimiter + } + err := rateLimiter.Wait(ctx) + if err != nil { + return nil, err + } + } + + stream, err := s.h.NewStream(ctx, params.selectedPeer, StoreQueryID_v300) if err != nil { logger.Error("creating stream to peer", zap.Error(err)) if ps, ok := s.h.Peerstore().(peerstore.WakuPeerstore); ok { - ps.AddConnFailure(selectedPeer) + ps.AddConnFailure(params.selectedPeer) } return nil, err } diff --git a/vendor/github.com/waku-org/go-waku/waku/v2/protocol/store/options.go b/vendor/github.com/waku-org/go-waku/waku/v2/protocol/store/options.go index b38afd53ab9..b8deba47cdf 100644 --- a/vendor/github.com/waku-org/go-waku/waku/v2/protocol/store/options.go +++ b/vendor/github.com/waku-org/go-waku/waku/v2/protocol/store/options.go @@ -19,6 +19,7 @@ type Parameters struct { pageLimit uint64 forward bool includeData bool + skipRatelimit bool } type RequestOption func(*Parameters) error @@ -115,6 +116,14 @@ func IncludeData(v bool) RequestOption { } } +// Skips the rate limiting for the current request (might cause the store request to fail with TOO_MANY_REQUESTS (429)) +func SkipRateLimit() RequestOption { + return func(params *Parameters) error { + params.skipRatelimit = true + return nil + } +} + // Default options to be used when querying a store node for results func DefaultOptions() []RequestOption { return []RequestOption{ diff --git a/vendor/github.com/waku-org/go-waku/waku/v2/protocol/store/pb/validation.go b/vendor/github.com/waku-org/go-waku/waku/v2/protocol/store/pb/validation.go index f54dea90e0a..542c6a05208 100644 --- a/vendor/github.com/waku-org/go-waku/waku/v2/protocol/store/pb/validation.go +++ b/vendor/github.com/waku-org/go-waku/waku/v2/protocol/store/pb/validation.go @@ -2,6 +2,7 @@ package pb import ( "errors" + "fmt" ) // MaxContentTopics is the maximum number of allowed contenttopics in a query @@ -10,7 +11,6 @@ const MaxContentTopics = 10 var ( errMissingRequestID = errors.New("missing RequestId field") errMessageHashOtherFields = errors.New("cannot use MessageHashes with ContentTopics/PubsubTopic") - errRequestIDMismatch = errors.New("requestID in response does not match request") errMaxContentTopics = errors.New("exceeds the maximum number of ContentTopics allowed") errEmptyContentTopic = errors.New("one or more content topics specified is empty") errMissingPubsubTopic = errors.New("missing PubsubTopic field") @@ -57,8 +57,8 @@ func (x *StoreQueryRequest) Validate() error { } func (x *StoreQueryResponse) Validate(requestID string) error { - if x.RequestId != "" && x.RequestId != requestID { - return errRequestIDMismatch + if x.RequestId != "" && x.RequestId != "N/A" && x.RequestId != requestID { + return fmt.Errorf("requestID %s in response does not match requestID in request %s", x.RequestId, requestID) } if x.StatusCode == nil { diff --git a/vendor/github.com/waku-org/go-waku/waku/v2/protocol/store/result.go b/vendor/github.com/waku-org/go-waku/waku/v2/protocol/store/result.go index 5ea4765ec8b..604d6453c2d 100644 --- a/vendor/github.com/waku-org/go-waku/waku/v2/protocol/store/result.go +++ b/vendor/github.com/waku-org/go-waku/waku/v2/protocol/store/result.go @@ -39,14 +39,14 @@ func (r *Result) Response() *pb.StoreQueryResponse { return r.storeResponse } -func (r *Result) Next(ctx context.Context) error { +func (r *Result) Next(ctx context.Context, opts ...RequestOption) error { if r.cursor == nil { r.done = true r.messages = nil return nil } - newResult, err := r.store.next(ctx, r) + newResult, err := r.store.next(ctx, r, opts...) if err != nil { return err } diff --git a/vendor/modules.txt b/vendor/modules.txt index 67645f7a701..d97e64bfe20 100644 --- a/vendor/modules.txt +++ b/vendor/modules.txt @@ -1010,13 +1010,14 @@ github.com/waku-org/go-discover/discover/v5wire github.com/waku-org/go-libp2p-rendezvous github.com/waku-org/go-libp2p-rendezvous/db github.com/waku-org/go-libp2p-rendezvous/pb -# github.com/waku-org/go-waku v0.8.1-0.20240904143057-f9e7895202da +# github.com/waku-org/go-waku v0.8.1-0.20240919183940-d9361e5302c3 ## explicit; go 1.21 github.com/waku-org/go-waku/logging github.com/waku-org/go-waku/tests github.com/waku-org/go-waku/waku/persistence github.com/waku-org/go-waku/waku/v2/api/common github.com/waku-org/go-waku/waku/v2/api/filter +github.com/waku-org/go-waku/waku/v2/api/history github.com/waku-org/go-waku/waku/v2/api/missing github.com/waku-org/go-waku/waku/v2/api/publish github.com/waku-org/go-waku/waku/v2/discv5 diff --git a/wakuv2/history_processor_wrapper.go b/wakuv2/history_processor_wrapper.go new file mode 100644 index 00000000000..b0b0ca3063d --- /dev/null +++ b/wakuv2/history_processor_wrapper.go @@ -0,0 +1,24 @@ +package wakuv2 + +import ( + "github.com/libp2p/go-libp2p/core/peer" + "github.com/status-im/status-go/wakuv2/common" + "github.com/waku-org/go-waku/waku/v2/api/history" + "github.com/waku-org/go-waku/waku/v2/protocol" +) + +type HistoryProcessorWrapper struct { + waku *Waku +} + +func NewHistoryProcessorWrapper(waku *Waku) history.HistoryProcessor { + return &HistoryProcessorWrapper{waku} +} + +func (hr *HistoryProcessorWrapper) OnEnvelope(env *protocol.Envelope, processEnvelopes bool) error { + return hr.waku.OnNewEnvelopes(env, common.StoreMessageType, processEnvelopes) +} + +func (hr *HistoryProcessorWrapper) OnRequestFailed(requestID []byte, peerID peer.ID, err error) { + hr.waku.onHistoricMessagesRequestFailed(requestID, peerID, err) +} diff --git a/wakuv2/waku.go b/wakuv2/waku.go index 574381b5e19..1ceea7eda93 100644 --- a/wakuv2/waku.go +++ b/wakuv2/waku.go @@ -36,7 +36,6 @@ import ( "github.com/jellydator/ttlcache/v3" "github.com/libp2p/go-libp2p/core/peer" "github.com/libp2p/go-libp2p/core/peerstore" - "github.com/libp2p/go-libp2p/p2p/protocol/ping" "github.com/multiformats/go-multiaddr" "go.uber.org/zap" @@ -57,6 +56,7 @@ import ( "github.com/libp2p/go-libp2p/core/metrics" filterapi "github.com/waku-org/go-waku/waku/v2/api/filter" + "github.com/waku-org/go-waku/waku/v2/api/history" "github.com/waku-org/go-waku/waku/v2/api/missing" "github.com/waku-org/go-waku/waku/v2/api/publish" "github.com/waku-org/go-waku/waku/v2/dnsdisc" @@ -162,6 +162,9 @@ type Waku struct { onlineChecker *onlinechecker.DefaultOnlineChecker state connection.State + StorenodeCycle *history.StorenodeCycle + HistoryRetriever *history.HistoryRetriever + logger *zap.Logger // NTP Synced timesource @@ -346,6 +349,10 @@ func New(nodeKey *ecdsa.PrivateKey, fleet string, cfg *Config, logger *zap.Logge } waku.options = opts + + waku.StorenodeCycle = history.NewStorenodeCycle(logger) + waku.HistoryRetriever = history.NewHistoryRetriever(waku.node.Store(), NewHistoryProcessorWrapper(waku), logger) + waku.logger.Info("setup the go-waku node successfully") return waku, nil @@ -984,61 +991,6 @@ func (w *Waku) ConfirmMessageDelivered(hashes []gethcommon.Hash) { w.messageSender.MessagesDelivered(hashes) } -func (w *Waku) SetStorePeerID(peerID peer.ID) { - w.messageSender.SetStorePeerID(peerID) -} - -func (w *Waku) Query(ctx context.Context, peerID peer.ID, query store.FilterCriteria, cursor []byte, opts []store.RequestOption, processEnvelopes bool) ([]byte, int, error) { - requestID := protocol.GenerateRequestID() - - opts = append(opts, - store.WithRequestID(requestID), - store.WithPeer(peerID), - store.WithCursor(cursor)) - - logger := w.logger.With(zap.String("requestID", hexutil.Encode(requestID)), zap.Stringer("peerID", peerID)) - - logger.Debug("store.query", - logutils.WakuMessageTimestamp("startTime", query.TimeStart), - logutils.WakuMessageTimestamp("endTime", query.TimeEnd), - zap.Strings("contentTopics", query.ContentTopics.ToList()), - zap.String("pubsubTopic", query.PubsubTopic), - zap.String("cursor", hexutil.Encode(cursor)), - ) - - queryStart := time.Now() - result, err := w.node.Store().Query(ctx, query, opts...) - queryDuration := time.Since(queryStart) - if err != nil { - logger.Error("error querying storenode", zap.Error(err)) - - if w.onHistoricMessagesRequestFailed != nil { - w.onHistoricMessagesRequestFailed(requestID, peerID, err) - } - return nil, 0, err - } - - messages := result.Messages() - envelopesCount := len(messages) - w.logger.Debug("store.query response", zap.Duration("queryDuration", queryDuration), zap.Int("numMessages", envelopesCount), zap.Bool("hasCursor", result.IsComplete() && result.Cursor() != nil)) - for _, mkv := range messages { - msg := mkv.Message - - // Temporarily setting RateLimitProof to nil so it matches the WakuMessage protobuffer we are sending - // See https://github.com/vacp2p/rfc/issues/563 - mkv.Message.RateLimitProof = nil - - envelope := protocol.NewEnvelope(msg, msg.GetTimestamp(), query.PubsubTopic) - - err = w.OnNewEnvelopes(envelope, common.StoreMessageType, processEnvelopes) - if err != nil { - return nil, 0, err - } - } - - return result.Cursor(), envelopesCount, nil -} - // OnNewEnvelope is an interface from Waku FilterManager API that gets invoked when any new message is received by Filter. func (w *Waku) OnNewEnvelope(env *protocol.Envelope) error { return w.OnNewEnvelopes(env, common.RelayedMessageType, false) @@ -1056,6 +1008,8 @@ func (w *Waku) Start() error { return fmt.Errorf("failed to create a go-waku node: %v", err) } + w.StorenodeCycle.Start(w.ctx, w.node.Host()) + w.goingOnline = make(chan struct{}) if err = w.node.Start(w.ctx); err != nil { @@ -1214,7 +1168,7 @@ func (w *Waku) startMessageSender() error { if w.cfg.EnableStoreConfirmationForMessagesSent { msgStoredChan := make(chan gethcommon.Hash, 1000) msgExpiredChan := make(chan gethcommon.Hash, 1000) - messageSentCheck := publish.NewMessageSentCheck(w.ctx, w.node.Store(), w.node.Timesource(), msgStoredChan, msgExpiredChan, w.logger) + messageSentCheck := publish.NewMessageSentCheck(w.ctx, w.node.Store(), w.StorenodeCycle, w.node.Timesource(), msgStoredChan, msgExpiredChan, w.logger) sender.WithMessageSentCheck(messageSentCheck) go func() { @@ -1798,19 +1752,6 @@ func (w *Waku) PeerID() peer.ID { return w.node.Host().ID() } -func (w *Waku) PingPeer(ctx context.Context, peerID peer.ID) (time.Duration, error) { - pingResultCh := ping.Ping(ctx, w.node.Host(), peerID) - select { - case <-ctx.Done(): - return 0, ctx.Err() - case r := <-pingResultCh: - if r.Error != nil { - return 0, r.Error - } - return r.RTT, nil - } -} - func (w *Waku) Peerstore() peerstore.Peerstore { return w.node.Host().Peerstore() }