From acae673a7ecc53e1f0030b1c8512b7bff0124b33 Mon Sep 17 00:00:00 2001 From: daniel1302 Date: Wed, 31 May 2023 16:12:18 +0200 Subject: [PATCH 1/3] fix: implement pagination for accounts and deposits --- difftool/diff/datanode_client.go | 76 +++++++++++++++++++++++++------- difftool/diff/diff_report.go | 10 ++++- 2 files changed, 68 insertions(+), 18 deletions(-) diff --git a/difftool/diff/datanode_client.go b/difftool/diff/datanode_client.go index fa34753..bb3ee97 100644 --- a/difftool/diff/datanode_client.go +++ b/difftool/diff/datanode_client.go @@ -207,15 +207,36 @@ func (dnc *dataNodeClient) Collect() (*Result, error) { } func (dnc *dataNodeClient) listAccounts() ([]*dn.AccountBalance, error) { - accResp, err := dnc.datanode.ListAccounts(context.Background(), &dn.ListAccountsRequest{}) - if err != nil { - return nil, err - } - accounts := make([]*dn.AccountBalance, 0, len(accResp.Accounts.Edges)) - for _, ae := range accResp.Accounts.Edges { - accounts = append(accounts, ae.Node) + result := []*dn.AccountBalance{} + + lastRecordCursor := "" + for { + requestInput := dn.ListAccountsRequest{} + if len(lastRecordCursor) > 0 { + requestInput = dn.ListAccountsRequest{ + Pagination: &dn.Pagination{ + After: &lastRecordCursor, + }, + } + } + + accResp, err := dnc.datanode.ListAccounts(context.Background(), &requestInput) + if err != nil { + return nil, err + } + + for _, ae := range accResp.Accounts.Edges { + result = append(result, ae.Node) + } + + if accResp.Accounts == nil || accResp.Accounts.PageInfo == nil || !accResp.Accounts.PageInfo.HasNextPage { + break + } + + lastRecordCursor = accResp.Accounts.PageInfo.EndCursor } - return accounts, nil + + return result, nil } func (dnc *dataNodeClient) listOrders() ([]*vega.Order, error) { @@ -291,15 +312,36 @@ func (dnc *dataNodeClient) getVegaTime() (int64, error) { } func (dnc *dataNodeClient) listDelegations() ([]*vega.Delegation, error) { - delegationResp, err := dnc.datanode.ListDelegations(context.Background(), &dn.ListDelegationsRequest{}) - if err != nil { - return nil, err - } - delegations := make([]*vega.Delegation, 0, len(delegationResp.Delegations.Edges)) - for _, d := range delegationResp.Delegations.Edges { - delegations = append(delegations, d.Node) + result := []*vega.Delegation{} + + lastRecordCursor := "" + for { + requestInput := dn.ListDelegationsRequest{} + if len(lastRecordCursor) > 0 { + requestInput = dn.ListDelegationsRequest{ + Pagination: &dn.Pagination{ + After: &lastRecordCursor, + }, + } + } + + delegationResp, err := dnc.datanode.ListDelegations(context.Background(), &requestInput) + if err != nil { + return nil, err + } + + for _, d := range delegationResp.Delegations.Edges { + result = append(result, d.Node) + } + + if delegationResp.Delegations == nil || delegationResp.Delegations.PageInfo == nil || !delegationResp.Delegations.PageInfo.HasNextPage { + break + } + + lastRecordCursor = delegationResp.Delegations.PageInfo.EndCursor } - return delegations, nil + + return result, nil } func (dnc *dataNodeClient) getEpoch() (*vega.Epoch, error) { @@ -421,7 +463,7 @@ func (dnc *dataNodeClient) listLiquidityProvisions(market string) ([]*vega.Liqui } lps := make([]*vega.LiquidityProvision, 0, len(resp.LiquidityProvisions.Edges)) for _, lpe := range resp.LiquidityProvisions.Edges { - if lpe.Node.Status == vega.LiquidityProvision_STATUS_PENDING || lpe.Node.Status == vega.LiquidityProvision_STATUS_ACTIVE || lpe.Node.Status == vega.LiquidityProvision_STATUS_UNDEPLOYED { + if lpe.Node.Status == vega.LiquidityProvision_STATUS_PENDING || lpe.Node.Status == vega.LiquidityProvision_STATUS_ACTIVE || lpe.Node.Status == vega.LiquidityProvision_STATUS_UNDEPLOYED { lps = append(lps, lpe.Node) } } diff --git a/difftool/diff/diff_report.go b/difftool/diff/diff_report.go index 2ef24fb..036e5f6 100644 --- a/difftool/diff/diff_report.go +++ b/difftool/diff/diff_report.go @@ -520,7 +520,15 @@ func diffTransfers(coreSnapshot *Result, dn *Result) Status { } for i, a := range core { - d := datanode[i] + // shadow to make sure the original data is not changed + a := *a + d := *(datanode[i]) + + // We ignore the difference in the timestamp, as the transfer timestamp + // is updated when the network is started from the checkpoint. + a.Timestamp = 0 + d.Timestamp = 0 + if a.String() != d.String() { return getValueMismatchStatus("transfers", core, datanode) } From 3fd17d52dc5ad5983aacd7c85e7f65d2cc39b060 Mon Sep 17 00:00:00 2001 From: daniel1302 Date: Thu, 1 Jun 2023 19:52:21 +0200 Subject: [PATCH 2/3] feat: fix the linting issue for the difftool --- difftool/diff/diff_report.go | 15 +++++++++++---- 1 file changed, 11 insertions(+), 4 deletions(-) diff --git a/difftool/diff/diff_report.go b/difftool/diff/diff_report.go index 036e5f6..8af97a9 100644 --- a/difftool/diff/diff_report.go +++ b/difftool/diff/diff_report.go @@ -520,16 +520,23 @@ func diffTransfers(coreSnapshot *Result, dn *Result) Status { } for i, a := range core { - // shadow to make sure the original data is not changed - a := *a - d := *(datanode[i]) + d := datanode[i] + coreTransferTimestamp := a.Timestamp + dnTransferTimestamp := d.Timestamp // We ignore the difference in the timestamp, as the transfer timestamp // is updated when the network is started from the checkpoint. a.Timestamp = 0 d.Timestamp = 0 - if a.String() != d.String() { + transfetMatch := a.String() == d.String() + + // restore the original value as copy is not trivial due to Mutex + // inside of the proto message + a.Timestamp = coreTransferTimestamp + d.Timestamp = dnTransferTimestamp + + if !transfetMatch { return getValueMismatchStatus("transfers", core, datanode) } } From b58bf6b16608da6338951fe797884403196f8e5d Mon Sep 17 00:00:00 2001 From: Daniel Date: Thu, 1 Jun 2023 21:19:51 +0200 Subject: [PATCH 3/3] feat: update changelog --- CHANGELOG.md | 1 + 1 file changed, 1 insertion(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index b69f396..bb6bae3 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -81,6 +81,7 @@ - [173](https://github.com/vegaprotocol/vegatools/issues/173) - Add `MarketTracker` to checkpoint parser - [262](https://github.com/vegaprotocol/vegatools/issues/262) - Update vega dependency, ignore slippage factors in market and last block in epoch - [279](https://github.com/vegaprotocol/vegatools/issues/279) - Update to work with new datanode API for querying live orders / liquidity provisions +- [286](https://github.com/vegaprotocol/vegatools/pull/286) - Add support for pagination in the accounts and delegations queries to data-node ## 0.41.1 *2021-08-31*