From d121441dae5ee0b12618ed1b1631b81b71eb7d88 Mon Sep 17 00:00:00 2001 From: daniel1302 Date: Thu, 17 Nov 2022 00:34:15 +0100 Subject: [PATCH 01/10] feat: refactor price proxy to make it more readable and more optimal --- config/config.go | 45 +++++-- config/config_test.go | 4 +- pricing/bitstamp.go | 68 ----------- pricing/coingecko.go | 119 +++++++++---------- pricing/coinmarketcap.go | 249 ++++++++++++++++++++++++++++----------- pricing/ftx.go | 90 -------------- pricing/pricing.go | 194 +++++------------------------- service/service.go | 42 ++----- utils/slice.go | 11 ++ 9 files changed, 328 insertions(+), 494 deletions(-) delete mode 100644 pricing/bitstamp.go delete mode 100644 pricing/ftx.go create mode 100644 utils/slice.go diff --git a/config/config.go b/config/config.go index 4336e7f..247f0e4 100644 --- a/config/config.go +++ b/config/config.go @@ -6,6 +6,7 @@ import ( "errors" "fmt" "net/url" + "strings" "time" log "github.com/sirupsen/logrus" @@ -21,29 +22,45 @@ type ServerConfig struct { // PriceConfig describes one price setting, which uses one source. type PriceConfig struct { - Source string `yaml:"source"` - Base string `yaml:"base"` - Quote string `yaml:"quote"` - Factor float64 `yaml:"factor"` - Wander bool `yaml:"wander"` + Source string `yaml:"source"` + Base string `yaml:"base"` + Quote string `yaml:"quote"` + QuoteOverride string `yaml:"quote_override"` + Factor float64 `yaml:"factor"` + Wander bool `yaml:"wander"` } // SourceConfig describes one source setting (e.g. one API endpoint). // The URL has "{base}" and "{quote}" replaced at runtime with entries from PriceConfig. type SourceConfig struct { - Name string `yaml:"name"` - URL url.URL `yaml:"url"` - SleepReal int `yaml:"sleepReal"` - SleepWander int `yaml:"sleepWander"` + Name string `yaml:"name"` + URL url.URL `yaml:"url"` + AuthKeyEnvName string `yaml:"auth_key_env_name"` + SleepReal int `yaml:"sleepReal"` + SleepWander int `yaml:"sleepWander"` } +type PriceList []PriceConfig + // Config describes the top level config file format. type Config struct { Server *ServerConfig `yaml:"server"` - Prices []*PriceConfig `yaml:"prices"` + Prices PriceList `yaml:"prices"` Sources []*SourceConfig `yaml:"sources"` } +func (pl PriceList) GetBySource(source string) PriceList { + result := PriceList{} + + for _, price := range pl { + if price.Source == source { + result = append(result, price) + } + } + + return result +} + var ( // ErrNil indicates that a nil/null pointer was encountered. ErrNil = errors.New("nil pointer") @@ -147,3 +164,11 @@ func (ps SourceConfig) String() string { return fmt.Sprintf("{SourceConfig Name:%s URL:%s SleepReal:%ds SleepWander:%ds}", ps.Name, ps.URL.String(), ps.SleepReal, ps.SleepWander) } + +func (ps SourceConfig) IsCoinGecko() bool { + return strings.Contains(ps.URL.Host, "coingecko.com") +} + +func (ps SourceConfig) IsCoinMarketCap() bool { + return strings.Contains(ps.URL.Host, "coinmarketcap.com") +} diff --git a/config/config_test.go b/config/config_test.go index d8647ed..2550c4d 100644 --- a/config/config_test.go +++ b/config/config_test.go @@ -43,11 +43,11 @@ func TestCheckConfig(t *testing.T) { err = config.CheckConfig(&cfg) assert.True(t, strings.HasPrefix(err.Error(), config.ErrMissingEmptyConfigSection.Error())) - cfg.Prices = []*config.PriceConfig{} + cfg.Prices = []config.PriceConfig{} err = config.CheckConfig(&cfg) assert.True(t, strings.HasPrefix(err.Error(), config.ErrMissingEmptyConfigSection.Error())) - cfg.Prices = append(cfg.Prices, &config.PriceConfig{}) + cfg.Prices = append(cfg.Prices, config.PriceConfig{}) err = config.CheckConfig(&cfg) assert.True(t, strings.HasPrefix(err.Error(), config.ErrInvalidValue.Error())) diff --git a/pricing/bitstamp.go b/pricing/bitstamp.go deleted file mode 100644 index df5fd2c..0000000 --- a/pricing/bitstamp.go +++ /dev/null @@ -1,68 +0,0 @@ -package pricing - -import ( - "encoding/json" - "fmt" - "io/ioutil" - "net/http" - "strconv" - "time" - - "code.vegaprotocol.io/priceproxy/config" - "github.com/pkg/errors" -) - -type bitstampResponse struct { - High string `json:"high"` - Last string `json:"last"` - Timestamp string `json:"timestamp"` - Bid string `json:"bid"` - Vwap string `json:"vwap"` - Volume string `json:"volume"` - Low string `json:"low"` - Ask string `json:"ask"` - Open string `json:"open"` -} - -var _ fetchPriceFunc = getPriceBitStamp - -func getPriceBitStamp(pricecfg config.PriceConfig, sourcecfg config.SourceConfig, client *http.Client, req *http.Request) (PriceInfo, error) { - resp, err := client.Do(req) - if err != nil { - return PriceInfo{}, errors.Wrap(err, "failed to perform HTTP request") - } - defer resp.Body.Close() - - content, err := ioutil.ReadAll(resp.Body) - if err != nil { - return PriceInfo{}, errors.Wrap(err, "failed to read HTTP response body") - } - - if resp.StatusCode != http.StatusOK { - return PriceInfo{}, fmt.Errorf("bitstamp returned HTTP %d (%s)", resp.StatusCode, string(content)) - } - - var response bitstampResponse - if err = json.Unmarshal(content, &response); err != nil { - return PriceInfo{}, errors.Wrap(err, "failed to parse HTTP response as JSON") - } - - if response.Last == "" { - return PriceInfo{}, errors.New("bitstamp returned an empty Last price") - } - - var p float64 - p, err = strconv.ParseFloat(response.Last, 64) - if err != nil { - return PriceInfo{}, err - } - if p <= 0.0 { - return PriceInfo{}, fmt.Errorf("bitstamp returned zero/negative price: %f", p) - } - t := time.Now().Round(0) - return PriceInfo{ - LastUpdatedReal: t, - LastUpdatedWander: t, - Price: p * pricecfg.Factor, - }, nil -} diff --git a/pricing/coingecko.go b/pricing/coingecko.go index 39baf4d..f555a34 100644 --- a/pricing/coingecko.go +++ b/pricing/coingecko.go @@ -14,19 +14,10 @@ import ( ) var ( - coingeckoExtraPairs = []config.PriceConfig{} coingeckoSourceName = "coingecko" + supportedQuotes = []string{"ETH", "EUR", "USD", "BTC", "DAI"} ) -func coingeckoAddExtraPriceConfig(priceconfig config.PriceConfig) error { - coingeckoExtraPairs = append(coingeckoExtraPairs, priceconfig) - return nil -} - -type priceBoard interface { - UpdatePrice(pricecfg config.PriceConfig, newPrice PriceInfo) -} - func coingeckoStartFetching( board priceBoard, sourcecfg config.SourceConfig, @@ -38,11 +29,13 @@ func coingeckoStartFetching( ctx = context.Background() err error ) + log.WithFields(log.Fields{ "sourceName": coingeckoSourceName, "URL": fetchURL, "rateLimitDuration": oneRequestEvery, }).Infof("Starting Coingecko Fetching\n") + for { if err = rateLimiter.Wait(ctx); err != nil { log.WithFields(log.Fields{ @@ -66,60 +59,63 @@ func coingeckoStartFetching( continue } - for base, data := range *prices { - board.UpdatePrice( - config.PriceConfig{ - Source: coingeckoSourceName, - Base: base, - Quote: "ETH", - Factor: 1.0, - Wander: true, - }, - PriceInfo{ - Price: data.ETH, - LastUpdatedReal: time.Unix(int64(data.LastUpdatedAt), 0), - LastUpdatedWander: time.Now().Round(0), - }, - ) - } + for _, price := range board.PriceList(sourcecfg.Name) { + priceUpdated := false + CoinGeckoLoop: + for coingeckoBase, coingeckoData := range *prices { + if price.Base == coingeckoBase { + var fetchedPrice float64 - for _, extraPair := range coingeckoExtraPairs { - base, ok := (*prices)[extraPair.Base] - if !ok { - log.WithFields(log.Fields{ - "sourceName": coingeckoSourceName, - "URL": fetchURL, - "rateLimitDuration": oneRequestEvery, - }).Errorf("Failed to get base %s for extra pair %v\n", extraPair.Base, extraPair) - continue - } - var price float64 - if strings.EqualFold(extraPair.Quote, "EUR") { - price = base.EUR - } else if strings.EqualFold(extraPair.Quote, "USD") { - price = base.USD - } else if strings.EqualFold(extraPair.Quote, "BTC") { - price = base.BTC - } else { - quote, ok := (*prices)[extraPair.Quote] - if !ok { - log.WithFields(log.Fields{ - "sourceName": coingeckoSourceName, - "URL": fetchURL, - "rateLimitDuration": oneRequestEvery, - }).Errorf("Failed to get quote %s for extra pair %v\n", extraPair.Source, extraPair) - continue + switch strings.ToUpper(price.Quote) { + case "ETH": + fetchedPrice = coingeckoData.ETH + case "BTC": + fetchedPrice = coingeckoData.BTC + case "USD": + fetchedPrice = coingeckoData.USD + case "EUR": + fetchedPrice = coingeckoData.EUR + case "DAI": + fetchedPrice = coingeckoData.DAI + default: + log.WithFields(log.Fields{ + "sourceName": coingeckoSourceName, + "base": price.Base, + "quote": price.Quote, + "quote_override": price.QuoteOverride, + }).Errorf("price quote is invalid, got %s, expecting one of %v", price.Quote, supportedQuotes) + break CoinGeckoLoop + } + + if fetchedPrice == 0 { + log.WithFields(log.Fields{ + "sourceName": coingeckoSourceName, + "base": price.Base, + "quote": price.Quote, + "quote_override": price.QuoteOverride, + }).Warnf("collected price in the quote current is 0, consider selecting different quote and overwrite it with the `quote_override` parameter") + } + + board.UpdatePrice( + price, + PriceInfo{ + Price: fetchedPrice, + LastUpdatedReal: time.Unix(int64(coingeckoData.LastUpdatedAt), 0), + LastUpdatedWander: time.Now().Round(0), + }, + ) + priceUpdated = true } - price = base.USD / quote.USD } - board.UpdatePrice( - extraPair, - PriceInfo{ - Price: price, - LastUpdatedReal: time.Unix(int64(base.LastUpdatedAt), 0), - LastUpdatedWander: time.Now().Round(0), - }, - ) + + if !priceUpdated { + log.WithFields(log.Fields{ + "sourceName": coingeckoSourceName, + "base": price.Base, + "quote": price.Quote, + "quote_override": price.QuoteOverride, + }).Errorf("price not found in the coingecko API") + } } } } @@ -129,6 +125,7 @@ type coingeckoFetchData map[string]struct { EUR float64 `json:"eur"` BTC float64 `json:"btc"` ETH float64 `json:"eth"` + DAI float64 `json:"dai"` LastUpdatedAt uint64 `json:"last_updated_at"` } diff --git a/pricing/coinmarketcap.go b/pricing/coinmarketcap.go index 21761f7..d74fc21 100644 --- a/pricing/coinmarketcap.go +++ b/pricing/coinmarketcap.go @@ -1,102 +1,217 @@ package pricing import ( + "context" "encoding/json" "fmt" - "io/ioutil" "net/http" "os" "strings" "time" "code.vegaprotocol.io/priceproxy/config" - "github.com/pkg/errors" + log "github.com/sirupsen/logrus" + "golang.org/x/time/rate" ) -type cmcStatusResponse struct { - Timestamp string `json:"timestamp"` - ErrorCode int `json:"error_code"` - ErrorMessage string `json:"error_message"` - Elapsed int `json:"elapsed"` - CreditCount int `json:"credit_count"` - Notice string `json:"notice"` -} +func coinmarketcapStartFetching( + board priceBoard, + sourcecfg config.SourceConfig, +) { + var ( + fetchURL = sourcecfg.URL + oneRequestEvery = time.Duration(sourcecfg.SleepReal) * time.Second + rateLimiter = rate.NewLimiter(rate.Every(oneRequestEvery), 1) + ctx = context.Background() + err error + ) + + apiKey := "" + if sourcecfg.AuthKeyEnvName != "" { + apiKey = os.Getenv(sourcecfg.AuthKeyEnvName) + } + + if apiKey == "" { + log.WithFields(log.Fields{ + "sourceName": coingeckoSourceName, + "URL": sourcecfg.URL, + "AuthKeyEnvName": sourcecfg.AuthKeyEnvName, + }).Warnf("The API key is empty. Use the `auth_key_env_name` config for the source and export corresponding environment name") + } + + fetchURLQuery := fetchURL.Query() + fetchURLQuery.Add("CMC_PRO_API_KEY", apiKey) + fetchURL.RawQuery = fetchURLQuery.Encode() + + for { + if err = rateLimiter.Wait(ctx); err != nil { + log.WithFields(log.Fields{ + "error": err.Error(), + "sourceName": sourcecfg.Name, + "URL": sourcecfg.URL.String(), + "rateLimitDuration": oneRequestEvery, + }).Errorln("Rate Limiter Failed. Falling back to Sleep.") + // fallback + time.Sleep(oneRequestEvery) + } + + coinmarketcapData, err := coinmarketcapSingleFetch(fetchURL.String()) + if err != nil { + log.WithFields(log.Fields{ + "error": err.Error(), + "sourceName": sourcecfg.Name, + "URL": sourcecfg.URL.String(), + "rateLimitDuration": oneRequestEvery, + }).Errorln("failed to get trading data.") + } + + for _, price := range board.PriceList(sourcecfg.Name) { + fetchedCurrency := coinmarketcapData.GetCurrency(price.Base) + if fetchedCurrency == nil { + log.WithFields(log.Fields{ + "sourceName": coingeckoSourceName, + "base": price.Base, + "quote": price.Quote, + "quote_override": price.QuoteOverride, + }).Errorln("price not returned from the API") + continue + } -type cmcQuoteResponse struct { - Price float64 `json:"price"` - Volume24h float64 `json:"volume_24h"` - PercentChange1h float64 `json:"percent_change_1h"` - PercentChange24h float64 `json:"percent_change_24h"` - PercentChange7d float64 `json:"percent_change_7d"` - MarketCap float64 `json:"market_cap"` - LastUpdated string `json:"last_updated"` + fetchedQuote := fetchedCurrency.QuoteByName(price.Quote) + fetchedPrice := 0.0 + fetchedLastUpdate := fetchedCurrency.LastUpdated + if fetchedQuote == nil { + log.WithFields(log.Fields{ + "sourceName": coingeckoSourceName, + "base": price.Base, + "quote": price.Quote, + "quote_override": price.QuoteOverride, + }).Warnf("collected price in the quote current is 0, consider selecting different quote and overwrite it with the `quote_override` parameter") + } else { + fetchedPrice = fetchedQuote.Price + fetchedLastUpdate = fetchedQuote.LastUpdated + } + + parsedTime, err := time.Parse(time.RFC3339, fetchedLastUpdate) + if err != nil { + log.WithFields(log.Fields{ + "error": err.Error(), + "sourceName": coingeckoSourceName, + "base": price.Base, + "quote": price.Quote, + "quote_override": price.QuoteOverride, + "last_updated_time": fetchedLastUpdate, + }).Warnf("cannot parse fetched last_updated time with the ISO8601 format") + } + + if fetchedPrice == 0 { + log.WithFields(log.Fields{ + "sourceName": coingeckoSourceName, + "base": price.Base, + "quote": price.Quote, + "quote_override": price.QuoteOverride, + }).Debug("Quote/Base rate not found directly, trying conversion") + + fetchedPrice = coinmarketcapData.ConvertPrice(price.Base, price.Quote) + } + + if fetchedPrice == 0 { + log.WithFields(log.Fields{ + "sourceName": coingeckoSourceName, + "base": price.Base, + "quote": price.Quote, + "quote_override": price.QuoteOverride, + }).Warnf("collected price in the quote current is 0, consider selecting different quote and overwrite it with the `quote_override` parameter") + } + + board.UpdatePrice( + price, + PriceInfo{ + Price: fetchedPrice, + LastUpdatedReal: parsedTime, + LastUpdatedWander: time.Now().Round(0), + }, + ) + } + } } -type cmcDataResponse struct { - ID int `json:"id"` - Name string `json:"name"` - Symbol string `json:"symbol"` - Slug string `json:"slug"` - IsActive int `json:"is_active"` - LastUpdated string `json:"last_updated"` - Quote map[string]cmcQuoteResponse `json:"quote"` +type coinmarketcapQuoteData struct { + Price float64 `json:"price"` + LastUpdated string `json:"last_updated"` } -type cmcResponse struct { - Status cmcStatusResponse `json:"status"` - Data map[string]cmcDataResponse `json:"data"` +type coinmarketcapCurrencyData struct { + Name string `json:"name"` + Symbol string `json:"symbol"` + Slug string `json:"slug"` + LastUpdated string `json:"last_updated"` + + Quote map[string]coinmarketcapQuoteData `json:"quote"` } -func headersCoinmarketcap() (map[string][]string, error) { - headers := make(map[string][]string, 1) +type coinmarketcapFetchData struct { + Data []coinmarketcapCurrencyData `json:"data"` +} - fn := os.ExpandEnv("$HOME/coinmarketcap-apikey.txt") - apiKey, err := ioutil.ReadFile(fn) // #nosec G304 - if err != nil { - return nil, errors.Wrap(err, fmt.Sprintf("failed to read API key file %s", fn)) +func (data coinmarketcapFetchData) GetCurrency(name string) *coinmarketcapCurrencyData { + for _, currencyData := range data.Data { + if strings.EqualFold(currencyData.Name, name) || strings.EqualFold(currencyData.Slug, name) || strings.EqualFold(currencyData.Symbol, name) { + return ¤cyData + } } - headers["X-CMC_PRO_API_KEY"] = []string{string(apiKey)} - return headers, nil + + return nil } -func getPriceCoinmarketcap(pricecfg config.PriceConfig, sourcecfg config.SourceConfig, client *http.Client, req *http.Request) (PriceInfo, error) { - if strings.HasPrefix(pricecfg.Quote, "XYZ") { - // Inject a hidden price config, for competitions. - pricecfg.Base = "" - pricecfg.Quote = "" - } - resp, err := client.Do(req) - if err != nil { - return PriceInfo{}, errors.Wrap(err, "failed to perform HTTP request") +func (currency coinmarketcapCurrencyData) QuoteByName(name string) *coinmarketcapQuoteData { + for qName, qData := range currency.Quote { + if strings.EqualFold(qName, name) { + return &qData + } } - defer resp.Body.Close() + return nil +} - content, err := ioutil.ReadAll(resp.Body) - if err != nil { - return PriceInfo{}, errors.Wrap(err, "failed to read HTTP response body") +func (data coinmarketcapFetchData) ConvertPrice(base, quote string) float64 { + baseCurrency := data.GetCurrency(base) + quoteCurrency := data.GetCurrency(quote) + + if baseCurrency == nil || quoteCurrency == nil { + return 0.0 } - if resp.StatusCode != http.StatusOK { - return PriceInfo{}, fmt.Errorf("got HTTP %d (%s)", resp.StatusCode, string(content)) + for qName, qData := range baseCurrency.Quote { + // try luck with direct conversion + if strings.EqualFold(qName, baseCurrency.Name) || strings.EqualFold(qName, baseCurrency.Slug) || strings.EqualFold(qName, baseCurrency.Symbol) { + return qData.Price + } + + // conversion by common currency(e.g USD) + if commonCurrency := quoteCurrency.QuoteByName(qName); commonCurrency != nil { + return qData.Price / commonCurrency.Price + } + + // todo: search by ConvertPrice(quote, qName), but may create deadlock... + // so we have to impleent timeout + error mechanism here } - var response cmcResponse - if err = json.Unmarshal(content, &response); err != nil { - return PriceInfo{}, errors.Wrap(err, "failed to parse HTTP response as JSON") + return data.ConvertPrice(quote, base) +} + +func coinmarketcapSingleFetch(url string) (*coinmarketcapFetchData, error) { + resp, err := http.Get(url) + if err != nil { + return nil, fmt.Errorf("failed to get coinmarketcap data, %w", err) } - data, ok := response.Data[pricecfg.Base] - if !ok { - return PriceInfo{}, fmt.Errorf("failed to find Base in response: %s", pricecfg.Base) + if resp.StatusCode != http.StatusOK { + return nil, fmt.Errorf("failed to get coinmarketcap data: expected status 200, got %d", resp.StatusCode) } - quote, ok := data.Quote[pricecfg.Quote] - if !ok { - return PriceInfo{}, fmt.Errorf("failed to find Quote in response: %s", pricecfg.Quote) + defer resp.Body.Close() + var prices coinmarketcapFetchData + if err = json.NewDecoder(resp.Body).Decode(&prices); err != nil { + return nil, fmt.Errorf("failed to parse coinmarketcap data, %w", err) } - t := time.Now().Round(0) - return PriceInfo{ - LastUpdatedReal: t, - LastUpdatedWander: t, - Price: quote.Price * pricecfg.Factor, - }, nil + return &prices, nil } diff --git a/pricing/ftx.go b/pricing/ftx.go deleted file mode 100644 index 8c048e4..0000000 --- a/pricing/ftx.go +++ /dev/null @@ -1,90 +0,0 @@ -package pricing - -// Source: https://ftx.com/ -// Docs: https://docs.ftx.com/ - -import ( - "encoding/json" - "fmt" - "io/ioutil" - "net/http" - "time" - - "code.vegaprotocol.io/priceproxy/config" - "github.com/pkg/errors" -) - -type ftxResultResponse struct { - Ask float64 `json:"ask"` - BaseCurrency string `json:"baseCurrency"` - Bid float64 `json:"symbol"` - Change1h float64 `json:"change1h"` - Change24h float64 `json:"change24h"` - ChangeBod float64 `json:"changeBod"` - Enabled bool `json:"enabled"` - HighLeverageFeeExempt bool `json:"highLeverageFeeExempt"` - Last float64 `json:"last"` - MinProvideSize float64 `json:"minProvideSize"` - Name string `json:"name"` - PostOnly bool `json:"postOnly"` - Price float64 `json:"price"` - PriceIncrement float64 `json:"priceIncrement"` - QuoteCurrency string `json:"quoteCurrency"` - QuoteVolume24h float64 `json:"quoteVolume24h"` - Restricted bool `json:"restricted"` - SizeIncrement float64 `json:"sizeIncrement"` - Type string `json:"type"` - Underlying string `json:"underlying"` - VolumeUSD24h float64 `json:"volumeUsd24h"` -} - -type ftxResponse struct { - Result ftxResultResponse `json:"result"` - Success bool `json:"success"` -} - -var _ fetchPriceFunc = getPriceFTX - -func getPriceFTX(pricecfg config.PriceConfig, sourcecfg config.SourceConfig, client *http.Client, req *http.Request) (PriceInfo, error) { - resp, err := client.Do(req) - if err != nil { - return PriceInfo{}, err - } - defer resp.Body.Close() - - content, err := ioutil.ReadAll(resp.Body) - if err != nil { - return PriceInfo{}, errors.Wrap(err, "failed to read HTTP response body") - } - - if resp.StatusCode != http.StatusOK { - return PriceInfo{}, fmt.Errorf("ftx.com returned HTTP %d (%s)", resp.StatusCode, string(content)) - } - - var response ftxResponse - // if err = json.NewDecoder(resp.Body).Decode(&response); err != nil {...} - if err = json.Unmarshal(content, &response); err != nil { - return PriceInfo{}, errors.Wrap(err, "failed to parse HTTP response as JSON") - } - if !response.Success { - return PriceInfo{}, fmt.Errorf("ftx.com returned success=false") - } - - price := response.Result.Price - - if price <= 0.0 { - // Sometimes null/zero. - price = response.Result.Last - } - - if price <= 0.0 { - return PriceInfo{}, fmt.Errorf("ftx.com returned zero/negative for Price and Last") - } - - t := time.Now().Round(0) - return PriceInfo{ - LastUpdatedReal: t, - LastUpdatedWander: t, - Price: price * pricecfg.Factor, - }, nil -} diff --git a/pricing/pricing.go b/pricing/pricing.go index 94e1221..4dc89e7 100644 --- a/pricing/pricing.go +++ b/pricing/pricing.go @@ -1,18 +1,13 @@ package pricing import ( - "context" "fmt" - "math" - "math/rand" - "net/http" "net/url" "strings" "sync" "time" "code.vegaprotocol.io/priceproxy/config" - log "github.com/sirupsen/logrus" ) const minPrice = 0.00001 @@ -27,36 +22,41 @@ type PriceInfo struct { } // Engine is the source of price information from multiple external/internal/fake sources. +// //go:generate go run github.com/golang/mock/mockgen -destination mocks/engine_mock.go -package mocks code.vegaprotocol.io/priceproxy/pricing Engine type Engine interface { AddSource(sourcecfg config.SourceConfig) error GetSource(name string) (config.SourceConfig, error) GetSources() ([]config.SourceConfig, error) - AddPrice(pricecfg config.PriceConfig) error - WaitForPrice(pricecfg config.PriceConfig) PriceInfo + PriceList(source string) config.PriceList GetPrice(pricecfg config.PriceConfig) (PriceInfo, error) GetPrices() map[config.PriceConfig]PriceInfo UpdatePrice(pricecfg config.PriceConfig, newPrice PriceInfo) - StartFetching() + StartFetching() error +} + +type priceBoard interface { + PriceList(source string) config.PriceList + UpdatePrice(pricecfg config.PriceConfig, newPrice PriceInfo) } type engine struct { - prices map[config.PriceConfig]PriceInfo - pricesMu sync.Mutex + priceList config.PriceList + prices map[config.PriceConfig]PriceInfo + pricesMu sync.Mutex sources map[string]config.SourceConfig sourcesMu sync.Mutex } -type fetchPriceFunc func(pricecfg config.PriceConfig, sourcecfg config.SourceConfig, client *http.Client, req *http.Request) (PriceInfo, error) - // NewEngine creates a new pricing engine. -func NewEngine() Engine { +func NewEngine(prices config.PriceList) Engine { e := engine{ - prices: make(map[config.PriceConfig]PriceInfo), - sources: make(map[string]config.SourceConfig), + priceList: prices, + prices: make(map[config.PriceConfig]PriceInfo), + sources: make(map[string]config.SourceConfig), } return &e } @@ -106,61 +106,6 @@ func (e *engine) GetSources() ([]config.SourceConfig, error) { return response, nil } -func (e *engine) AddPrice(pricecfg config.PriceConfig) error { - e.pricesMu.Lock() - _, found := e.prices[pricecfg] - e.pricesMu.Unlock() - if found { - return fmt.Errorf("price already exists: %s", pricecfg.String()) - } - - source, err := e.GetSource(pricecfg.Source) - if err != nil { - return fmt.Errorf("failed to get price source for %s: %w", pricecfg.Source, err) - } - - headers := map[string][]string{} - - if source.Name == "bitstamp" { - go e.stream(pricecfg, source, nil, headers, getPriceBitStamp) - } else if source.Name == "coinmarketcap" { - headers, err = headersCoinmarketcap() - if err != nil { - return fmt.Errorf("failed to create HTTP headers for %s: %w", source.Name, err) - } - go e.stream(pricecfg, source, nil, headers, getPriceCoinmarketcap) - } else if strings.HasPrefix(source.Name, "ftx-") { - go e.stream(pricecfg, source, nil, headers, getPriceFTX) - } else if source.Name == coingeckoSourceName { - coingeckoAddExtraPriceConfig(pricecfg) - } else { - return fmt.Errorf("no source for %s", source.Name) - } - return nil -} - -func (e *engine) WaitForPrice(pricecfg config.PriceConfig) PriceInfo { - sublog := log.WithFields(log.Fields{ - "priceConfig": pricecfg.String(), - }) - - sublog.Debug("Waiting for first price") - s := 10 // milliseconds - for { - pi, err := e.GetPrice(pricecfg) - if err != nil { - sublog.WithFields(log.Fields{"err": err.Error()}).Debug("Waiting for first price") - } else { - sublog.WithFields(log.Fields{"price": pi.Price}).Debug("Got first price") - if pi.Price > 0.0 { - return pi - } - } - time.Sleep(time.Duration(s) * time.Millisecond) - s *= 2 - } -} - func (e *engine) GetPrice(pricecfg config.PriceConfig) (PriceInfo, error) { e.pricesMu.Lock() defer e.pricesMu.Unlock() @@ -189,110 +134,25 @@ func (e *engine) UpdatePrice(pricecfg config.PriceConfig, newPrice PriceInfo) { e.pricesMu.Unlock() } -func (e *engine) StartFetching() { - if sourcecfg, ok := e.sources[coingeckoSourceName]; ok { - go coingeckoStartFetching(e, sourcecfg) - } +func (e *engine) PriceList(source string) config.PriceList { + return e.priceList.GetBySource(source) } -func (e *engine) stream(pricecfg config.PriceConfig, sourcecfg config.SourceConfig, u *url.URL, headers map[string][]string, fetchPrice fetchPriceFunc) { - if u == nil { - p2 := config.PriceConfig{ - Base: pricecfg.Base, - Quote: pricecfg.Quote, - Wander: pricecfg.Wander, +func (e *engine) StartFetching() error { + for _, sourceConfig := range e.sources { + if sourceConfig.IsCoinGecko() { + go coingeckoStartFetching(e, sourceConfig) + continue } - if strings.HasPrefix(p2.Quote, "XYZ") { - // Inject a hidden price config, for competitions. - p2.Base = "" - p2.Quote = "" - } - u = urlWithBaseQuote(sourcecfg.URL, p2) - } - sublog := log.WithFields(log.Fields{ - "base": pricecfg.Base, - "quote": pricecfg.Quote, - "source": sourcecfg.Name, - "source-url": u.String(), - }) - - annualisedSleepReal := float64(sourcecfg.SleepReal) / 365.25 / 86400.0 - kappa := 1.0 / annualisedSleepReal - - client := http.Client{} - req, err := http.NewRequestWithContext(context.Background(), http.MethodGet, u.String(), nil) - if err != nil { - sublog.WithFields(log.Fields{ - "error": err.Error(), - }, - ).Fatal("Failed to create HTTP request") - } - for headerName, headerValueList := range headers { - for _, headerValue := range headerValueList { - req.Header.Add(headerName, headerValue) + if sourceConfig.IsCoinMarketCap() { + go coinmarketcapStartFetching(e, sourceConfig) + continue } - } - - var rpi, realPriceInfo, priceInfo PriceInfo - // Get price for the first time - for err != nil || realPriceInfo.Price == 0 { - realPriceInfo, err = fetchPrice(pricecfg, sourcecfg, &client, req) - if err != nil { - sublog.WithFields(log.Fields{ - "error": err.Error(), - }).Debug("Failed to fetch real price for the first time") - } - time.Sleep(time.Duration(sourcecfg.SleepWander) * time.Second) + return fmt.Errorf("source %s not supported", sourceConfig.String()) } - e.UpdatePrice(pricecfg, realPriceInfo) - sublog.WithFields(log.Fields{ - "realPriceInfo": realPriceInfo.String(), - }).Debug("Fetched real price for the first time") - for { - cutoff := time.Now().Round(0).Add(time.Duration(-sourcecfg.SleepReal) * time.Second) - if realPriceInfo.LastUpdatedReal.Before(cutoff) { - rpi, err = fetchPrice(pricecfg, sourcecfg, &client, req) - if err == nil { - realPriceInfo = rpi - e.UpdatePrice(pricecfg, realPriceInfo) - sublog.WithFields(log.Fields{ - "realPriceInfo": realPriceInfo.String(), - }).Debug("Fetched real price") - } else { - sublog.WithFields(log.Fields{ - "error": err.Error(), - }).Warning("Failed to fetch real price") - } - } - - if pricecfg.Wander { - priceInfo, err = e.GetPrice(pricecfg) - if err == nil { - // make the price wander - sigma := 1.0 - wander := kappa*(realPriceInfo.Price-priceInfo.Price)*annualisedSleepReal + sigma*priceInfo.Price*math.Sqrt(annualisedSleepReal)*rand.NormFloat64() - priceInfo.Price += wander - if priceInfo.Price < minPrice { - priceInfo.Price = minPrice - } - priceInfo.LastUpdatedWander = time.Now().Round(0) - e.UpdatePrice(pricecfg, priceInfo) - sublog.WithFields(log.Fields{ - "kappa": kappa, - "sigma": sigma, - "wander": wander, - "newPrice": priceInfo.String(), - }).Debug("Wandered price") - } else { - sublog.WithFields(log.Fields{ - "error": err.Error(), - }).Warning("Failed to fetch price") - } - } - time.Sleep(time.Duration(sourcecfg.SleepWander) * time.Second) - } + return nil } func (pi PriceInfo) String() string { diff --git a/service/service.go b/service/service.go index 21f8d27..dba347d 100644 --- a/service/service.go +++ b/service/service.go @@ -34,6 +34,7 @@ type PriceResponse struct { Source string `json:"source"` Base string `json:"base"` Quote string `json:"quote"` + QuoteReal string `json:"quote_real"` Price float64 `json:"price"` LastUpdatedReal string `json:"lastUpdatedReal"` LastUpdatedWander string `json:"lastUpdatedWander"` @@ -106,7 +107,7 @@ func (s *Service) Stop() { } func (s *Service) initPricingEngine() error { - s.pe = pricing.NewEngine() + s.pe = pricing.NewEngine(s.config.Prices) for _, sourcecfg := range s.config.Sources { err := s.pe.AddSource(*sourcecfg) if err != nil { @@ -126,32 +127,8 @@ func (s *Service) initPricingEngine() error { }).Info("Added source") } - for _, pricecfg := range s.config.Prices { - err := s.pe.AddPrice(*pricecfg) - if err != nil { - log.WithFields(log.Fields{ - "error": err.Error(), - "source": pricecfg.Source, - "base": pricecfg.Base, - "quote": pricecfg.Quote, - "wander": pricecfg.Wander, - }).Fatal("Failed to add price") - } - log.WithFields(log.Fields{ - "source": pricecfg.Source, - "base": pricecfg.Base, - "quote": pricecfg.Quote, - "wander": pricecfg.Wander, - }).Info("Added price") - } - - s.pe.StartFetching() - - for _, pricecfg := range s.config.Prices { - pi := s.pe.WaitForPrice(*pricecfg) - log.WithFields(log.Fields{ - "price": pi.Price, - }).Info("Got first price") + if err := s.pe.StartFetching(); err != nil { + return fmt.Errorf("failed to start fetching: %w", err) } return nil @@ -187,11 +164,18 @@ func (s *Service) PricesGet(w http.ResponseWriter, r *http.Request, ps httproute (base == "" || base == k.Base) && (quote == "" || quote == k.Quote) && (wanderPtr == nil || *wanderPtr == k.Wander) { + + quote = k.Quote + if k.QuoteOverride != "" { + quote = k.QuoteOverride + } + response.Prices = append(response.Prices, &PriceResponse{ Source: k.Source, Base: k.Base, - Quote: k.Quote, - Price: v.Price, + Quote: quote, + QuoteReal: k.Quote, + Price: v.Price * k.Factor, LastUpdatedReal: v.LastUpdatedReal.String(), LastUpdatedWander: v.LastUpdatedWander.String(), }) diff --git a/utils/slice.go b/utils/slice.go new file mode 100644 index 0000000..e55a2b8 --- /dev/null +++ b/utils/slice.go @@ -0,0 +1,11 @@ +package utils + +func InSlice[T comparable](item T, slice []T) bool { + for _, val := range slice { + if item == val { + return true + } + } + + return false +} From dac8f93b5dd108d1ca964146e64e2c20786973f7 Mon Sep 17 00:00:00 2001 From: daniel1302 Date: Thu, 17 Nov 2022 02:19:04 +0100 Subject: [PATCH 02/10] feat: refactor + bitstamp added --- .vscode/launch.json | 21 ++++ README.md | 2 +- cmd/priceproxy/main.go | 4 +- config.yaml | 221 +++++++++++++++++++++++++++++++++++ config/config.go | 12 +- config/config_test.go | 6 +- go.mod | 2 +- pricing/bitstamp.go | 207 ++++++++++++++++++++++++++++++++ pricing/coingecko.go | 4 +- pricing/coinmarketcap.go | 4 +- pricing/http.go | 29 +++++ pricing/mocks/engine_mock.go | 6 +- pricing/pricing.go | 34 ++---- service/service.go | 29 ++--- 14 files changed, 524 insertions(+), 57 deletions(-) create mode 100644 .vscode/launch.json create mode 100644 config.yaml create mode 100644 pricing/bitstamp.go create mode 100644 pricing/http.go diff --git a/.vscode/launch.json b/.vscode/launch.json new file mode 100644 index 0000000..27558c4 --- /dev/null +++ b/.vscode/launch.json @@ -0,0 +1,21 @@ +{ + // Use IntelliSense to learn about possible attributes. + // Hover to view descriptions of existing attributes. + // For more information, visit: https://go.microsoft.com/fwlink/?linkid=830387 + "version": "0.2.0", + "configurations": [ + { + "name": "priceproxy", + "type": "go", + "request": "launch", + "mode": "auto", + "program": "${workspaceFolder}/cmd/priceproxy/main.go", + "args": [ + "-config", "${workspaceFolder}/config.yaml" + ], + "env": { + "CMC_PRO_API_KEY": "XXXXXXXXXXXXX" + } + } + ] +} \ No newline at end of file diff --git a/README.md b/README.md index 432376b..3c97ab2 100644 --- a/README.md +++ b/README.md @@ -13,7 +13,7 @@ See the [releases page](https://github.com/vegaprotocol/priceproxy/releases/). Either run `go get`: ```bash -go get code.vegaprotocol.io/priceproxy/cmd/priceproxy@latest +go get github.com/vegaprotocol/priceproxy/cmd/priceproxy@latest ``` Or clone the repository: diff --git a/cmd/priceproxy/main.go b/cmd/priceproxy/main.go index 0551595..3c40a8e 100644 --- a/cmd/priceproxy/main.go +++ b/cmd/priceproxy/main.go @@ -15,8 +15,8 @@ import ( "github.com/jinzhu/configor" log "github.com/sirupsen/logrus" - "code.vegaprotocol.io/priceproxy/config" - "code.vegaprotocol.io/priceproxy/service" + "github.com/vegaprotocol/priceproxy/config" + "github.com/vegaprotocol/priceproxy/service" ) var ( diff --git a/config.yaml b/config.yaml new file mode 100644 index 0000000..c327d1a --- /dev/null +++ b/config.yaml @@ -0,0 +1,221 @@ +server: + listen: ":80" + logformat: text # json, text + loglevel: debug + env: prod # dev, prod + +sources: + - name: bitstamp + # Avoid BitStamp, it has a tendency to return HTTP 404 if the UserAgent is not a browser. + sleepReal: 60 # seconds + url: + scheme: https + host: www.bitstamp.net + path: /api/v2/ticker/ + + - name: coinmarketcap + # requires API key from $HOME/coinmarketcap-apikey.txt + sleepReal: 1200 # seconds + auth_key_env_name: "CMC_PRO_API_KEY" + url: + scheme: https + host: pro-api.coinmarketcap.com + path: /v1/cryptocurrency/listings/latest + + - name: coingecko + sleepReal: 30 # seconds + url: + scheme: https + host: api.coingecko.com + path: /api/v3/simple/price + rawquery: ids=solana,ethereum,bitcoin,terra-luna-2,uniswap,dai,aave,litecoin,optimism,monero,cosmos&vs_currencies=usd,eur,btc,eth&include_last_updated_at=true + + # - name: ethgasstation + # sleepReal: 1800 # seconds + # sleepWander: 30 # seconds + # # Injected at runtime: url.rawquery="api-key=..." + # url: + # scheme: https + # host: ethgasstation.info + # path: /api/ethgasAPI.json + + # - name: exchangeratesapi + # sleepReal: 600 # seconds + # sleepWander: 30 # seconds + # url: + # scheme: https + # host: api.exchangeratesapi.io + # path: /latest + # rawquery: base={base}&symbols={quote} + + # - name: finhub + # sleepReal: 1800 # seconds + # sleepWander: 30 # seconds + # url: + # scheme: https + # host: finnhub.io + # path: /api/v1/quote + # rawquery: "symbol={base}" # yes, Quote is not mentioned + + # - name: ftx-aapl + # sleepReal: 60 # seconds + # sleepWander: 30 # seconds + # url: + # scheme: https + # host: ftx.com + # # Note: The format is TICKER-MMDD (month + day), keep it up to date + # path: /api/markets/AAPL-1230 + + # - name: ftx-aave + # sleepReal: 60 # seconds + # sleepWander: 30 # seconds + # url: + # scheme: https + # host: ftx.com + # path: /api/markets/AAVE-PERP + + # - name: ftx-tsla + # sleepReal: 60 # seconds + # sleepWander: 30 # seconds + # url: + # scheme: https + # host: ftx.com + # # Note: The format is TICKER-MMDD (month + day), keep it up to date + # path: /api/markets/TSLA-1230 + + # - name: ftx-uni + # sleepReal: 60 # seconds + # sleepWander: 30 # seconds + # url: + # scheme: https + # host: ftx.com + # path: /api/markets/UNI-PERP + +prices: + - source: coinmarketcap + base: BTC + quote: USD + factor: 1.0 + wander: true + + - source: coinmarketcap + base: ETH + quote: BTC + factor: 1.0 + wander: true + + - source: coinmarketcap + base: ETH + quote: DAI + factor: 1.0 + wander: true + + - source: coinmarketcap + base: LUNA + quote: USD + factor: 1.0 + wander: true + + - source: coinmarketcap + base: XMR + quote: USD + factor: 1.0 + wander: true + + # - source: ftx-aave + # base: AAVE + # quote: DAI + # factor: 1.0 + # wander: true + + # - source: ftx-uni + # base: UNI + # quote: DAI + # factor: 1.0 + # wander: true + + # - source: ftx-aapl + # base: AAPL + # quote: USD + # factor: 1.0 + # wander: true + + # - source: ftx-tsla + # base: TSLA + # quote: EURO + # factor: 0.951 # USD -> EURO + # wander: true + + - source: coingecko + base: aave + quote: dai + factor: 1.0 + wander: true + + - source: coingecko + base: uniswap + quote: dai + factor: 1.0 + wander: true + + - source: coingecko + base: terra-luna-2 + quote: usd + factor: 1.0 + wander: true + + - source: coingecko + base: litecoin + quote: usd + factor: 1.0 + wander: true + + - source: coingecko + base: ethereum + quote: btc + factor: 1.0 + wander: true + + - source: coingecko + base: bitcoin + quote: usd + factor: 1.0 + wander: true + + - source: coingecko + base: monero + quote: usd + factor: 1.0 + wander: true + + - source: coingecko + base: cosmos + quote: usd + factor: 1.0 + wander: true + + - source: coingecko + base: optimism + quote: usd + factor: 1.0 + wander: true + + + - source: bitstamp + base: IMX + quote: USD + factor: 1.0 + wander: true + + - source: bitstamp + base: IMX + quote: ETH + factor: 1.0 + wander: true + + + - source: bitstamp + base: IMX + quote: BTC + factor: 1.0 + wander: true diff --git a/config/config.go b/config/config.go index 247f0e4..0e99c48 100644 --- a/config/config.go +++ b/config/config.go @@ -37,7 +37,6 @@ type SourceConfig struct { URL url.URL `yaml:"url"` AuthKeyEnvName string `yaml:"auth_key_env_name"` SleepReal int `yaml:"sleepReal"` - SleepWander int `yaml:"sleepWander"` } type PriceList []PriceConfig @@ -91,9 +90,6 @@ func CheckConfig(cfg *Config) error { if sourcecfg.SleepReal == 0 { return fmt.Errorf("%s: sleepReal", ErrInvalidValue.Error()) } - if sourcecfg.SleepWander == 0 { - return fmt.Errorf("%s: sleepWander", ErrInvalidValue.Error()) - } } if cfg.Prices == nil { @@ -161,8 +157,8 @@ func (pc PriceConfig) String() string { } func (ps SourceConfig) String() string { - return fmt.Sprintf("{SourceConfig Name:%s URL:%s SleepReal:%ds SleepWander:%ds}", - ps.Name, ps.URL.String(), ps.SleepReal, ps.SleepWander) + return fmt.Sprintf("{SourceConfig Name:%s URL:%s SleepReal:%ds}", + ps.Name, ps.URL.String(), ps.SleepReal) } func (ps SourceConfig) IsCoinGecko() bool { @@ -172,3 +168,7 @@ func (ps SourceConfig) IsCoinGecko() bool { func (ps SourceConfig) IsCoinMarketCap() bool { return strings.Contains(ps.URL.Host, "coinmarketcap.com") } + +func (ps SourceConfig) IsBitstamp() bool { + return strings.Contains(ps.URL.Host, "bitstamp.net") +} diff --git a/config/config_test.go b/config/config_test.go index 2550c4d..1c39912 100644 --- a/config/config_test.go +++ b/config/config_test.go @@ -7,7 +7,7 @@ import ( "github.com/stretchr/testify/assert" - "code.vegaprotocol.io/priceproxy/config" + "github.com/vegaprotocol/priceproxy/config" ) func TestCheckConfig(t *testing.T) { @@ -39,7 +39,6 @@ func TestCheckConfig(t *testing.T) { err = config.CheckConfig(&cfg) assert.True(t, strings.HasPrefix(err.Error(), config.ErrInvalidValue.Error())) - cfg.Sources[0].SleepWander = 1 err = config.CheckConfig(&cfg) assert.True(t, strings.HasPrefix(err.Error(), config.ErrMissingEmptyConfigSection.Error())) @@ -89,8 +88,7 @@ func TestConfigStringFuncs(t *testing.T) { Path: "/path", RawQuery: "a=b&x=y", }, - SleepReal: 11, - SleepWander: 7, + SleepReal: 11, } assert.Equal(t, "{SourceConfig Name:NNN URL:https://example.com/path?a=b&x=y SleepReal:11s SleepWander:7s}", ps.String()) } diff --git a/go.mod b/go.mod index 4be0f6d..d754bf6 100644 --- a/go.mod +++ b/go.mod @@ -1,4 +1,4 @@ -module code.vegaprotocol.io/priceproxy +module github.com/vegaprotocol/priceproxy go 1.18 diff --git a/pricing/bitstamp.go b/pricing/bitstamp.go new file mode 100644 index 0000000..1f8e4f4 --- /dev/null +++ b/pricing/bitstamp.go @@ -0,0 +1,207 @@ +package pricing + +import ( + "context" + "encoding/json" + "fmt" + "net/http" + "strconv" + "strings" + "time" + + log "github.com/sirupsen/logrus" + "github.com/vegaprotocol/priceproxy/config" + "golang.org/x/time/rate" +) + +func bitstampStartFetching( + board priceBoard, + sourcecfg config.SourceConfig, +) { + var ( + fetchURL = sourcecfg.URL.String() + oneRequestEvery = time.Duration(sourcecfg.SleepReal) * time.Second + rateLimiter = rate.NewLimiter(rate.Every(oneRequestEvery), 1) + ctx = context.Background() + err error + ) + + log.WithFields(log.Fields{ + "sourceName": sourcecfg.Name, + "URL": fetchURL, + "rateLimitDuration": oneRequestEvery, + }).Infof("Starting Coingecko Fetching\n") + + for { + if err = rateLimiter.Wait(ctx); err != nil { + log.WithFields(log.Fields{ + "error": err.Error(), + "sourceName": sourcecfg.Name, + "URL": fetchURL, + "rateLimitDuration": oneRequestEvery, + }).Errorln("Rate Limiter Failed. Falling back to Sleep.") + // fallback + time.Sleep(oneRequestEvery) + } + + prices, err := bitstampSingleFetch(fetchURL) + if err != nil { + log.WithFields(log.Fields{ + "error": err.Error(), + "sourceName": sourcecfg.Name, + "URL": fetchURL, + "rateLimitDuration": oneRequestEvery, + }).Errorf("Retry in %d sec.\n", oneRequestEvery) + continue + } + + for _, price := range board.PriceList(sourcecfg.Name) { + var ( + fetchedTimestamp time.Time = time.Now() + fetchedPrice float64 + ) + + if currency := prices.Currency(price.Base, price.Quote); currency != nil { + fetchedTimestamp = currency.UnixTimestamp() + fetchedPrice = currency.Price() + } + + if fetchedPrice == 0 { + log.WithFields(log.Fields{ + "sourceName": sourcecfg.Name, + "base": price.Base, + "quote": price.Quote, + "quote_override": price.QuoteOverride, + }).Debug("Quote/Base rate not found directly, trying conversion") + + if currency := prices.Convert(price.Base, price.Quote); currency != nil { + fetchedTimestamp = currency.UnixTimestamp() + fetchedPrice = currency.Price() + } + } + + if fetchedPrice == 0 { + log.WithFields(log.Fields{ + "sourceName": sourcecfg.Name, + "base": price.Base, + "quote": price.Quote, + "quote_override": price.QuoteOverride, + }).Warnf("fetched price in the quote current is 0, consider selecting different quote and overwrite it with the `quote_override` parameter") + } + + board.UpdatePrice( + price, + PriceInfo{ + Price: fetchedPrice, + LastUpdatedReal: fetchedTimestamp, + LastUpdatedWander: time.Now().Round(0), + }, + ) + } + } +} + +type bitstampCurrencyData struct { + Timestamp string `json:"timestamp"` + Last string `json:"last"` + Pair string `json:"pair"` +} + +type bitstampFetchData []bitstampCurrencyData + +func (fd bitstampCurrencyData) Base() string { + pairSlice := strings.Split(fd.Pair, "/") + + return pairSlice[0] +} + +func (fd bitstampCurrencyData) UnixTimestamp() time.Time { + timestamp, err := strconv.ParseInt(fd.Timestamp, 10, 0) + if err != nil { + return time.Now() + } + + return time.Unix(timestamp, 0) +} + +func (fd bitstampCurrencyData) Price() float64 { + price, err := strconv.ParseFloat(fd.Timestamp, 0) + if err != nil { + return 0.0 + } + + return price +} + +func (fd bitstampCurrencyData) Quote() string { + pairSlice := strings.Split(fd.Pair, "/") + + if len(pairSlice) < 2 { + return pairSlice[0] + } + return pairSlice[1] +} + +func (fd bitstampFetchData) Currency(base, quote string) *bitstampCurrencyData { + for _, currency := range fd { + if strings.EqualFold(currency.Base(), base) && strings.EqualFold(currency.Quote(), quote) { + return ¤cy + } + } + + return nil +} + +func (fd bitstampFetchData) Convert(base, quote string) *bitstampCurrencyData { + basePrices := []bitstampCurrencyData{} + quotePrices := []bitstampCurrencyData{} + + if currency := fd.Currency(base, quote); currency != nil { + return currency + } + + for _, price := range fd { + if strings.EqualFold(price.Base(), base) { + basePrices = append(basePrices, price) + } + + if strings.EqualFold(price.Base(), quote) { + quotePrices = append(quotePrices, price) + } + } + + // find common currency and calculate value + for _, basePrice := range basePrices { + for _, quotePrice := range quotePrices { + if strings.EqualFold(basePrice.Quote(), quotePrice.Quote()) { + return &bitstampCurrencyData{ + Timestamp: basePrice.Timestamp, + Pair: fmt.Sprintf("%s/%s", base, quote), + Last: fmt.Sprintf("%f", (basePrice.Price() / quotePrice.Price())), + } + } + } + } + + return nil +} + +func bitstampSingleFetch(url string) (bitstampFetchData, error) { + resp, err := http.Get(url) + if err != nil { + return nil, fmt.Errorf("failed to get bitstamp data, %w", err) + } + + if resp.StatusCode != http.StatusOK { + return nil, fmt.Errorf("failed to get bitstamp data: expected status 200, got %d", resp.StatusCode) + } + + defer resp.Body.Close() + var prices bitstampFetchData + if err = json.NewDecoder(resp.Body).Decode(&prices); err != nil { + return nil, fmt.Errorf("failed to parse bitstamp data, %w", err) + } + return prices, nil +} + +//https://www.bitstamp.net/api/v2/ticker/ diff --git a/pricing/coingecko.go b/pricing/coingecko.go index f555a34..6dcac0d 100644 --- a/pricing/coingecko.go +++ b/pricing/coingecko.go @@ -8,8 +8,8 @@ import ( "strings" "time" - "code.vegaprotocol.io/priceproxy/config" log "github.com/sirupsen/logrus" + "github.com/vegaprotocol/priceproxy/config" "golang.org/x/time/rate" ) @@ -93,7 +93,7 @@ func coingeckoStartFetching( "base": price.Base, "quote": price.Quote, "quote_override": price.QuoteOverride, - }).Warnf("collected price in the quote current is 0, consider selecting different quote and overwrite it with the `quote_override` parameter") + }).Warnf("fetched price in the quote current is 0, consider selecting different quote and overwrite it with the `quote_override` parameter") } board.UpdatePrice( diff --git a/pricing/coinmarketcap.go b/pricing/coinmarketcap.go index d74fc21..e8e11fe 100644 --- a/pricing/coinmarketcap.go +++ b/pricing/coinmarketcap.go @@ -9,8 +9,8 @@ import ( "strings" "time" - "code.vegaprotocol.io/priceproxy/config" log "github.com/sirupsen/logrus" + "github.com/vegaprotocol/priceproxy/config" "golang.org/x/time/rate" ) @@ -121,7 +121,7 @@ func coinmarketcapStartFetching( "base": price.Base, "quote": price.Quote, "quote_override": price.QuoteOverride, - }).Warnf("collected price in the quote current is 0, consider selecting different quote and overwrite it with the `quote_override` parameter") + }).Warnf("fetched price in the quote current is 0, consider selecting different quote and overwrite it with the `quote_override` parameter") } board.UpdatePrice( diff --git a/pricing/http.go b/pricing/http.go new file mode 100644 index 0000000..75bc926 --- /dev/null +++ b/pricing/http.go @@ -0,0 +1,29 @@ +package pricing + +import ( + "net/url" + "strings" + "time" + + "github.com/vegaprotocol/priceproxy/config" +) + +func httpStartFetching( + board priceBoard, + sourcecfg config.SourceConfig, +) { + // TODO: implement when needed + + for { + time.Sleep(time.Minute) + } +} + +func urlWithBaseQuote(u url.URL, pricecfg config.PriceConfig) *url.URL { + result := u + result.Path = strings.Replace(result.Path, "{base}", pricecfg.Base, 1) + result.Path = strings.Replace(result.Path, "{quote}", pricecfg.Quote, 1) + result.RawQuery = strings.Replace(result.RawQuery, "{base}", pricecfg.Base, 1) + result.RawQuery = strings.Replace(result.RawQuery, "{quote}", pricecfg.Quote, 1) + return &result +} diff --git a/pricing/mocks/engine_mock.go b/pricing/mocks/engine_mock.go index ad0f291..27b5b2a 100644 --- a/pricing/mocks/engine_mock.go +++ b/pricing/mocks/engine_mock.go @@ -1,12 +1,12 @@ // Code generated by MockGen. DO NOT EDIT. -// Source: code.vegaprotocol.io/priceproxy/pricing (interfaces: Engine) +// Source: github.com/vegaprotocol/priceproxy/pricing (interfaces: Engine) // Package mocks is a generated GoMock package. package mocks import ( - config "code.vegaprotocol.io/priceproxy/config" - pricing "code.vegaprotocol.io/priceproxy/pricing" + config "github.com/vegaprotocol/priceproxy/config" + pricing "github.com/vegaprotocol/priceproxy/pricing" gomock "github.com/golang/mock/gomock" reflect "reflect" ) diff --git a/pricing/pricing.go b/pricing/pricing.go index 4dc89e7..b2cb214 100644 --- a/pricing/pricing.go +++ b/pricing/pricing.go @@ -2,12 +2,10 @@ package pricing import ( "fmt" - "net/url" - "strings" "sync" "time" - "code.vegaprotocol.io/priceproxy/config" + "github.com/vegaprotocol/priceproxy/config" ) const minPrice = 0.00001 @@ -23,7 +21,7 @@ type PriceInfo struct { // Engine is the source of price information from multiple external/internal/fake sources. // -//go:generate go run github.com/golang/mock/mockgen -destination mocks/engine_mock.go -package mocks code.vegaprotocol.io/priceproxy/pricing Engine +//go:generate go run github.com/golang/mock/mockgen -destination mocks/engine_mock.go -package mocks github.com/vegaprotocol/priceproxy/pricing Engine type Engine interface { AddSource(sourcecfg config.SourceConfig) error GetSource(name string) (config.SourceConfig, error) @@ -45,7 +43,7 @@ type priceBoard interface { type engine struct { priceList config.PriceList prices map[config.PriceConfig]PriceInfo - pricesMu sync.Mutex + pricesMu sync.RWMutex sources map[string]config.SourceConfig sourcesMu sync.Mutex @@ -65,9 +63,6 @@ func (e *engine) AddSource(sourcecfg config.SourceConfig) error { if sourcecfg.SleepReal == 0 { return fmt.Errorf("invalid source config: sleepReal is zero") } - if sourcecfg.SleepWander == 0 { - return fmt.Errorf("invalid source config: sleepWander is zero") - } e.sourcesMu.Lock() defer e.sourcesMu.Unlock() @@ -107,8 +102,8 @@ func (e *engine) GetSources() ([]config.SourceConfig, error) { } func (e *engine) GetPrice(pricecfg config.PriceConfig) (PriceInfo, error) { - e.pricesMu.Lock() - defer e.pricesMu.Unlock() + e.pricesMu.RLock() + defer e.pricesMu.RUnlock() pi, found := e.prices[pricecfg] if !found { @@ -118,8 +113,8 @@ func (e *engine) GetPrice(pricecfg config.PriceConfig) (PriceInfo, error) { } func (e *engine) GetPrices() map[config.PriceConfig]PriceInfo { - e.pricesMu.Lock() - defer e.pricesMu.Unlock() + e.pricesMu.RLock() + defer e.pricesMu.RUnlock() results := map[config.PriceConfig]PriceInfo{} for k, v := range e.prices { @@ -148,8 +143,12 @@ func (e *engine) StartFetching() error { go coinmarketcapStartFetching(e, sourceConfig) continue } + if sourceConfig.IsBitstamp() { + go bitstampStartFetching(e, sourceConfig) + continue + } - return fmt.Errorf("source %s not supported", sourceConfig.String()) + go httpStartFetching(e, sourceConfig) } return nil @@ -159,12 +158,3 @@ func (pi PriceInfo) String() string { return fmt.Sprintf("{PriceInfo Price:%f LastUpdatedReal:%s LastUpdatedWander:%s}", pi.Price, pi.LastUpdatedReal.String(), pi.LastUpdatedWander.String()) } - -func urlWithBaseQuote(u url.URL, pricecfg config.PriceConfig) *url.URL { - result := u - result.Path = strings.Replace(result.Path, "{base}", pricecfg.Base, 1) - result.Path = strings.Replace(result.Path, "{quote}", pricecfg.Quote, 1) - result.RawQuery = strings.Replace(result.RawQuery, "{base}", pricecfg.Base, 1) - result.RawQuery = strings.Replace(result.RawQuery, "{quote}", pricecfg.Quote, 1) - return &result -} diff --git a/service/service.go b/service/service.go index dba347d..6cb353c 100644 --- a/service/service.go +++ b/service/service.go @@ -8,8 +8,8 @@ import ( "strconv" "time" - "code.vegaprotocol.io/priceproxy/config" - "code.vegaprotocol.io/priceproxy/pricing" + "github.com/vegaprotocol/priceproxy/config" + "github.com/vegaprotocol/priceproxy/pricing" "github.com/julienschmidt/httprouter" log "github.com/sirupsen/logrus" @@ -112,18 +112,16 @@ func (s *Service) initPricingEngine() error { err := s.pe.AddSource(*sourcecfg) if err != nil { log.WithFields(log.Fields{ - "error": err.Error(), - "name": sourcecfg.Name, - "sleepReal": sourcecfg.SleepReal, - "sleepWander": sourcecfg.SleepWander, - "url": sourcecfg.URL.String(), + "error": err.Error(), + "name": sourcecfg.Name, + "sleepReal": sourcecfg.SleepReal, + "url": sourcecfg.URL.String(), }).Fatal("Failed to add source") } log.WithFields(log.Fields{ - "name": sourcecfg.Name, - "sleepReal": sourcecfg.SleepReal, - "sleepWander": sourcecfg.SleepWander, - "url": sourcecfg.URL.String(), + "name": sourcecfg.Name, + "sleepReal": sourcecfg.SleepReal, + "url": sourcecfg.URL.String(), }).Info("Added source") } @@ -159,26 +157,29 @@ func (s *Service) PricesGet(w http.ResponseWriter, r *http.Request, ps httproute response := PricesResponse{ Prices: make([]*PriceResponse, 0), } + for k, v := range s.pe.GetPrices() { if (source == "" || source == k.Source) && (base == "" || base == k.Base) && (quote == "" || quote == k.Quote) && (wanderPtr == nil || *wanderPtr == k.Wander) { - quote = k.Quote + returnedQuote := k.Quote if k.QuoteOverride != "" { - quote = k.QuoteOverride + returnedQuote = k.QuoteOverride } response.Prices = append(response.Prices, &PriceResponse{ Source: k.Source, Base: k.Base, - Quote: quote, + Quote: returnedQuote, QuoteReal: k.Quote, Price: v.Price * k.Factor, LastUpdatedReal: v.LastUpdatedReal.String(), LastUpdatedWander: v.LastUpdatedWander.String(), }) + } else { + fmt.Printf("%v", k) } } writeSuccess(w, response, http.StatusOK) From 018c5d7d5c8bbe44b412aa7d0722f5b4000eb7c6 Mon Sep 17 00:00:00 2001 From: daniel1302 Date: Thu, 17 Nov 2022 02:23:55 +0100 Subject: [PATCH 03/10] feat: fix tests --- config/config_test.go | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/config/config_test.go b/config/config_test.go index 1c39912..993eced 100644 --- a/config/config_test.go +++ b/config/config_test.go @@ -36,9 +36,6 @@ func TestCheckConfig(t *testing.T) { assert.True(t, strings.HasPrefix(err.Error(), config.ErrInvalidValue.Error())) cfg.Sources[0].SleepReal = 1 - err = config.CheckConfig(&cfg) - assert.True(t, strings.HasPrefix(err.Error(), config.ErrInvalidValue.Error())) - err = config.CheckConfig(&cfg) assert.True(t, strings.HasPrefix(err.Error(), config.ErrMissingEmptyConfigSection.Error())) @@ -90,5 +87,5 @@ func TestConfigStringFuncs(t *testing.T) { }, SleepReal: 11, } - assert.Equal(t, "{SourceConfig Name:NNN URL:https://example.com/path?a=b&x=y SleepReal:11s SleepWander:7s}", ps.String()) + assert.Equal(t, "{SourceConfig Name:NNN URL:https://example.com/path?a=b&x=y SleepReal:11s}", ps.String()) } From c3903f84d7c876360bdcc227f9ef89d78cc44b9d Mon Sep 17 00:00:00 2001 From: daniel1302 Date: Thu, 17 Nov 2022 02:31:42 +0100 Subject: [PATCH 04/10] feat: upgrade go to 1.19 and upgrade packages --- go.mod | 21 +++++++++------------ go.sum | 58 ++++++++++++++++++++++++++++++++++------------------------ 2 files changed, 43 insertions(+), 36 deletions(-) diff --git a/go.mod b/go.mod index d754bf6..da9ad99 100644 --- a/go.mod +++ b/go.mod @@ -1,24 +1,21 @@ module github.com/vegaprotocol/priceproxy -go 1.18 +go 1.19 require ( - github.com/golang/mock v1.5.0 + github.com/golang/mock v1.6.0 github.com/jinzhu/configor v1.2.1 github.com/julienschmidt/httprouter v1.3.0 - github.com/pkg/errors v0.9.1 - github.com/sirupsen/logrus v1.8.1 - github.com/stretchr/testify v1.7.0 + github.com/sirupsen/logrus v1.9.0 + github.com/stretchr/testify v1.8.1 + golang.org/x/time v0.2.0 ) require ( - github.com/BurntSushi/toml v0.3.1 // indirect + github.com/BurntSushi/toml v1.2.1 // indirect github.com/davecgh/go-spew v1.1.1 // indirect - github.com/kr/pretty v0.1.0 // indirect github.com/pmezard/go-difflib v1.0.0 // indirect - golang.org/x/sys v0.0.0-20220704084225-05e143d24a9e // indirect - golang.org/x/time v0.0.0-20220609170525-579cf78fd858 - gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127 // indirect - gopkg.in/yaml.v2 v2.2.2 // indirect - gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c // indirect + golang.org/x/sys v0.2.0 // indirect + gopkg.in/yaml.v2 v2.4.0 // indirect + gopkg.in/yaml.v3 v3.0.1 // indirect ) diff --git a/go.sum b/go.sum index 57347e2..d519248 100644 --- a/go.sum +++ b/go.sum @@ -1,50 +1,60 @@ -github.com/BurntSushi/toml v0.3.1 h1:WXkYYl6Yr3qBf1K79EBnL4mak0OimBfB0XUf9Vl28OQ= github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU= +github.com/BurntSushi/toml v1.2.1 h1:9F2/+DoOYIOksmaJFPw1tGFy1eDnIJXg+UHjuD8lTak= +github.com/BurntSushi/toml v1.2.1/go.mod h1:CxXYINrC8qIiEnFrOxCa7Jy5BFHlXnUU2pbicEuybxQ= github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= -github.com/golang/mock v1.5.0 h1:jlYHihg//f7RRwuPfptm04yp4s7O6Kw8EZiVYIGcH0g= -github.com/golang/mock v1.5.0/go.mod h1:CWnOUgYIOo4TcNZ0wHX3YZCqsaM1I1Jvs6v3mP3KVu8= +github.com/golang/mock v1.6.0 h1:ErTB+efbowRARo13NNdxyJji2egdxLGQhRaY+DUumQc= +github.com/golang/mock v1.6.0/go.mod h1:p6yTPP+5HYm5mzsMV8JkE6ZKdX+/wYM6Hr+LicevLPs= github.com/jinzhu/configor v1.2.1 h1:OKk9dsR8i6HPOCZR8BcMtcEImAFjIhbJFZNyn5GCZko= github.com/jinzhu/configor v1.2.1/go.mod h1:nX89/MOmDba7ZX7GCyU/VIaQ2Ar2aizBl2d3JLF/rDc= github.com/julienschmidt/httprouter v1.3.0 h1:U0609e9tgbseu3rBINet9P48AI/D3oJs4dN7jwJOQ1U= github.com/julienschmidt/httprouter v1.3.0/go.mod h1:JR6WtHb+2LUe8TCKY3cZOxFyyO8IZAc4RVcycCCAKdM= -github.com/kr/pretty v0.1.0 h1:L/CwN0zerZDmRFUapSPitk6f+Q3+0za1rQkzVuMiMFI= -github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo= -github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ= -github.com/kr/text v0.1.0 h1:45sCR5RtlFHMR4UwH9sdQ5TC8v0qDQCHnXt+kaKSTVE= -github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI= -github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4= -github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= -github.com/sirupsen/logrus v1.8.1 h1:dJKuHgqk1NNQlqoA6BTlM1Wf9DOH3NBjQyu0h9+AZZE= -github.com/sirupsen/logrus v1.8.1/go.mod h1:yWOB1SBYBC5VeMP7gHvWumXLIWorT60ONWic61uBYv0= +github.com/sirupsen/logrus v1.9.0 h1:trlNQbNUG3OdDrDil03MCb1H2o9nJ1x4/5LYw7byDE0= +github.com/sirupsen/logrus v1.9.0/go.mod h1:naHLuLoDiP4jHNo9R0sCBMtWGeIprob74mVsIT4qYEQ= github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= -github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs= -github.com/stretchr/testify v1.7.0 h1:nwc3DEeHmmLAfoZucVR881uASk0Mfjw8xYJ99tb5CcY= +github.com/stretchr/objx v0.4.0/go.mod h1:YvHI0jy2hoMjB+UWwv71VJQ9isScKT/TqJzVSSt89Yw= +github.com/stretchr/objx v0.5.0/go.mod h1:Yh+to48EsGEfYuaHDzXPcE3xhTkx73EhmCGUpEOglKo= github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= +github.com/stretchr/testify v1.7.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= +github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO+kdMU+MU= +github.com/stretchr/testify v1.8.1 h1:w7B6lhMri9wdJUVmEZPGGhZzrYTPvgJArz7wNPgYKsk= +github.com/stretchr/testify v1.8.1/go.mod h1:w2LPCIKwWwSfY2zedu0+kehJoqGctiVI29o6fzry7u4= +github.com/yuin/goldmark v1.3.5/go.mod h1:mwnBkeHKe2W/ZEtQ+71ViKU8L12m81fl3OWwC1Zlc8k= golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= -golang.org/x/mod v0.3.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= +golang.org/x/mod v0.4.2/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20210405180319-a5a99cb37ef4/go.mod h1:p54w0d4576C0XHj96bSt6lcn1PtDYWL6XObtHCRCNQM= golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20210220032951-036812b2e83c/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20191026070338-33540a1f6037/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20220704084225-05e143d24a9e h1:CsOuNlbOuf0mzxJIefr6Q4uAUetRUwZE4qt7VfzP+xo= -golang.org/x/sys v0.0.0-20220704084225-05e143d24a9e/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20210330210617-4fbd30eecc44/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20210510120138-977fb7262007/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20220715151400-c0bba94af5f8/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.2.0 h1:ljd4t30dBnAvMZaQCevtY0xLLD0A+bRZXbgLMLU1F/A= +golang.org/x/sys v0.2.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= -golang.org/x/time v0.0.0-20220609170525-579cf78fd858 h1:Dpdu/EMxGMFgq0CeYMh4fazTD2vtlZRYE7wyynxJb9U= -golang.org/x/time v0.0.0-20220609170525-579cf78fd858/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= +golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= +golang.org/x/time v0.2.0 h1:52I/1L54xyEQAYdtcSuxtiT84KGYTBGXwayxmIpNJhE= +golang.org/x/time v0.2.0/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= +golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= +golang.org/x/tools v0.1.1/go.mod h1:o0xws9oXOQQZyjljx8fwUC0k7L1pTE6eaCbjGeHmOkk= golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= +golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= +gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405 h1:yhCVgyC4o1eVCa2tZl7eS0r+SDo693bJlVdllGtEeKM= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= -gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127 h1:qIbj1fsPNlZgppZ+VLlY7N33q108Sa+fhmuc+sWQYwY= -gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= -gopkg.in/yaml.v2 v2.2.2 h1:ZCJp+EgiOT7lHqUV2J862kp8Qj64Jo6az82+3Td9dZw= gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= -gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c h1:dUUwHk2QECo/6vqA44rthZ8ie2QXMNeKRTHCNY2nXvo= +gopkg.in/yaml.v2 v2.4.0 h1:D8xgwECY7CYvx+Y2n4sBz93Jn9JRvxdiyyo8CTfuKaY= +gopkg.in/yaml.v2 v2.4.0/go.mod h1:RDklbk79AGWmwhnvt/jBztapEOGDOx6ZbXqjP6csGnQ= gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= +gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= +gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= From 0e640e258b0039e97f5a7b7be30b4fa8c891a117 Mon Sep 17 00:00:00 2001 From: daniel1302 Date: Thu, 17 Nov 2022 02:33:14 +0100 Subject: [PATCH 05/10] feat: upgrade golangci --- .github/workflows/continous-integration.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/continous-integration.yml b/.github/workflows/continous-integration.yml index d8e5aef..2708b51 100644 --- a/.github/workflows/continous-integration.yml +++ b/.github/workflows/continous-integration.yml @@ -33,6 +33,6 @@ jobs: - name: golangci-lint uses: golangci/golangci-lint-action@v3 with: - version: v1.45 + version: v1.50.1 args: --config .golangci.toml \ No newline at end of file From 5ce31862a62984bc57da958633500c7b97175337 Mon Sep 17 00:00:00 2001 From: daniel1302 Date: Thu, 17 Nov 2022 02:49:01 +0100 Subject: [PATCH 06/10] fix: fix linter issues --- .golangci.toml | 5 +++++ pricing/bitstamp.go | 8 ++++---- pricing/coingecko.go | 2 +- pricing/coinmarketcap.go | 2 +- pricing/http.go | 23 +++++++++++++---------- pricing/mocks/engine_mock.go | 2 +- pricing/pricing.go | 4 ++-- service/service.go | 5 ++--- 8 files changed, 29 insertions(+), 22 deletions(-) diff --git a/.golangci.toml b/.golangci.toml index 27849bd..1c78b3c 100644 --- a/.golangci.toml +++ b/.golangci.toml @@ -11,6 +11,10 @@ max-same-issues = 0 [linters] enable-all = true disable = [ + "nosnakecase", + "structcheck", + "varcheck", + "deadcode", "promlinter", "wrapcheck", "tagliatelle", @@ -61,6 +65,7 @@ disable = [ "errchkjson", "ifshort", "predeclared", + "exhaustruct", "nolintlint", ] diff --git a/pricing/bitstamp.go b/pricing/bitstamp.go index 1f8e4f4..988d29e 100644 --- a/pricing/bitstamp.go +++ b/pricing/bitstamp.go @@ -116,7 +116,7 @@ func (fd bitstampCurrencyData) Base() string { } func (fd bitstampCurrencyData) UnixTimestamp() time.Time { - timestamp, err := strconv.ParseInt(fd.Timestamp, 10, 0) + timestamp, err := strconv.ParseInt(fd.Timestamp, 10, 64) if err != nil { return time.Now() } @@ -125,7 +125,7 @@ func (fd bitstampCurrencyData) UnixTimestamp() time.Time { } func (fd bitstampCurrencyData) Price() float64 { - price, err := strconv.ParseFloat(fd.Timestamp, 0) + price, err := strconv.ParseFloat(fd.Timestamp, 64) if err != nil { return 0.0 } @@ -187,7 +187,7 @@ func (fd bitstampFetchData) Convert(base, quote string) *bitstampCurrencyData { } func bitstampSingleFetch(url string) (bitstampFetchData, error) { - resp, err := http.Get(url) + resp, err := http.Get(url) // nolint:noctx if err != nil { return nil, fmt.Errorf("failed to get bitstamp data, %w", err) } @@ -204,4 +204,4 @@ func bitstampSingleFetch(url string) (bitstampFetchData, error) { return prices, nil } -//https://www.bitstamp.net/api/v2/ticker/ +// https://www.bitstamp.net/api/v2/ticker/ diff --git a/pricing/coingecko.go b/pricing/coingecko.go index 6dcac0d..7833212 100644 --- a/pricing/coingecko.go +++ b/pricing/coingecko.go @@ -130,7 +130,7 @@ type coingeckoFetchData map[string]struct { } func coingeckoSingleFetch(url string) (*coingeckoFetchData, error) { - resp, err := http.Get(url) + resp, err := http.Get(url) // nolint:noctx if err != nil { return nil, fmt.Errorf("failed to get coingecko data, %w", err) } diff --git a/pricing/coinmarketcap.go b/pricing/coinmarketcap.go index e8e11fe..23575b9 100644 --- a/pricing/coinmarketcap.go +++ b/pricing/coinmarketcap.go @@ -200,7 +200,7 @@ func (data coinmarketcapFetchData) ConvertPrice(base, quote string) float64 { } func coinmarketcapSingleFetch(url string) (*coinmarketcapFetchData, error) { - resp, err := http.Get(url) + resp, err := http.Get(url) // nolint:noctx if err != nil { return nil, fmt.Errorf("failed to get coinmarketcap data, %w", err) } diff --git a/pricing/http.go b/pricing/http.go index 75bc926..7d513d9 100644 --- a/pricing/http.go +++ b/pricing/http.go @@ -1,10 +1,9 @@ package pricing import ( - "net/url" - "strings" "time" + log "github.com/sirupsen/logrus" "github.com/vegaprotocol/priceproxy/config" ) @@ -16,14 +15,18 @@ func httpStartFetching( for { time.Sleep(time.Minute) + + log.WithFields(log.Fields{ + "sourceName": sourcecfg.Name, + }).Errorf("You are trying to use the fetcher which is not implemented yet. Try to use different one: bitstamp, coingecko, coinmarketcap") } } -func urlWithBaseQuote(u url.URL, pricecfg config.PriceConfig) *url.URL { - result := u - result.Path = strings.Replace(result.Path, "{base}", pricecfg.Base, 1) - result.Path = strings.Replace(result.Path, "{quote}", pricecfg.Quote, 1) - result.RawQuery = strings.Replace(result.RawQuery, "{base}", pricecfg.Base, 1) - result.RawQuery = strings.Replace(result.RawQuery, "{quote}", pricecfg.Quote, 1) - return &result -} +// func urlWithBaseQuote(u url.URL, pricecfg config.PriceConfig) *url.URL { +// result := u +// result.Path = strings.Replace(result.Path, "{base}", pricecfg.Base, 1) +// result.Path = strings.Replace(result.Path, "{quote}", pricecfg.Quote, 1) +// result.RawQuery = strings.Replace(result.RawQuery, "{base}", pricecfg.Base, 1) +// result.RawQuery = strings.Replace(result.RawQuery, "{quote}", pricecfg.Quote, 1) +// return &result +// } diff --git a/pricing/mocks/engine_mock.go b/pricing/mocks/engine_mock.go index 27b5b2a..cdf680a 100644 --- a/pricing/mocks/engine_mock.go +++ b/pricing/mocks/engine_mock.go @@ -5,9 +5,9 @@ package mocks import ( + gomock "github.com/golang/mock/gomock" config "github.com/vegaprotocol/priceproxy/config" pricing "github.com/vegaprotocol/priceproxy/pricing" - gomock "github.com/golang/mock/gomock" reflect "reflect" ) diff --git a/pricing/pricing.go b/pricing/pricing.go index b2cb214..e57e347 100644 --- a/pricing/pricing.go +++ b/pricing/pricing.go @@ -8,8 +8,6 @@ import ( "github.com/vegaprotocol/priceproxy/config" ) -const minPrice = 0.00001 - // PriceInfo describes a price from a source. // The price may be a real updated from an upstream source, or one that has been wandered. // The LastUpdated timstamps indicate when the price was last fetched for real and when (if at all) it was last wandered. @@ -53,6 +51,8 @@ type engine struct { func NewEngine(prices config.PriceList) Engine { e := engine{ priceList: prices, + pricesMu: sync.RWMutex{}, + sourcesMu: sync.Mutex{}, prices: make(map[config.PriceConfig]PriceInfo), sources: make(map[string]config.SourceConfig), } diff --git a/service/service.go b/service/service.go index 6cb353c..90ce7bb 100644 --- a/service/service.go +++ b/service/service.go @@ -50,6 +50,8 @@ func NewService(config config.Config) (*Service, error) { s := &Service{ Router: httprouter.New(), config: config, + server: nil, + pe: nil, } if err := s.initPricingEngine(); err != nil { @@ -163,7 +165,6 @@ func (s *Service) PricesGet(w http.ResponseWriter, r *http.Request, ps httproute (base == "" || base == k.Base) && (quote == "" || quote == k.Quote) && (wanderPtr == nil || *wanderPtr == k.Wander) { - returnedQuote := k.Quote if k.QuoteOverride != "" { returnedQuote = k.QuoteOverride @@ -178,8 +179,6 @@ func (s *Service) PricesGet(w http.ResponseWriter, r *http.Request, ps httproute LastUpdatedReal: v.LastUpdatedReal.String(), LastUpdatedWander: v.LastUpdatedWander.String(), }) - } else { - fmt.Printf("%v", k) } } writeSuccess(w, response, http.StatusOK) From 936e17518f939326776071e3e2e4778645d41d7b Mon Sep 17 00:00:00 2001 From: daniel1302 Date: Thu, 17 Nov 2022 11:26:23 +0100 Subject: [PATCH 07/10] feat: peer review changes --- cmd/priceproxy/main.go | 4 ++-- config/config_test.go | 2 +- go.mod | 2 +- pricing/bitstamp.go | 2 +- pricing/coingecko.go | 2 +- pricing/coinmarketcap.go | 2 +- pricing/http.go | 2 +- pricing/mocks/engine_mock.go | 6 +++--- pricing/pricing.go | 4 ++-- service/service.go | 4 ++-- 10 files changed, 15 insertions(+), 15 deletions(-) diff --git a/cmd/priceproxy/main.go b/cmd/priceproxy/main.go index 3c40a8e..0551595 100644 --- a/cmd/priceproxy/main.go +++ b/cmd/priceproxy/main.go @@ -15,8 +15,8 @@ import ( "github.com/jinzhu/configor" log "github.com/sirupsen/logrus" - "github.com/vegaprotocol/priceproxy/config" - "github.com/vegaprotocol/priceproxy/service" + "code.vegaprotocol.io/priceproxy/config" + "code.vegaprotocol.io/priceproxy/service" ) var ( diff --git a/config/config_test.go b/config/config_test.go index 993eced..79b96ca 100644 --- a/config/config_test.go +++ b/config/config_test.go @@ -7,7 +7,7 @@ import ( "github.com/stretchr/testify/assert" - "github.com/vegaprotocol/priceproxy/config" + "code.vegaprotocol.io/priceproxy/config" ) func TestCheckConfig(t *testing.T) { diff --git a/go.mod b/go.mod index da9ad99..e907bbf 100644 --- a/go.mod +++ b/go.mod @@ -1,4 +1,4 @@ -module github.com/vegaprotocol/priceproxy +module code.vegaprotocol.io/priceproxy go 1.19 diff --git a/pricing/bitstamp.go b/pricing/bitstamp.go index 988d29e..1866654 100644 --- a/pricing/bitstamp.go +++ b/pricing/bitstamp.go @@ -9,8 +9,8 @@ import ( "strings" "time" + "code.vegaprotocol.io/priceproxy/config" log "github.com/sirupsen/logrus" - "github.com/vegaprotocol/priceproxy/config" "golang.org/x/time/rate" ) diff --git a/pricing/coingecko.go b/pricing/coingecko.go index 7833212..059e6ba 100644 --- a/pricing/coingecko.go +++ b/pricing/coingecko.go @@ -8,8 +8,8 @@ import ( "strings" "time" + "code.vegaprotocol.io/priceproxy/config" log "github.com/sirupsen/logrus" - "github.com/vegaprotocol/priceproxy/config" "golang.org/x/time/rate" ) diff --git a/pricing/coinmarketcap.go b/pricing/coinmarketcap.go index 23575b9..5c402b2 100644 --- a/pricing/coinmarketcap.go +++ b/pricing/coinmarketcap.go @@ -9,8 +9,8 @@ import ( "strings" "time" + "code.vegaprotocol.io/priceproxy/config" log "github.com/sirupsen/logrus" - "github.com/vegaprotocol/priceproxy/config" "golang.org/x/time/rate" ) diff --git a/pricing/http.go b/pricing/http.go index 7d513d9..51e9bc6 100644 --- a/pricing/http.go +++ b/pricing/http.go @@ -3,8 +3,8 @@ package pricing import ( "time" + "code.vegaprotocol.io/priceproxy/config" log "github.com/sirupsen/logrus" - "github.com/vegaprotocol/priceproxy/config" ) func httpStartFetching( diff --git a/pricing/mocks/engine_mock.go b/pricing/mocks/engine_mock.go index cdf680a..defff3e 100644 --- a/pricing/mocks/engine_mock.go +++ b/pricing/mocks/engine_mock.go @@ -1,13 +1,13 @@ // Code generated by MockGen. DO NOT EDIT. -// Source: github.com/vegaprotocol/priceproxy/pricing (interfaces: Engine) +// Source: code.vegaprotocol.io/priceproxy/pricing (interfaces: Engine) // Package mocks is a generated GoMock package. package mocks import ( gomock "github.com/golang/mock/gomock" - config "github.com/vegaprotocol/priceproxy/config" - pricing "github.com/vegaprotocol/priceproxy/pricing" + config "code.vegaprotocol.io/priceproxy/config" + pricing "code.vegaprotocol.io/priceproxy/pricing" reflect "reflect" ) diff --git a/pricing/pricing.go b/pricing/pricing.go index e57e347..1a53f5b 100644 --- a/pricing/pricing.go +++ b/pricing/pricing.go @@ -5,7 +5,7 @@ import ( "sync" "time" - "github.com/vegaprotocol/priceproxy/config" + "code.vegaprotocol.io/priceproxy/config" ) // PriceInfo describes a price from a source. @@ -19,7 +19,7 @@ type PriceInfo struct { // Engine is the source of price information from multiple external/internal/fake sources. // -//go:generate go run github.com/golang/mock/mockgen -destination mocks/engine_mock.go -package mocks github.com/vegaprotocol/priceproxy/pricing Engine +//go:generate go run github.com/golang/mock/mockgen -destination mocks/engine_mock.go -package mocks code.vegaprotocol.io/priceproxy/pricing Engine type Engine interface { AddSource(sourcecfg config.SourceConfig) error GetSource(name string) (config.SourceConfig, error) diff --git a/service/service.go b/service/service.go index 90ce7bb..7a05294 100644 --- a/service/service.go +++ b/service/service.go @@ -8,8 +8,8 @@ import ( "strconv" "time" - "github.com/vegaprotocol/priceproxy/config" - "github.com/vegaprotocol/priceproxy/pricing" + "code.vegaprotocol.io/priceproxy/config" + "code.vegaprotocol.io/priceproxy/pricing" "github.com/julienschmidt/httprouter" log "github.com/sirupsen/logrus" From b9c82cf90f5b5271fecda6f1324569ff7b68ba19 Mon Sep 17 00:00:00 2001 From: daniel1302 Date: Thu, 17 Nov 2022 13:18:41 +0100 Subject: [PATCH 08/10] feat: prepare production config --- config.yaml | 231 +++++++++++++++++++++------------------ config/config.go | 1 + pricing/bitstamp.go | 2 +- pricing/coingecko.go | 77 +++++++++++-- pricing/coinmarketcap.go | 19 +--- service/service.go | 8 +- 6 files changed, 209 insertions(+), 129 deletions(-) diff --git a/config.yaml b/config.yaml index c327d1a..f9917bf 100644 --- a/config.yaml +++ b/config.yaml @@ -14,9 +14,8 @@ sources: path: /api/v2/ticker/ - name: coinmarketcap - # requires API key from $HOME/coinmarketcap-apikey.txt sleepReal: 1200 # seconds - auth_key_env_name: "CMC_PRO_API_KEY" + auth_key_env_name: "CMC_PRO_API_KEY" # this env variable must be exported url: scheme: https host: pro-api.coinmarketcap.com @@ -28,16 +27,7 @@ sources: scheme: https host: api.coingecko.com path: /api/v3/simple/price - rawquery: ids=solana,ethereum,bitcoin,terra-luna-2,uniswap,dai,aave,litecoin,optimism,monero,cosmos&vs_currencies=usd,eur,btc,eth&include_last_updated_at=true - - # - name: ethgasstation - # sleepReal: 1800 # seconds - # sleepWander: 30 # seconds - # # Injected at runtime: url.rawquery="api-key=..." - # url: - # scheme: https - # host: ethgasstation.info - # path: /api/ethgasAPI.json + rawquery: ids=solana,ethereum,bitcoin,terra-luna-2,uniswap,dai,aave,aapl,litecoin,optimism,monero,cosmos&vs_currencies=usd,eur,btc,eth&include_last_updated_at=true # - name: exchangeratesapi # sleepReal: 600 # seconds @@ -48,118 +38,108 @@ sources: # path: /latest # rawquery: base={base}&symbols={quote} - # - name: finhub - # sleepReal: 1800 # seconds - # sleepWander: 30 # seconds - # url: - # scheme: https - # host: finnhub.io - # path: /api/v1/quote - # rawquery: "symbol={base}" # yes, Quote is not mentioned - - # - name: ftx-aapl - # sleepReal: 60 # seconds - # sleepWander: 30 # seconds - # url: - # scheme: https - # host: ftx.com - # # Note: The format is TICKER-MMDD (month + day), keep it up to date - # path: /api/markets/AAPL-1230 - - # - name: ftx-aave - # sleepReal: 60 # seconds - # sleepWander: 30 # seconds - # url: - # scheme: https - # host: ftx.com - # path: /api/markets/AAVE-PERP - - # - name: ftx-tsla - # sleepReal: 60 # seconds - # sleepWander: 30 # seconds - # url: - # scheme: https - # host: ftx.com - # # Note: The format is TICKER-MMDD (month + day), keep it up to date - # path: /api/markets/TSLA-1230 - - # - name: ftx-uni - # sleepReal: 60 # seconds - # sleepWander: 30 # seconds - # url: - # scheme: https - # host: ftx.com - # path: /api/markets/UNI-PERP prices: + # Faked markets not available in our sources + - source: coinmarketcap + base: AAVE + base_override: AAPL + quote: USD + quote_override: USD + factor: 1.2 + wander: true + + - source: coingecko + base: monero + base_override: TSLA + quote: EUR + quote_override: EURO + factor: 1.4 + wander: true + + # Real currencies - source: coinmarketcap base: BTC quote: USD factor: 1.0 wander: true - - source: coinmarketcap - base: ETH - quote: BTC + - source: coingecko + base: monero + quote: ETH factor: 1.0 wander: true - - source: coinmarketcap - base: ETH - quote: DAI + - source: coingecko + base: terra-luna-2 + quote: ETH + factor: 1.0 + wander: true + + - source: coingecko + base: aave + quote: dai factor: 1.0 wander: true - - source: coinmarketcap - base: LUNA - quote: USD + - source: coingecko + base: aave + base_override: AAVE + quote: dai + quote_override: DAI factor: 1.0 wander: true - - source: coinmarketcap - base: XMR - quote: USD + - source: coingecko + base: monero + quote: usd factor: 1.0 wander: true - # - source: ftx-aave - # base: AAVE - # quote: DAI - # factor: 1.0 - # wander: true + - source: coingecko + base: bitcoin + quote: ETH + factor: 1.0 + wander: true - # - source: ftx-uni - # base: UNI - # quote: DAI - # factor: 1.0 - # wander: true + - source: coingecko + base: cosmos + quote: ETH + factor: 1.0 + wander: true - # - source: ftx-aapl - # base: AAPL - # quote: USD - # factor: 1.0 - # wander: true + - source: coingecko + base: terra-luna-2 + quote: usd + factor: 1.0 + wander: true - # - source: ftx-tsla - # base: TSLA - # quote: EURO - # factor: 0.951 # USD -> EURO - # wander: true + - source: coingecko + base: solana + quote: ETH + factor: 1.0 + wander: true - source: coingecko - base: aave - quote: dai + base: cosmos + quote: usd + factor: 1.0 + wander: true + + - source: coingecko + base: ethereum + quote: ETH factor: 1.0 wander: true - source: coingecko base: uniswap - quote: dai + quote: ETH factor: 1.0 wander: true - source: coingecko - base: terra-luna-2 + base: optimism quote: usd factor: 1.0 wander: true @@ -170,52 +150,93 @@ prices: factor: 1.0 wander: true + - source: coingecko + base: litecoin + quote: ETH + factor: 1.0 + wander: true + + - source: coingecko + base: dai + quote: ETH + factor: 1.0 + wander: true + - source: coingecko base: ethereum quote: btc factor: 1.0 wander: true + - source: coinmarketcap + base: LUNC + quote: USD + factor: 1.0 + wander: true + + - source: coinmarketcap + base: LUNC + base_override: LUNA + quote: USD + factor: 1.0 + wander: true + + - source: coinmarketcap + base: XMR + quote: USD + factor: 1.0 + wander: true + - source: coingecko - base: bitcoin - quote: usd + base: optimism + quote: ETH + factor: 1.0 + wander: true + + - source: coinmarketcap + base: UNI + quote: DAI factor: 1.0 wander: true - source: coingecko - base: monero - quote: usd + base: uniswap + quote: dai factor: 1.0 wander: true - source: coingecko - base: cosmos + base: bitcoin quote: usd factor: 1.0 wander: true - source: coingecko - base: optimism - quote: usd + base: aave + quote: ETH factor: 1.0 wander: true - - - source: bitstamp - base: IMX - quote: USD + - source: coinmarketcap + base: ETH + quote: BTC + factor: 1.0 + wander: true + + - source: coinmarketcap + base: ETH + quote: DAI factor: 1.0 wander: true - source: bitstamp base: IMX - quote: ETH + quote: USD factor: 1.0 wander: true - - source: bitstamp base: IMX - quote: BTC + quote: ETH factor: 1.0 - wander: true + wander: true \ No newline at end of file diff --git a/config/config.go b/config/config.go index 0e99c48..2bd09ea 100644 --- a/config/config.go +++ b/config/config.go @@ -24,6 +24,7 @@ type ServerConfig struct { type PriceConfig struct { Source string `yaml:"source"` Base string `yaml:"base"` + BaseOverride string `yaml:"base_override"` Quote string `yaml:"quote"` QuoteOverride string `yaml:"quote_override"` Factor float64 `yaml:"factor"` diff --git a/pricing/bitstamp.go b/pricing/bitstamp.go index 1866654..98e78da 100644 --- a/pricing/bitstamp.go +++ b/pricing/bitstamp.go @@ -125,7 +125,7 @@ func (fd bitstampCurrencyData) UnixTimestamp() time.Time { } func (fd bitstampCurrencyData) Price() float64 { - price, err := strconv.ParseFloat(fd.Timestamp, 64) + price, err := strconv.ParseFloat(fd.Last, 64) if err != nil { return 0.0 } diff --git a/pricing/coingecko.go b/pricing/coingecko.go index 059e6ba..7312c87 100644 --- a/pricing/coingecko.go +++ b/pricing/coingecko.go @@ -14,8 +14,7 @@ import ( ) var ( - coingeckoSourceName = "coingecko" - supportedQuotes = []string{"ETH", "EUR", "USD", "BTC", "DAI"} + supportedQuotes = []string{"ETH", "EUR", "USD", "BTC", "DAI"} ) func coingeckoStartFetching( @@ -31,7 +30,7 @@ func coingeckoStartFetching( ) log.WithFields(log.Fields{ - "sourceName": coingeckoSourceName, + "sourceName": sourcecfg.Name, "URL": fetchURL, "rateLimitDuration": oneRequestEvery, }).Infof("Starting Coingecko Fetching\n") @@ -40,7 +39,7 @@ func coingeckoStartFetching( if err = rateLimiter.Wait(ctx); err != nil { log.WithFields(log.Fields{ "error": err.Error(), - "sourceName": coingeckoSourceName, + "sourceName": sourcecfg.Name, "URL": fetchURL, "rateLimitDuration": oneRequestEvery, }).Errorln("Rate Limiter Failed. Falling back to Sleep.") @@ -52,7 +51,7 @@ func coingeckoStartFetching( if err != nil { log.WithFields(log.Fields{ "error": err.Error(), - "sourceName": coingeckoSourceName, + "sourceName": sourcecfg.Name, "URL": fetchURL, "rateLimitDuration": oneRequestEvery, }).Errorf("Retry in %d sec.\n", oneRequestEvery) @@ -79,7 +78,7 @@ func coingeckoStartFetching( fetchedPrice = coingeckoData.DAI default: log.WithFields(log.Fields{ - "sourceName": coingeckoSourceName, + "sourceName": sourcecfg.Name, "base": price.Base, "quote": price.Quote, "quote_override": price.QuoteOverride, @@ -89,7 +88,20 @@ func coingeckoStartFetching( if fetchedPrice == 0 { log.WithFields(log.Fields{ - "sourceName": coingeckoSourceName, + "sourceName": sourcecfg.Name, + "base": price.Base, + "quote": price.Quote, + "quote_override": price.QuoteOverride, + }).Debug("Quote/Base rate not found directly, trying conversion") + + if convertedPrice := prices.Convert(price.Base, price.Quote); convertedPrice > 0 { + fetchedPrice = convertedPrice + } + } + + if fetchedPrice == 0 { + log.WithFields(log.Fields{ + "sourceName": sourcecfg.Name, "base": price.Base, "quote": price.Quote, "quote_override": price.QuoteOverride, @@ -110,7 +122,7 @@ func coingeckoStartFetching( if !priceUpdated { log.WithFields(log.Fields{ - "sourceName": coingeckoSourceName, + "sourceName": sourcecfg.Name, "base": price.Base, "quote": price.Quote, "quote_override": price.QuoteOverride, @@ -120,7 +132,7 @@ func coingeckoStartFetching( } } -type coingeckoFetchData map[string]struct { +type coingeckoCurrencyData struct { USD float64 `json:"usd"` EUR float64 `json:"eur"` BTC float64 `json:"btc"` @@ -129,6 +141,53 @@ type coingeckoFetchData map[string]struct { LastUpdatedAt uint64 `json:"last_updated_at"` } +type coingeckoFetchData map[string]coingeckoCurrencyData + +func (fd coingeckoFetchData) Convert(base, quote string) float64 { + var baseData *coingeckoCurrencyData + var quoteData *coingeckoCurrencyData + for name, data := range fd { + data := data + if strings.EqualFold(base, name) { + baseData = &data + } + + if strings.EqualFold(quote, name) { + quoteData = &data + } + + if baseData != nil && quoteData != nil { + break + } + } + + if baseData == nil || quoteData == nil { + return 0.0 + } + + if baseData.USD > 0 && quoteData.USD > 0 { + return baseData.USD / quoteData.USD + } + + if baseData.EUR > 0 && quoteData.EUR > 0 { + return baseData.EUR / quoteData.EUR + } + + if baseData.DAI > 0 && quoteData.DAI > 0 { + return baseData.DAI / quoteData.DAI + } + + if baseData.BTC > 0 && quoteData.BTC > 0 { + return baseData.BTC / quoteData.BTC + } + + if baseData.ETH > 0 && quoteData.ETH > 0 { + return baseData.ETH / quoteData.ETH + } + + return 0.0 +} + func coingeckoSingleFetch(url string) (*coingeckoFetchData, error) { resp, err := http.Get(url) // nolint:noctx if err != nil { diff --git a/pricing/coinmarketcap.go b/pricing/coinmarketcap.go index 5c402b2..b2b4470 100644 --- a/pricing/coinmarketcap.go +++ b/pricing/coinmarketcap.go @@ -33,7 +33,7 @@ func coinmarketcapStartFetching( if apiKey == "" { log.WithFields(log.Fields{ - "sourceName": coingeckoSourceName, + "sourceName": sourcecfg.Name, "URL": sourcecfg.URL, "AuthKeyEnvName": sourcecfg.AuthKeyEnvName, }).Warnf("The API key is empty. Use the `auth_key_env_name` config for the source and export corresponding environment name") @@ -69,7 +69,7 @@ func coinmarketcapStartFetching( fetchedCurrency := coinmarketcapData.GetCurrency(price.Base) if fetchedCurrency == nil { log.WithFields(log.Fields{ - "sourceName": coingeckoSourceName, + "sourceName": sourcecfg.Name, "base": price.Base, "quote": price.Quote, "quote_override": price.QuoteOverride, @@ -80,14 +80,7 @@ func coinmarketcapStartFetching( fetchedQuote := fetchedCurrency.QuoteByName(price.Quote) fetchedPrice := 0.0 fetchedLastUpdate := fetchedCurrency.LastUpdated - if fetchedQuote == nil { - log.WithFields(log.Fields{ - "sourceName": coingeckoSourceName, - "base": price.Base, - "quote": price.Quote, - "quote_override": price.QuoteOverride, - }).Warnf("collected price in the quote current is 0, consider selecting different quote and overwrite it with the `quote_override` parameter") - } else { + if fetchedQuote != nil { fetchedPrice = fetchedQuote.Price fetchedLastUpdate = fetchedQuote.LastUpdated } @@ -96,7 +89,7 @@ func coinmarketcapStartFetching( if err != nil { log.WithFields(log.Fields{ "error": err.Error(), - "sourceName": coingeckoSourceName, + "sourceName": sourcecfg.Name, "base": price.Base, "quote": price.Quote, "quote_override": price.QuoteOverride, @@ -106,7 +99,7 @@ func coinmarketcapStartFetching( if fetchedPrice == 0 { log.WithFields(log.Fields{ - "sourceName": coingeckoSourceName, + "sourceName": sourcecfg.Name, "base": price.Base, "quote": price.Quote, "quote_override": price.QuoteOverride, @@ -117,7 +110,7 @@ func coinmarketcapStartFetching( if fetchedPrice == 0 { log.WithFields(log.Fields{ - "sourceName": coingeckoSourceName, + "sourceName": sourcecfg.Name, "base": price.Base, "quote": price.Quote, "quote_override": price.QuoteOverride, diff --git a/service/service.go b/service/service.go index 7a05294..6c01839 100644 --- a/service/service.go +++ b/service/service.go @@ -33,6 +33,7 @@ type Service struct { type PriceResponse struct { Source string `json:"source"` Base string `json:"base"` + BaseReal string `json:"base_real"` Quote string `json:"quote"` QuoteReal string `json:"quote_real"` Price float64 `json:"price"` @@ -169,10 +170,15 @@ func (s *Service) PricesGet(w http.ResponseWriter, r *http.Request, ps httproute if k.QuoteOverride != "" { returnedQuote = k.QuoteOverride } + returnedBase := k.Base + if k.BaseOverride != "" { + returnedBase = k.BaseOverride + } response.Prices = append(response.Prices, &PriceResponse{ Source: k.Source, - Base: k.Base, + Base: returnedBase, + BaseReal: k.Base, Quote: returnedQuote, QuoteReal: k.Quote, Price: v.Price * k.Factor, From 59a3febd593cd23d8691200634351b033aeaf33e Mon Sep 17 00:00:00 2001 From: daniel1302 Date: Thu, 17 Nov 2022 14:44:32 +0100 Subject: [PATCH 09/10] feat: peer review changes --- config.yaml | 2 +- pricing/bitstamp.go | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/config.yaml b/config.yaml index f9917bf..1fd9388 100644 --- a/config.yaml +++ b/config.yaml @@ -14,7 +14,7 @@ sources: path: /api/v2/ticker/ - name: coinmarketcap - sleepReal: 1200 # seconds + sleepReal: 400 # seconds auth_key_env_name: "CMC_PRO_API_KEY" # this env variable must be exported url: scheme: https diff --git a/pricing/bitstamp.go b/pricing/bitstamp.go index 98e78da..3101590 100644 --- a/pricing/bitstamp.go +++ b/pricing/bitstamp.go @@ -30,7 +30,7 @@ func bitstampStartFetching( "sourceName": sourcecfg.Name, "URL": fetchURL, "rateLimitDuration": oneRequestEvery, - }).Infof("Starting Coingecko Fetching\n") + }).Infof("Starting bitstamp Fetching\n") for { if err = rateLimiter.Wait(ctx); err != nil { From 97bd39dd79e6ecc69f12f271c5dcf724b1999d0a Mon Sep 17 00:00:00 2001 From: daniel1302 Date: Thu, 17 Nov 2022 15:03:22 +0100 Subject: [PATCH 10/10] feat: fix linting issue --- pricing/coingecko.go | 4 +--- pricing/mocks/engine_mock.go | 2 +- pricing/pricing.go | 12 ++++++++++++ 3 files changed, 14 insertions(+), 4 deletions(-) diff --git a/pricing/coingecko.go b/pricing/coingecko.go index 7312c87..66487a3 100644 --- a/pricing/coingecko.go +++ b/pricing/coingecko.go @@ -13,9 +13,7 @@ import ( "golang.org/x/time/rate" ) -var ( - supportedQuotes = []string{"ETH", "EUR", "USD", "BTC", "DAI"} -) +var supportedQuotes = []string{"ETH", "EUR", "USD", "BTC", "DAI"} func coingeckoStartFetching( board priceBoard, diff --git a/pricing/mocks/engine_mock.go b/pricing/mocks/engine_mock.go index defff3e..ad0f291 100644 --- a/pricing/mocks/engine_mock.go +++ b/pricing/mocks/engine_mock.go @@ -5,9 +5,9 @@ package mocks import ( - gomock "github.com/golang/mock/gomock" config "code.vegaprotocol.io/priceproxy/config" pricing "code.vegaprotocol.io/priceproxy/pricing" + gomock "github.com/golang/mock/gomock" reflect "reflect" ) diff --git a/pricing/pricing.go b/pricing/pricing.go index 1a53f5b..43c8fb3 100644 --- a/pricing/pricing.go +++ b/pricing/pricing.go @@ -133,7 +133,19 @@ func (e *engine) PriceList(source string) config.PriceList { return e.priceList.GetBySource(source) } +func (e *engine) initPrices() { + for _, price := range e.priceList { + e.UpdatePrice(price, PriceInfo{ + Price: 0.0, + LastUpdatedReal: time.Unix(0, 0), + LastUpdatedWander: time.Now(), + }) + } +} + func (e *engine) StartFetching() error { + e.initPrices() + for _, sourceConfig := range e.sources { if sourceConfig.IsCoinGecko() { go coingeckoStartFetching(e, sourceConfig)