From dff54fd17496fc607127763c53e4e061debb105e Mon Sep 17 00:00:00 2001 From: MaysWind Date: Sun, 10 Jan 2021 01:07:37 +0800 Subject: [PATCH] refactor exchange rates data backend, supports multi data source, supports timeout --- cmd/initializer.go | 8 ++ conf/lab.ini | 7 ++ pkg/api/exchange_rates.go | 40 +++---- pkg/errs/setting.go | 9 +- .../euro_central_bank_datasource.go | 103 ++++++++++++++++++ .../exchange_rates_datasource.go | 15 +++ .../exchange_rates_datasource_container.go | 26 +++++ pkg/models/exchange_rate.go | 59 ---------- pkg/settings/setting.go | 29 +++++ 9 files changed, 211 insertions(+), 85 deletions(-) create mode 100644 pkg/exchangerates/euro_central_bank_datasource.go create mode 100644 pkg/exchangerates/exchange_rates_datasource.go create mode 100644 pkg/exchangerates/exchange_rates_datasource_container.go diff --git a/cmd/initializer.go b/cmd/initializer.go index 63240f6b..6a155860 100644 --- a/cmd/initializer.go +++ b/cmd/initializer.go @@ -7,6 +7,7 @@ import ( "github.com/urfave/cli/v2" "github.com/mayswind/lab/pkg/datastore" + "github.com/mayswind/lab/pkg/exchangerates" "github.com/mayswind/lab/pkg/log" "github.com/mayswind/lab/pkg/settings" "github.com/mayswind/lab/pkg/utils" @@ -65,6 +66,13 @@ func initializeSystem(c *cli.Context) (*settings.Config, error) { return nil, err } + err = exchangerates.InitializeExchangeRatesDataSource(config) + + if err != nil { + log.BootErrorf("[initializer.initializeSystem] initializes exchange rates data source failed, because %s", err.Error()) + return nil, err + } + cfgJson, _ := json.Marshal(getConfigWithNoSensitiveData(config)) log.BootInfof("[initializer.initializeSystem] has loaded configuration %s", cfgJson) diff --git a/conf/lab.ini b/conf/lab.ini index 7960fe86..e14c4b2f 100644 --- a/conf/lab.ini +++ b/conf/lab.ini @@ -109,3 +109,10 @@ enable_register = true [data] # Set to true to allow users to export their data enable_export = true + +[exchange_rates] +# Exchange rates data source, supports "euro_central_bank" currently +data_source = euro_central_bank + +# Requesting exchange rates data timeout (milliseconds), default is 10000 (10 seconds) +request_timeout = 10000 diff --git a/pkg/api/exchange_rates.go b/pkg/api/exchange_rates.go index 593d87be..5bcef04b 100644 --- a/pkg/api/exchange_rates.go +++ b/pkg/api/exchange_rates.go @@ -1,19 +1,17 @@ package api import ( - "encoding/xml" "io/ioutil" "net/http" + "time" "github.com/mayswind/lab/pkg/core" "github.com/mayswind/lab/pkg/errs" + "github.com/mayswind/lab/pkg/exchangerates" "github.com/mayswind/lab/pkg/log" - "github.com/mayswind/lab/pkg/models" + "github.com/mayswind/lab/pkg/settings" ) -// EuroCentralBankExchangeRateUrl represents euro central bank exchange rate date url -const EuroCentralBankExchangeRateUrl = "https://www.ecb.europa.eu/stats/eurofxref/eurofxref-daily.xml" - // ExchangeRatesApi represents exchange rate api type ExchangeRatesApi struct{} @@ -24,8 +22,19 @@ var ( // LatestExchangeRateHandler returns latest exchange rate data func (a *ExchangeRatesApi) LatestExchangeRateHandler(c *core.Context) (interface{}, *errs.Error) { + dataSource := exchangerates.Container.Current + + if dataSource == nil { + return nil, errs.ErrInvalidExchangeRatesDataSource + } + uid := c.GetCurrentUid() - resp, err := http.Get(EuroCentralBankExchangeRateUrl) + + client := &http.Client{ + Timeout: time.Duration(settings.Container.Current.ExchangeRatesRequestTimeout) * time.Millisecond, + } + + resp, err := client.Get(dataSource.GetRequestUrl()) if err != nil { log.ErrorfWithRequestId(c, "[exchange_rates.LatestExchangeRateHandler] failed to request latest exchange rate data for user \"uid:%d\", because %s", uid, err.Error()) @@ -39,25 +48,12 @@ func (a *ExchangeRatesApi) LatestExchangeRateHandler(c *core.Context) (interface defer resp.Body.Close() body, err := ioutil.ReadAll(resp.Body) - euroCentralBankData := &models.EuroCentralBankExchangeRateData{} - err = xml.Unmarshal(body, euroCentralBankData) + latestExchangeRateResponse, err := dataSource.Parse(c, body) if err != nil { - log.ErrorfWithRequestId(c, "[exchange_rates.LatestExchangeRateHandler] failed to parse xml data for user \"uid:%d\", response is %s, because %s", uid, string(body), err.Error()) - return nil, errs.ErrFailedToRequestRemoteApi + log.ErrorfWithRequestId(c, "[exchange_rates.LatestExchangeRateHandler] failed to parse response for user \"uid:%d\", because %s", uid, err.Error()) + return nil, errs.Or(err, errs.ErrFailedToRequestRemoteApi) } - latestExchangeRateResponse := euroCentralBankData.ToLatestExchangeRateResponse() - - if latestExchangeRateResponse == nil { - log.ErrorfWithRequestId(c, "[exchange_rates.LatestExchangeRateHandler] failed to parse latest exchange rate data for user \"uid:%d\", response is %s,", uid, string(body)) - return nil, errs.ErrFailedToRequestRemoteApi - } - - latestExchangeRateResponse.ExchangeRates = append(latestExchangeRateResponse.ExchangeRates, &models.LatestExchangeRate{ - Currency: "EUR", - Rate: "1", - }) - return latestExchangeRateResponse, nil } diff --git a/pkg/errs/setting.go b/pkg/errs/setting.go index f8ab43ba..1be135a5 100644 --- a/pkg/errs/setting.go +++ b/pkg/errs/setting.go @@ -4,8 +4,9 @@ import "net/http" // Error codes related to settings var ( - ErrInvalidProtocol = NewSystemError(SystemSubcategorySetting, 0, http.StatusInternalServerError, "invalid server protocol") - ErrInvalidLogMode = NewSystemError(SystemSubcategorySetting, 1, http.StatusInternalServerError, "invalid log mode") - ErrGettingLocalAddress = NewSystemError(SystemSubcategorySetting, 2, http.StatusInternalServerError, "failed to get local address") - ErrInvalidUuidMode = NewSystemError(SystemSubcategorySetting, 3, http.StatusInternalServerError, "invalid uuid mode") + ErrInvalidProtocol = NewSystemError(SystemSubcategorySetting, 0, http.StatusInternalServerError, "invalid server protocol") + ErrInvalidLogMode = NewSystemError(SystemSubcategorySetting, 1, http.StatusInternalServerError, "invalid log mode") + ErrGettingLocalAddress = NewSystemError(SystemSubcategorySetting, 2, http.StatusInternalServerError, "failed to get local address") + ErrInvalidUuidMode = NewSystemError(SystemSubcategorySetting, 3, http.StatusInternalServerError, "invalid uuid mode") + ErrInvalidExchangeRatesDataSource = NewSystemError(SystemSubcategorySetting, 4, http.StatusInternalServerError, "invalid exchange rates data source") ) diff --git a/pkg/exchangerates/euro_central_bank_datasource.go b/pkg/exchangerates/euro_central_bank_datasource.go new file mode 100644 index 00000000..2f510987 --- /dev/null +++ b/pkg/exchangerates/euro_central_bank_datasource.go @@ -0,0 +1,103 @@ +package exchangerates + +import ( + "encoding/xml" + + "github.com/mayswind/lab/pkg/core" + "github.com/mayswind/lab/pkg/errs" + "github.com/mayswind/lab/pkg/log" + "github.com/mayswind/lab/pkg/models" +) + +const euroCentralBankExchangeRateUrl = "https://www.ecb.europa.eu/stats/eurofxref/eurofxref-daily.xml" +const euroCentralBankDataSource = "European Central Bank" +const euroCentralBankBaseCurrency = "EUR" + +// EuroCentralBankDataSource defines the structure of exchange rates data source of euro central bank +type EuroCentralBankDataSource struct { + ExchangeRatesDataSource +} + +// EuroCentralBankExchangeRateData represents the whole data from euro central bank +type EuroCentralBankExchangeRateData struct { + XMLName xml.Name `xml:"Envelope"` + AllExchangeRates []*EuroCentralBankExchangeRates `xml:"Cube>Cube"` +} + +// EuroCentralBankExchangeRates represents the exchange rates data from euro central bank +type EuroCentralBankExchangeRates struct { + Date string `xml:"time,attr"` + ExchangeRates []*EuroCentralBankExchangeRate `xml:"Cube"` +} + +// EuroCentralBankExchangeRate represents the exchange rate data from euro central bank +type EuroCentralBankExchangeRate struct { + Currency string `xml:"currency,attr"` + Rate string `xml:"rate,attr"` +} + +// ToLatestExchangeRateResponse returns a view-object according to original data from euro central bank +func (e *EuroCentralBankExchangeRateData) ToLatestExchangeRateResponse() *models.LatestExchangeRateResponse { + if len(e.AllExchangeRates) < 1 { + return nil + } + + latestEuroCentralBankExchangeRate := e.AllExchangeRates[0] + + if len(latestEuroCentralBankExchangeRate.ExchangeRates) < 1 { + return nil + } + + exchangeRates := make([]*models.LatestExchangeRate, len(latestEuroCentralBankExchangeRate.ExchangeRates)) + + for i := 0; i < len(latestEuroCentralBankExchangeRate.ExchangeRates); i++ { + exchangeRates[i] = latestEuroCentralBankExchangeRate.ExchangeRates[i].ToLatestExchangeRate() + } + + latestExchangeRateResp := &models.LatestExchangeRateResponse{ + DataSource: euroCentralBankDataSource, + Date: latestEuroCentralBankExchangeRate.Date, + BaseCurrency: euroCentralBankBaseCurrency, + ExchangeRates: exchangeRates, + } + + return latestExchangeRateResp +} + +// ToLatestExchangeRate returns a data pair according to original data from euro central bank +func (e *EuroCentralBankExchangeRate) ToLatestExchangeRate() *models.LatestExchangeRate { + return &models.LatestExchangeRate{ + Currency: e.Currency, + Rate: e.Rate, + } +} + +// GetRequestUrl returns the euro central bank data source url +func (e *EuroCentralBankDataSource) GetRequestUrl() string { + return euroCentralBankExchangeRateUrl +} + +// Parse returns the common response entity according to the euro central bank data source raw response +func (e *EuroCentralBankDataSource) Parse(c *core.Context, content []byte) (*models.LatestExchangeRateResponse, error) { + euroCentralBankData := &EuroCentralBankExchangeRateData{} + err := xml.Unmarshal(content, euroCentralBankData) + + if err != nil { + log.ErrorfWithRequestId(c, "[euro_central_bank_datasource.Parse] failed to parse xml data, content is %s, because %s", string(content), err.Error()) + return nil, errs.ErrFailedToRequestRemoteApi + } + + latestExchangeRateResponse := euroCentralBankData.ToLatestExchangeRateResponse() + + if latestExchangeRateResponse == nil { + log.ErrorfWithRequestId(c, "[euro_central_bank_datasource.Parse] failed to parse latest exchange rate data, content is %s", string(content)) + return nil, errs.ErrFailedToRequestRemoteApi + } + + latestExchangeRateResponse.ExchangeRates = append(latestExchangeRateResponse.ExchangeRates, &models.LatestExchangeRate{ + Currency: euroCentralBankBaseCurrency, + Rate: "1", + }) + + return latestExchangeRateResponse, nil +} diff --git a/pkg/exchangerates/exchange_rates_datasource.go b/pkg/exchangerates/exchange_rates_datasource.go new file mode 100644 index 00000000..ceb68a79 --- /dev/null +++ b/pkg/exchangerates/exchange_rates_datasource.go @@ -0,0 +1,15 @@ +package exchangerates + +import ( + "github.com/mayswind/lab/pkg/core" + "github.com/mayswind/lab/pkg/models" +) + +// ExchangeRatesDataSource defines the structure of exchange rates data source +type ExchangeRatesDataSource interface { + // GetRequestUrl returns the data source url + GetRequestUrl() string + + // Parse returns the common response entity according to the data source raw response + Parse(c *core.Context, content []byte) (*models.LatestExchangeRateResponse, error) +} diff --git a/pkg/exchangerates/exchange_rates_datasource_container.go b/pkg/exchangerates/exchange_rates_datasource_container.go new file mode 100644 index 00000000..ec12cd36 --- /dev/null +++ b/pkg/exchangerates/exchange_rates_datasource_container.go @@ -0,0 +1,26 @@ +package exchangerates + +import ( + "github.com/mayswind/lab/pkg/errs" + "github.com/mayswind/lab/pkg/settings" +) + +// ExchangeRatesDataSourceContainer contains the current exchange rates data source +type ExchangeRatesDataSourceContainer struct { + Current ExchangeRatesDataSource +} + +// Initialize a exchange rates data source container singleton instance +var ( + Container = &ExchangeRatesDataSourceContainer{} +) + +// InitializeExchangeRatesDataSource initializes the current exchange rates data source according to the config +func InitializeExchangeRatesDataSource(config *settings.Config) error { + if config.ExchangeRatesDataSource == settings.EuroCentralBankDataSource { + Container.Current = &EuroCentralBankDataSource{} + return nil + } + + return errs.ErrInvalidExchangeRatesDataSource +} diff --git a/pkg/models/exchange_rate.go b/pkg/models/exchange_rate.go index 40649ac4..2aeafe7f 100644 --- a/pkg/models/exchange_rate.go +++ b/pkg/models/exchange_rate.go @@ -1,10 +1,5 @@ package models -import "encoding/xml" - -const euroCentralBankDataSource = "European Central Bank" -const euroCentralBankBaseCurrency = "EUR" - // LatestExchangeRateResponse returns a view-object which contains latest exchange rate type LatestExchangeRateResponse struct { DataSource string `json:"dataSource"` @@ -18,57 +13,3 @@ type LatestExchangeRate struct { Currency string `json:"currency"` Rate string `json:"rate"` } - -// EuroCentralBankExchangeRateData represents the whole data from euro central bank -type EuroCentralBankExchangeRateData struct { - XMLName xml.Name `xml:"Envelope"` - AllExchangeRates []*EuroCentralBankExchangeRates `xml:"Cube>Cube"` -} - -// EuroCentralBankExchangeRates represents the exchange rates data from euro central bank -type EuroCentralBankExchangeRates struct { - Date string `xml:"time,attr"` - ExchangeRates []*EuroCentralBankExchangeRate `xml:"Cube"` -} - -// EuroCentralBankExchangeRate represents the exchange rate data from euro central bank -type EuroCentralBankExchangeRate struct { - Currency string `xml:"currency,attr"` - Rate string `xml:"rate,attr"` -} - -// ToLatestExchangeRateResponse returns a view-object according to original data from euro central bank -func (e EuroCentralBankExchangeRateData) ToLatestExchangeRateResponse() *LatestExchangeRateResponse { - if len(e.AllExchangeRates) < 1 { - return nil - } - - latestEuroCentralBankExchangeRate := e.AllExchangeRates[0] - - if len(latestEuroCentralBankExchangeRate.ExchangeRates) < 1 { - return nil - } - - exchangeRates := make([]*LatestExchangeRate, len(latestEuroCentralBankExchangeRate.ExchangeRates)) - - for i := 0; i < len(latestEuroCentralBankExchangeRate.ExchangeRates); i++ { - exchangeRates[i] = latestEuroCentralBankExchangeRate.ExchangeRates[i].ToLatestExchangeRate() - } - - latestExchangeRateResp := &LatestExchangeRateResponse{ - DataSource: euroCentralBankDataSource, - Date: latestEuroCentralBankExchangeRate.Date, - BaseCurrency: euroCentralBankBaseCurrency, - ExchangeRates: exchangeRates, - } - - return latestExchangeRateResp -} - -// ToLatestExchangeRate returns a data pair according to original data from euro central bank -func (e EuroCentralBankExchangeRate) ToLatestExchangeRate() *LatestExchangeRate { - return &LatestExchangeRate{ - Currency: e.Currency, - Rate: e.Rate, - } -} diff --git a/pkg/settings/setting.go b/pkg/settings/setting.go index a06ca5e6..c286296e 100644 --- a/pkg/settings/setting.go +++ b/pkg/settings/setting.go @@ -62,6 +62,11 @@ const ( InternalUuidGeneratorType string = "internal" ) +// Exchange rates data source types +const ( + EuroCentralBankDataSource string = "euro_central_bank" +) + const ( defaultAppName string = "lab" @@ -81,6 +86,8 @@ const ( defaultSecretKey string = "lab" defaultTokenExpiredTime int = 604800 // 7 days defaultTemporaryTokenExpiredTime int = 300 // 5 minutes + + defaultExchangeRatesDataRequestTimeout int = 10000 // 10 seconds ) // DatabaseConfig represents the database setting config @@ -156,6 +163,10 @@ type Config struct { // Data EnableDataExport bool + + // Exchange Rates + ExchangeRatesDataSource string + ExchangeRatesRequestTimeout int } // LoadConfiguration loads setting config from given config file path @@ -223,6 +234,12 @@ func LoadConfiguration(configFilePath string) (*Config, error) { return nil, err } + err = loadExchangeRatesConfiguration(config, cfgFile, "exchange_rates") + + if err != nil { + return nil, err + } + return config, nil } @@ -392,6 +409,18 @@ func loadDataConfiguration(config *Config, configFile *ini.File, sectionName str return nil } +func loadExchangeRatesConfiguration(config *Config, configFile *ini.File, sectionName string) error { + if getConfigItemStringValue(configFile, sectionName, "data_source") == EuroCentralBankDataSource { + config.ExchangeRatesDataSource = EuroCentralBankDataSource + } else { + return errs.ErrInvalidExchangeRatesDataSource + } + + config.ExchangeRatesRequestTimeout = getConfigItemIntValue(configFile, sectionName, "request_timeout", defaultExchangeRatesDataRequestTimeout) + + return nil +} + func getWorkingPath() (string, error) { workingPath := os.Getenv(labWorkDirEnvName)