From dec12e1780b0e41a2e7fe3d44f62015a1e8cb48c Mon Sep 17 00:00:00 2001 From: Massaki Archambault Date: Sun, 14 Nov 2021 10:44:39 -0500 Subject: [PATCH] add api rate limiter --- exporter.go | 9 ++++++--- go.mod | 1 + go.sum | 2 ++ heliumapi/query.go | 26 +++++++++++++++++++++++--- 4 files changed, 32 insertions(+), 6 deletions(-) diff --git a/exporter.go b/exporter.go index 5e7b761..73bc84b 100644 --- a/exporter.go +++ b/exporter.go @@ -13,6 +13,7 @@ import ( "github.com/helium-blockchain-exporter/heliumapi" "github.com/prometheus/client_golang/prometheus" "github.com/prometheus/client_golang/prometheus/promhttp" + "golang.org/x/time/rate" ) // metricInfo is a metric exported by the helium blockchain exporter @@ -646,10 +647,10 @@ func (e *Exporter) collectHotspotMetrics(wg *sync.WaitGroup, ch chan<- prometheu } // collect hotspot metric requiring extra queries in a new routine - wg.Add(5) + wg.Add(4) go e.collectHotspotBlocksDeltaMetrics(wg, ch, account, hotspotData) go e.collectHotspotWitnessesMetrics(wg, ch, account, hotspotData) - go e.collectHotspotWitnessedMetrics(wg, ch, account, hotspotData) + // go e.collectHotspotWitnessedMetrics(wg, ch, account, hotspotData) go e.collectHotspotActivityMetrics(wg, ch, account, hotspotData) go e.collectHotspotRewardsMetrics(wg, ch, account, hotspotData) @@ -828,6 +829,7 @@ func main() { fApiUrl := flag.String("apiUrl", "https://helium-api.stakejoy.com", "The helium api url") fApiTimeout := flag.String("apiTimeout", "10s", "The request timeout to the helium api.") + fApiRateLimit := flag.Int("maxApiRate", 10, "The maximum number of helium api calls per seconds.") fHeliumAccounts := flag.String("accounts", "", "A comma-delimited list of helium accounts to scrape (optional)") fMetricsPath := flag.String("metricpath", "/metrics", "The metrics path") fListenAddress := flag.String("listenAddress", "0.0.0.0", "The http server listen address") @@ -835,9 +837,10 @@ func main() { flag.Parse() heliumapi.ApiUrl = *fApiUrl + heliumapi.Limiter = rate.NewLimiter(rate.Every(time.Duration(1000/(*fApiRateLimit))*time.Millisecond), *fApiRateLimit) heliumapi.ApiTimeout, err = time.ParseDuration(*fApiTimeout) if err != nil { - log.Fatalf("failed to parse apitTimeout: %s", err.Error()) + log.Fatalf("failed to parse apiTimeout: %s", err.Error()) } heliumAccounts := strings.Split(*fHeliumAccounts, ",") diff --git a/go.mod b/go.mod index d0a766e..0cd4916 100644 --- a/go.mod +++ b/go.mod @@ -13,5 +13,6 @@ require ( github.com/prometheus/common v0.29.0 // indirect github.com/prometheus/procfs v0.6.0 // indirect golang.org/x/sys v0.0.0-20210615035016-665e8c7367d1 // indirect + golang.org/x/time v0.0.0-20210723032227-1f47c861a9ac // indirect google.golang.org/protobuf v1.26.0-rc.1 // indirect ) diff --git a/go.sum b/go.sum index 039b09e..2bdbdf8 100644 --- a/go.sum +++ b/go.sum @@ -317,6 +317,8 @@ golang.org/x/text v0.3.6/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/time v0.0.0-20181108054448-85acf8d2951c/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/time v0.0.0-20190308202827-9d24e82272b4/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/time v0.0.0-20191024005414-555d28b269f0/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= +golang.org/x/time v0.0.0-20210723032227-1f47c861a9ac h1:7zkz7BUtwNFFqcowJ+RIgu2MaV/MapERkDIy+mwPyjs= +golang.org/x/time v0.0.0-20210723032227-1f47c861a9ac/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= golang.org/x/tools v0.0.0-20190114222345-bf090417da8b/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= golang.org/x/tools v0.0.0-20190226205152-f727befe758c/go.mod h1:9Yl7xja0Znq3iFh3HoIrodX9oNMXvdceNzlUR8zjMvY= diff --git a/heliumapi/query.go b/heliumapi/query.go index addb6ee..42f6f08 100644 --- a/heliumapi/query.go +++ b/heliumapi/query.go @@ -7,12 +7,17 @@ import ( "io/ioutil" "net/http" "time" + + "golang.org/x/time/rate" ) var ( - ApiUrl = "https://api.helium.io" - ApiTimeout = time.Duration(10 * time.Second) - client = &http.Client{} + // those are set via flags on init + ApiUrl string + ApiTimeout time.Duration + Limiter *rate.Limiter + + client = &http.Client{} ) // createGetRequest create a GET request to the helium api @@ -22,6 +27,7 @@ func createGetRequest(path string, params map[string]string) (*http.Request, *co // setup headers req.Header.Add("Accept", "application/json") + req.Header.Add("User-Agent", "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/95.0.4638.54 Safari/537.36") // setup query param if there are any if params != nil { @@ -46,6 +52,13 @@ func getHeliumApi(path string, params *map[string]string) ([]byte, error) { params = &map[string]string{} } + // ensure we respect the rate limit + err := Limiter.Wait(context.Background()) + if err != nil { + return nil, err + } + + // create the request req, ctxCancel, err := createGetRequest(path, *params) if err != nil { return nil, fmt.Errorf("failed to create query request %v: %v", path, err) @@ -87,6 +100,7 @@ func getHeliumApiWithCursor(path string, params *map[string]string) ([][]byte, e Cursor string `json:"cursor"` } + // create the initial request res := [][]byte{} respCursor := ResponseWithCursor{} req, ctxCancel, err := createGetRequest(path, *params) @@ -96,6 +110,12 @@ func getHeliumApiWithCursor(path string, params *map[string]string) ([][]byte, e defer (*ctxCancel)() for { + // ensure we respect the rate limit + err := Limiter.Wait(context.Background()) + if err != nil { + return nil, err + } + // query the api resp, err := client.Do(req) if err != nil {