add api rate limiter

exporter.go

@@ -13,6 +13,7 @@ import (
 	"github.com/helium-blockchain-exporter/heliumapi"
 	"github.com/prometheus/client_golang/prometheus"
 	"github.com/prometheus/client_golang/prometheus/promhttp"
+	"golang.org/x/time/rate"
 )
 
 // metricInfo is a metric exported by the helium blockchain exporter
@@ -646,10 +647,10 @@ func (e *Exporter) collectHotspotMetrics(wg *sync.WaitGroup, ch chan<- prometheu
 		}
 
 		// collect hotspot metric requiring extra queries in a new routine
-		wg.Add(5)
+		wg.Add(4)
 		go e.collectHotspotBlocksDeltaMetrics(wg, ch, account, hotspotData)
 		go e.collectHotspotWitnessesMetrics(wg, ch, account, hotspotData)
-		go e.collectHotspotWitnessedMetrics(wg, ch, account, hotspotData)
+		// go e.collectHotspotWitnessedMetrics(wg, ch, account, hotspotData)
 		go e.collectHotspotActivityMetrics(wg, ch, account, hotspotData)
 		go e.collectHotspotRewardsMetrics(wg, ch, account, hotspotData)
 
@@ -828,6 +829,7 @@ func main() {
 
 	fApiUrl := flag.String("apiUrl", "https://helium-api.stakejoy.com", "The helium api url")
 	fApiTimeout := flag.String("apiTimeout", "10s", "The request timeout to the helium api.")
+	fApiRateLimit := flag.Int("maxApiRate", 10, "The maximum number of helium api calls per seconds.")
 	fHeliumAccounts := flag.String("accounts", "", "A comma-delimited list of helium accounts to scrape (optional)")
 	fMetricsPath := flag.String("metricpath", "/metrics", "The metrics path")
 	fListenAddress := flag.String("listenAddress", "0.0.0.0", "The http server listen address")
@@ -835,9 +837,10 @@ func main() {
 	flag.Parse()
 
 	heliumapi.ApiUrl = *fApiUrl
+	heliumapi.Limiter = rate.NewLimiter(rate.Every(time.Duration(1000/(*fApiRateLimit))*time.Millisecond), *fApiRateLimit)
 	heliumapi.ApiTimeout, err = time.ParseDuration(*fApiTimeout)
 	if err != nil {
-		log.Fatalf("failed to parse apitTimeout: %s", err.Error())
+		log.Fatalf("failed to parse apiTimeout: %s", err.Error())
 	}
 
 	heliumAccounts := strings.Split(*fHeliumAccounts, ",")

go.mod

@@ -13,5 +13,6 @@ require (
 	github.com/prometheus/common v0.29.0 // indirect
 	github.com/prometheus/procfs v0.6.0 // indirect
 	golang.org/x/sys v0.0.0-20210615035016-665e8c7367d1 // indirect
+	golang.org/x/time v0.0.0-20210723032227-1f47c861a9ac // indirect
 	google.golang.org/protobuf v1.26.0-rc.1 // indirect
 )

go.sum

@@ -317,6 +317,8 @@ golang.org/x/text v0.3.6/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=
 golang.org/x/time v0.0.0-20181108054448-85acf8d2951c/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ=
 golang.org/x/time v0.0.0-20190308202827-9d24e82272b4/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ=
 golang.org/x/time v0.0.0-20191024005414-555d28b269f0/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ=
+golang.org/x/time v0.0.0-20210723032227-1f47c861a9ac h1:7zkz7BUtwNFFqcowJ+RIgu2MaV/MapERkDIy+mwPyjs=
+golang.org/x/time v0.0.0-20210723032227-1f47c861a9ac/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ=
 golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
 golang.org/x/tools v0.0.0-20190114222345-bf090417da8b/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
 golang.org/x/tools v0.0.0-20190226205152-f727befe758c/go.mod h1:9Yl7xja0Znq3iFh3HoIrodX9oNMXvdceNzlUR8zjMvY=

heliumapi/query.go

@@ -7,12 +7,17 @@ import (
 	"io/ioutil"
 	"net/http"
 	"time"
+
+	"golang.org/x/time/rate"
 )
 
 var (
-	ApiUrl     = "https://api.helium.io"
+	// those are set via flags on init
-	ApiTimeout = time.Duration(10 * time.Second)
+	ApiUrl     string
-	client     = &http.Client{}
+	ApiTimeout time.Duration
+	Limiter    *rate.Limiter
+
+	client = &http.Client{}
 )
 
 // createGetRequest create a GET request to the helium api
@@ -22,6 +27,7 @@ func createGetRequest(path string, params map[string]string) (*http.Request, *co
 
 	// setup headers
 	req.Header.Add("Accept", "application/json")
+	req.Header.Add("User-Agent", "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/95.0.4638.54 Safari/537.36")
 
 	// setup query param if there are any
 	if params != nil {
@@ -46,6 +52,13 @@ func getHeliumApi(path string, params *map[string]string) ([]byte, error) {
 		params = &map[string]string{}
 	}
 
+	// ensure we respect the rate limit
+	err := Limiter.Wait(context.Background())
+	if err != nil {
+		return nil, err
+	}
+
+	// create the request
 	req, ctxCancel, err := createGetRequest(path, *params)
 	if err != nil {
 		return nil, fmt.Errorf("failed to create query request %v: %v", path, err)
@@ -87,6 +100,7 @@ func getHeliumApiWithCursor(path string, params *map[string]string) ([][]byte, e
 		Cursor string `json:"cursor"`
 	}
 
+	// create the initial request
 	res := [][]byte{}
 	respCursor := ResponseWithCursor{}
 	req, ctxCancel, err := createGetRequest(path, *params)
@@ -96,6 +110,12 @@ func getHeliumApiWithCursor(path string, params *map[string]string) ([][]byte, e
 	defer (*ctxCancel)()
 
 	for {
+		// ensure we respect the rate limit
+		err := Limiter.Wait(context.Background())
+		if err != nil {
+			return nil, err
+		}
+
 		// query the api
 		resp, err := client.Do(req)
 		if err != nil {