expose hotspot block gap

exporter.go

@@ -3,7 +3,6 @@ package main
 import (
 	"flag"
 	"log"
-	"math"
 	"net/http"
 	"os"
 	"strconv"
@@ -117,7 +116,7 @@ var (
 	statsBlocks = metricInfo{
 		prometheus.NewDesc(
 			prometheus.BuildFQName(namespace, "stats", "blocks"),
-			"The total height/number of blocks in the blockchain.",
+			"The height of the blockchain.",
 			nil, nil,
 		),
 		prometheus.GaugeValue,
@@ -199,7 +198,7 @@ var (
 	accountDepositsHnt = metricInfo{
 		prometheus.NewDesc(
 			prometheus.BuildFQName(namespace, "account", "deposits_hnt_total"),
-			"The number of HNT tokens deposited to this account.",
+			"The number of HNT tokens deposited to an account.",
 			commonAccountLabels, nil,
 		),
 		prometheus.CounterValue,
@@ -207,7 +206,7 @@ var (
 	accountWithdrawalsHnt = metricInfo{
 		prometheus.NewDesc(
 			prometheus.BuildFQName(namespace, "account", "withdrawals_hnt_total"),
-			"The number of HNT tokens withdrawn from this account.",
+			"The number of HNT tokens withdrawn from an account.",
 			commonAccountLabels, nil,
 		),
 		prometheus.CounterValue,
@@ -222,14 +221,6 @@ var (
 		),
 		prometheus.GaugeValue,
 	}
-	hotspotSynced = metricInfo{
-		prometheus.NewDesc(
-			prometheus.BuildFQName(namespace, "hotspot", "synced"),
-			"Whether a hotspot is synced with the blockchain.",
-			commonHotspotLabels, nil,
-		),
-		prometheus.GaugeValue,
-	}
 	hotspotRelayed = metricInfo{
 		prometheus.NewDesc(
 			prometheus.BuildFQName(namespace, "hotspot", "relayed"),
@@ -240,11 +231,27 @@ var (
 	}
 	hotspotBlocks = metricInfo{
 		prometheus.NewDesc(
-			prometheus.BuildFQName(namespace, "hotspot", "blocks_total"),
+			prometheus.BuildFQName(namespace, "hotspot", "blocks"),
 			"The block height of a hotspot. Check on the hotspot itself for the most recent data.",
 			commonHotspotLabels, nil,
 		),
-		prometheus.CounterValue,
+		prometheus.GaugeValue,
+	}
+	hotspotBlocksDelta = metricInfo{
+		prometheus.NewDesc(
+			prometheus.BuildFQName(namespace, "hotspot", "blocks_delta"),
+			"The gap between the height of a hotspot and the height of the blockchain. A large negative gap may indicate the hotspot is out of sync. Check on the hotspot itself for the most recent data.",
+			commonHotspotLabels, nil,
+		),
+		prometheus.GaugeValue,
+	}
+	hotspotStatusTimestamp = metricInfo{
+		prometheus.NewDesc(
+			prometheus.BuildFQName(namespace, "hotspot", "status_timestamp"),
+			"The the last time a hotspot gossiped its status on the blockchain.",
+			commonHotspotLabels, nil,
+		),
+		prometheus.GaugeValue,
 	}
 	hotspotRewardsScale = metricInfo{
 		prometheus.NewDesc(
@@ -334,9 +341,10 @@ func (e *Exporter) Describe(ch chan<- *prometheus.Desc) {
 	ch <- accountWithdrawalsHnt.Desc
 
 	ch <- hotspotUp.Desc
-	ch <- hotspotSynced.Desc
 	ch <- hotspotRelayed.Desc
+	ch <- hotspotBlocksDelta.Desc
 	ch <- hotspotBlocks.Desc
+	ch <- hotspotStatusTimestamp.Desc
 	ch <- hotspotRewardsScale.Desc
 	ch <- hotspot5dWitnesses.Desc
 	ch <- hotspot5dWitnessed.Desc
@@ -607,10 +615,16 @@ func (e *Exporter) collectHotspotMetrics(wg *sync.WaitGroup, ch chan<- prometheu
 		return
 	}
 
-	for _, hotspotData := range *hotspotsForAddress {
+	for _, hotspotData := range hotspotsForAddress {
+		statusTime, err := time.Parse(time.RFC3339Nano, hotspotData.Status.Timestamp)
+		if err != nil {
+			log.Println(err)
+			continue
+		}
+
 		// collect hotspot metric requiring extra queries in a new routine
 		wg.Add(5)
-		go e.collectHotspotSyncedMetrics(wg, ch, account, hotspotData)
+		go e.collectHotspotBlocksDeltaMetrics(wg, ch, account, hotspotData)
 		go e.collectHotspotWitnessesMetrics(wg, ch, account, hotspotData)
 		go e.collectHotspotWitnessedMetrics(wg, ch, account, hotspotData)
 		go e.collectHotspotActivityMetrics(wg, ch, account, hotspotData)
@@ -628,6 +642,10 @@ func (e *Exporter) collectHotspotMetrics(wg *sync.WaitGroup, ch chan<- prometheu
 			hotspotBlocks.Desc, hotspotBlocks.Type, float64(hotspotData.Status.Height),
 			account.Address, hotspotData.Address, hotspotData.Name,
 		)
+		ch <- prometheus.MustNewConstMetric(
+			hotspotStatusTimestamp.Desc, hotspotStatusTimestamp.Type, float64(statusTime.Unix()),
+			account.Address, hotspotData.Address, hotspotData.Name,
+		)
 		ch <- prometheus.MustNewConstMetric(
 			hotspotRewardsScale.Desc, hotspotRewardsScale.Type, float64(hotspotData.RewardScale),
 			account.Address, hotspotData.Address, hotspotData.Name,
@@ -643,11 +661,11 @@ func (e *Exporter) collectHotspotMetrics(wg *sync.WaitGroup, ch chan<- prometheu
 	}
 }
 
-// collectSyncedMetrics calculate if the hotspot is in sync with the blockchain
+// collectHotspotBlocksDeltaMetrics calculate the gap between the block height of a hotstop and the height of the chain
-func (e *Exporter) collectHotspotSyncedMetrics(wg *sync.WaitGroup, ch chan<- prometheus.Metric, account *Account, hotspotData heliumapi.Hotspot) {
+func (e *Exporter) collectHotspotBlocksDeltaMetrics(wg *sync.WaitGroup, ch chan<- prometheus.Metric, account *Account, hotspotData heliumapi.Hotspot) {
 	defer wg.Done()
 
-	isSynced := 0.
+	delta := 0.
 	if hotspotData.Status.Online == "online" && hotspotData.Status.Timestamp != "" {
 		statusTime, err := time.Parse(time.RFC3339Nano, hotspotData.Status.Timestamp)
 		if err != nil {
@@ -659,14 +677,11 @@ func (e *Exporter) collectHotspotSyncedMetrics(wg *sync.WaitGroup, ch chan<- pro
 			log.Println(err)
 			return
 		}
-		// if the block height and the height reported by the hotspot is within 10 blocks, we consider it in sync
+		delta = float64(hotspotData.Status.Height - heightAtUpdate)
-		if math.Abs(float64(hotspotData.Status.Height-heightAtUpdate)) <= 10 {
-			isSynced = 1.
-		}
 	}
 
 	ch <- prometheus.MustNewConstMetric(
-		hotspotSynced.Desc, hotspotActivity.Type, isSynced,
+		hotspotBlocksDelta.Desc, hotspotBlocksDelta.Type, delta,
 		account.Address, hotspotData.Address, hotspotData.Name,
 	)
 }
@@ -682,7 +697,7 @@ func (e *Exporter) collectHotspotWitnessesMetrics(wg *sync.WaitGroup, ch chan<-
 	}
 
 	ch <- prometheus.MustNewConstMetric(
-		hotspot5dWitnesses.Desc, hotspotActivity.Type, float64(len(*hotspotWitnesses)),
+		hotspot5dWitnesses.Desc, hotspotActivity.Type, float64(len(hotspotWitnesses)),
 		account.Address, hotspotData.Address, hotspotData.Name,
 	)
 }
@@ -698,7 +713,7 @@ func (e *Exporter) collectHotspotWitnessedMetrics(wg *sync.WaitGroup, ch chan<-
 	}
 
 	ch <- prometheus.MustNewConstMetric(
-		hotspot5dWitnessed.Desc, hotspotActivity.Type, float64(len(*hotspotWitnessed)),
+		hotspot5dWitnessed.Desc, hotspotActivity.Type, float64(len(hotspotWitnessed)),
 		account.Address, hotspotData.Address, hotspotData.Name,
 	)
 }

heliumapi/accounts.go

@@ -32,14 +32,15 @@ func GetAccountForAddress(account string) (*Account, error) {
 // query https://docs.helium.com/api/blockchain/accounts#activity-for-account
 func GetActivityForAccount(account string, filterTypes []string, minTime *time.Time, maxTime *time.Time) (*activity.Activities, error) {
 	path := "/v1/accounts/" + account + "/activity"
-	params := map[string]string{
+	params := map[string]string{}
-		"filter_types": strings.Join(filterTypes, ","),
+	if filterTypes != nil {
+		params["min_time"] = strings.Join(filterTypes, ",")
 	}
 	if minTime != nil {
-		params["min_time"] = minTime.UTC().Format("2006-01-02T15:04:05Z")
+		params["min_time"] = minTime.UTC().Format(timeFormat)
 	}
 	if maxTime != nil {
-		params["max_time"] = maxTime.UTC().Format("2006-01-02T15:04:05Z")
+		params["max_time"] = maxTime.UTC().Format(timeFormat)
 	}
 
 	// query the api
@@ -87,10 +88,10 @@ func GetRewardTotalsForAccount(account string, minTime *time.Time, maxTime *time
 	path := "/v1/accounts/" + account + "/rewards/sum"
 	params := map[string]string{}
 	if minTime != nil {
-		params["min_time"] = minTime.UTC().Format("2006-01-02T15:04:05Z")
+		params["min_time"] = minTime.UTC().Format(timeFormat)
 	}
 	if maxTime != nil {
-		params["max_time"] = maxTime.UTC().Format("2006-01-02T15:04:05Z")
+		params["max_time"] = maxTime.UTC().Format(timeFormat)
 	}
 
 	// query the api
@@ -110,7 +111,7 @@ func GetRewardTotalsForAccount(account string, minTime *time.Time, maxTime *time
 }
 
 // query https://docs.helium.com/api/blockchain/accounts#hotspots-for-account
-func GetHotspotsForAccount(account string) (*[]Hotspot, error) {
+func GetHotspotsForAccount(account string) ([]Hotspot, error) {
 	path := "/v1/accounts/" + account + "/hotspots"
 
 	// query the api
@@ -126,5 +127,5 @@ func GetHotspotsForAccount(account string) (*[]Hotspot, error) {
 		return nil, fmt.Errorf("failed to unmarshal response from %v: %v", path, err)
 	}
 
-	return &respobject.Data, nil
+	return respobject.Data, nil
 }

heliumapi/activity/activities.go

@@ -59,17 +59,17 @@ func NewActivities(resp ActivityResp) (*Activities, error) {
 			}
 			activities.PaymentV2 = append(activities.PaymentV2, paymentV2)
 		case "rewards_v1":
-			rewardV1 := RewardsV1{}
+			rewardsV1 := RewardsV1{}
-			if err := json.Unmarshal(activityRaw, &rewardV1); err != nil {
+			if err := json.Unmarshal(activityRaw, &rewardsV1); err != nil {
 				return nil, fmt.Errorf("failed to unmarshal %v: %v", activityType.Type, err)
 			}
-			activities.RewardsV1 = append(activities.RewardsV1, rewardV1)
+			activities.RewardsV1 = append(activities.RewardsV1, rewardsV1)
 		case "rewards_v2":
-			rewardV2 := RewardsV2{}
+			rewardsV2 := RewardsV2{}
-			if err := json.Unmarshal(activityRaw, &rewardV2); err != nil {
+			if err := json.Unmarshal(activityRaw, &rewardsV2); err != nil {
 				return nil, fmt.Errorf("failed to unmarshal %v: %v", activityType.Type, err)
 			}
-			activities.RewardsV2 = append(activities.RewardsV2, rewardV2)
+			activities.RewardsV2 = append(activities.RewardsV2, rewardsV2)
 		case "stake_validator_v1":
 			stakeValidatorV1 := StakeValidatorV1{}
 			if err := json.Unmarshal(activityRaw, &stakeValidatorV1); err != nil {

heliumapi/block.go

@@ -8,10 +8,11 @@ import (
 
 // query https://docs.helium.com/api/blockchain/blocks/#height
 func GetHeight(maxTime *time.Time) (int, error) {
+	// TODO: the result of this query could be cached
 	path := "/v1/blocks/height"
 	params := map[string]string{}
 	if maxTime != nil {
-		params["max_time"] = maxTime.UTC().Format("2006-01-02T15:04:05Z")
+		params["max_time"] = maxTime.UTC().Format(timeFormat)
 	}
 
 	// query the api

heliumapi/hotspots.go

@@ -31,10 +31,10 @@ func GetRewardsTotalForHotspot(hotspot string, minTime *time.Time, maxTime *time
 	path := "/v1/hotspots/" + hotspot + "/rewards/sum"
 	params := map[string]string{}
 	if minTime != nil {
-		params["min_time"] = minTime.UTC().Format("2006-01-02T15:04:05Z")
+		params["min_time"] = minTime.UTC().Format(timeFormat)
 	}
 	if maxTime != nil {
-		params["max_time"] = maxTime.UTC().Format("2006-01-02T15:04:05Z")
+		params["max_time"] = maxTime.UTC().Format(timeFormat)
 	}
 
 	// query the api
@@ -54,7 +54,7 @@ func GetRewardsTotalForHotspot(hotspot string, minTime *time.Time, maxTime *time
 }
 
 // query https://docs.helium.com/api/blockchain/hotspots#witnesses-for-a-hotspot
-func GetWitnessesForHotspot(hotspot string) (*[]Hotspot, error) {
+func GetWitnessesForHotspot(hotspot string) ([]Hotspot, error) {
 	path := "/v1/hotspots/" + hotspot + "/witnesses"
 
 	// query the api
@@ -70,11 +70,11 @@ func GetWitnessesForHotspot(hotspot string) (*[]Hotspot, error) {
 		return nil, fmt.Errorf("failed to unmarshal response from %v: %v", path, err)
 	}
 
-	return &respobject.Data, nil
+	return respobject.Data, nil
 }
 
 // query https://docs.helium.com/api/blockchain/hotspots#witnessed-for-a-hotspot
-func GetWitnessedForHotspot(hotspot string) (*[]Hotspot, error) {
+func GetWitnessedForHotspot(hotspot string) ([]Hotspot, error) {
 	path := "/v1/hotspots/" + hotspot + "/witnessed"
 
 	// query the api
@@ -90,5 +90,5 @@ func GetWitnessedForHotspot(hotspot string) (*[]Hotspot, error) {
 		return nil, fmt.Errorf("failed to unmarshal response from %v: %v", path, err)
 	}
 
-	return &respobject.Data, nil
+	return respobject.Data, nil
 }

heliumapi/query.go

@@ -4,7 +4,9 @@ import (
 	"encoding/json"
 	"fmt"
 	"io/ioutil"
+	"log"
 	"net/http"
+	"strings"
 )
 
 var (
@@ -14,7 +16,7 @@ var (
 
 // createGetRequest create a GET request to the helium api
 func createGetRequest(path string, params map[string]string) (*http.Request, error) {
-	req, err := http.NewRequest("GET", fmt.Sprintf("%s/%s", ApiUrl, path), nil)
+	req, err := http.NewRequest("GET", fmt.Sprintf("%s%s", ApiUrl, path), nil)
 
 	// setup headers
 	req.Header.Add("Accept", "application/json")
@@ -47,21 +49,22 @@ func getHeliumApi(path string, params *map[string]string) ([]byte, error) {
 	}
 
 	// query the api
+	log.Printf("querying %v", req.URL.String())
 	resp, err := client.Do(req)
 	if err != nil {
-		return nil, fmt.Errorf("failed to query %v: %v", req.URL.RequestURI(), err)
+		return nil, fmt.Errorf("failed to query %v: %v", req.URL.String(), err)
 	}
 	defer resp.Body.Close()
 
 	// validate the response status
 	if resp.StatusCode != 200 {
-		return nil, fmt.Errorf("failed to query %v: http status %v", req.URL.RequestURI(), resp.StatusCode)
+		return nil, fmt.Errorf("failed to query %v: http status %v", req.URL.String(), resp.StatusCode)
 	}
 
 	// read the response body
 	body, err := ioutil.ReadAll(resp.Body)
 	if err != nil {
-		return nil, fmt.Errorf("failed to read response body of %v: %v", req.URL.RequestURI(), err)
+		return nil, fmt.Errorf("failed to read response body of %v: %v", req.URL.String(), err)
 	}
 
 	return body, nil
@@ -91,27 +94,28 @@ func getHeliumApiWithCursor(path string, params *map[string]string) ([][]byte, e
 		// query the api
 		resp, err := client.Do(req)
 		if err != nil {
-			return nil, fmt.Errorf("failed to query %v: %v", req.URL.RequestURI(), err)
+			return nil, fmt.Errorf("failed to query %v: %v", req.URL.String(), err)
 		}
 		defer resp.Body.Close()
 
 		// read the response body and add it to the result array
+		log.Printf("querying %v", req.URL.String())
 		body, err := ioutil.ReadAll(resp.Body)
 		if err != nil {
-			return nil, fmt.Errorf("failed to read response body of %v: %v", req.URL.RequestURI(), err)
+			return nil, fmt.Errorf("failed to read response body of %v: %v", req.URL.String(), err)
 		}
 		res = append(res, body)
 
 		// validate the response status
 		if resp.StatusCode != 200 {
-			return nil, fmt.Errorf("failed to query %v: http status %v", req.URL.RequestURI(), resp.StatusCode)
+			return nil, fmt.Errorf("failed to query %v: http status %v", req.URL.String(), resp.StatusCode)
 		}
 
 		// parse the response body for a cursor
 		respCursor.Cursor = ""
 		err = json.Unmarshal(body, &respCursor)
 		if err != nil {
-			return nil, fmt.Errorf("failed to unmarshal response from %v: %v", req.URL.RequestURI(), err)
+			return nil, fmt.Errorf("failed to unmarshal response from %v: %v", req.URL.String(), err)
 		}
 
 		// continue querying until there is no longer a cursor
@@ -119,7 +123,7 @@ func getHeliumApiWithCursor(path string, params *map[string]string) ([][]byte, e
 			params = &map[string]string{"cursor": respCursor.Cursor}
 			req, err = createGetRequest(path, *params)
 			if err != nil {
-				return nil, fmt.Errorf("failed to create query request %v: %v", req.URL.RequestURI(), err)
+				return nil, fmt.Errorf("failed to create query request %v: %v", req.URL.String(), err)
 			}
 		} else {
 			break

heliumapi/types.go

@@ -1,5 +1,12 @@
 package heliumapi
 
+const (
+	// WORKAROUND: for some reason, helium api partially breaks when given the timezone information
+	// in "max_time" and "min_time", even if the example code suggest that it supports it.
+	// For this reason, we cannot use time.RFC3339 as the time format.
+	timeFormat = "2006-01-02T15:04:05Z"
+)
+
 type AccountResp struct {
 	Data Account `json:"data"`
 }