add hotspot sync and witnesses metrics

contrib/dashboard/records.yaml

@@ -0,0 +1,12 @@
+groups:
+  - name: helium-blockchain-exporter-rules
+    interval: 1m
+    rules:
+      - record: hotspot:helium_hotspot_rewards_hnt:increase15m
+        expr: sum by (account, hotspot, hotspot_name) (increase(helium_hotspot_rewards_hnt_total[15m]))
+      - record: type:helium_hotspot_activity:floor_increase15m
+        expr: sum by (account, hotspot, hotspot_name, type) (floor(increase(helium_hotspot_activity_total[15m])))
+      - record: hotspot:helium_hotspot_rewards_hnt:increase1d
+        expr: sum by (account, hotspot, hotspot_name) (increase(helium_hotspot_rewards_hnt_total[1d]))
+      - record: hotspot:helium_hotspot_rewards_hnt:increase1w
+        expr: sum by (account, hotspot, hotspot_name) (increase(helium_hotspot_rewards_hnt_total[1w]))

exporter.go

@@ -3,6 +3,7 @@ package main
 import (
 	"flag"
 	"log"
+	"math"
 	"net/http"
 	"os"
 	"strconv"
@@ -221,6 +222,14 @@ var (
 		),
 		prometheus.GaugeValue,
 	}
+	hotspotSynced = metricInfo{
+		prometheus.NewDesc(
+			prometheus.BuildFQName(namespace, "hotspot", "synced"),
+			"Whether a hotspot is synced with the blockchain.",
+			commonHotspotLabels, nil,
+		),
+		prometheus.GaugeValue,
+	}
 	hotspotRelayed = metricInfo{
 		prometheus.NewDesc(
 			prometheus.BuildFQName(namespace, "hotspot", "relayed"),
@@ -245,6 +254,22 @@ var (
 		),
 		prometheus.GaugeValue,
 	}
+	hotspot5dWitnesses = metricInfo{
+		prometheus.NewDesc(
+			prometheus.BuildFQName(namespace, "hotspot", "5d_witnesses"),
+			"The number of hotspots that witnessed this hotspot in the last 5 days.",
+			commonHotspotLabels, nil,
+		),
+		prometheus.GaugeValue,
+	}
+	hotspot5dWitnessed = metricInfo{
+		prometheus.NewDesc(
+			prometheus.BuildFQName(namespace, "hotspot", "5d_witnessed"),
+			"The number of hotspots this hotspot witnessed in the last 5 days.",
+			commonHotspotLabels, nil,
+		),
+		prometheus.GaugeValue,
+	}
 	hotspotGeocodeInfo = metricInfo{
 		prometheus.NewDesc(
 			prometheus.BuildFQName(namespace, "hotspot", "geocode_info"),
@@ -309,9 +334,12 @@ func (e *Exporter) Describe(ch chan<- *prometheus.Desc) {
 	ch <- accountWithdrawalsHnt.Desc
 
 	ch <- hotspotUp.Desc
+	ch <- hotspotSynced.Desc
 	ch <- hotspotRelayed.Desc
 	ch <- hotspotBlocks.Desc
 	ch <- hotspotRewardsScale.Desc
+	ch <- hotspot5dWitnesses.Desc
+	ch <- hotspot5dWitnessed.Desc
 	ch <- hotspotGeocodeInfo.Desc
 	ch <- hotspotAntennaInfo.Desc
 	ch <- hotspotActivity.Desc
@@ -449,6 +477,7 @@ func (e *Exporter) collectAccountRewardsTotalMetrics(wg *sync.WaitGroup, ch chan
 func (e *Exporter) collectAccountTransactionsMetrics(wg *sync.WaitGroup, ch chan<- prometheus.Metric, account *Account) {
 	defer wg.Done()
 
+	// We optimize our queries by asking the api only the activities we car about.
 	activityTypes := []string{
 		"add_gateway_v1",
 		"assert_location_v1",
@@ -463,13 +492,25 @@ func (e *Exporter) collectAccountTransactionsMetrics(wg *sync.WaitGroup, ch chan
 		"unstake_validator_v1",
 	}
 
-	// we can only want to allow a single instance of the routine doing
-	// calculations on the deposited and widthdrawn total
+	// We can only want to allow a single instance of the routine doing
+	// calculations on the deposited and widthdrawn total.
 	account.Tx.UpdateLock.Lock()
 	defer account.Tx.UpdateLock.Unlock()
 
-	now := time.Now()
-	activities, err := heliumapi.GetActivityForAccount(account.Address, activityTypes, &account.Tx.LastUpdate, &now)
+	// We want to keep in memory the timestamp of the last activity we
+	// received from the api. We cannot do something naive like [lastscrape, now]
+	// because the api can take a few seconds to sync with the chain and
+	// we can miss some activities by doing it that way.
+	lastActivityTime := account.Tx.LastUpdate.Unix()
+	updateLastActivityTime := func(newTime int64) {
+		if lastActivityTime < newTime {
+			lastActivityTime = newTime
+		}
+	}
+
+	// add 1 second to the last update time to avoid querying the same activity twice
+	minTime := account.Tx.LastUpdate.Add(time.Second * 1)
+	activities, err := heliumapi.GetActivityForAccount(account.Address, activityTypes, &minTime, nil)
 	if err != nil {
 		log.Println(err)
 		return
@@ -478,12 +519,15 @@ func (e *Exporter) collectAccountTransactionsMetrics(wg *sync.WaitGroup, ch chan
 	// impl based on https://github.com/helium/hotspot-app/blob/918563fba84d1abf4554a43a4d42bb838d017bd3/src/features/wallet/root/useActivityItem.tsx#L336
 	for _, activity := range activities.AddGatewayV1 {
 		account.Tx.WithdrawalTotal += activity.StakingFee
+		updateLastActivityTime(activity.Time)
 	}
 	for _, activity := range activities.AssertLocationV1 {
 		account.Tx.WithdrawalTotal += activity.StakingFee
+		updateLastActivityTime(activity.Time)
 	}
 	for _, activity := range activities.AssertLocationV2 {
 		account.Tx.WithdrawalTotal += activity.StakingFee
+		updateLastActivityTime(activity.Time)
 	}
 	for _, activity := range activities.PaymentV1 {
 		if activity.Payer == account.Address {
@@ -491,6 +535,7 @@ func (e *Exporter) collectAccountTransactionsMetrics(wg *sync.WaitGroup, ch chan
 		} else {
 			account.Tx.DepositTotal += activity.Amount
 		}
+		updateLastActivityTime(activity.Time)
 	}
 	for _, activity := range activities.PaymentV2 {
 		if activity.Payer == account.Address {
@@ -506,22 +551,27 @@ func (e *Exporter) collectAccountTransactionsMetrics(wg *sync.WaitGroup, ch chan
 				}
 			}
 		}
+		updateLastActivityTime(activity.Time)
 	}
 	for _, activity := range activities.RewardsV1 {
 		for _, reward := range activity.Rewards {
 			account.Tx.DepositTotal += reward.Amount
 		}
+		updateLastActivityTime(activity.Time)
 	}
 	for _, activity := range activities.RewardsV2 {
 		for _, reward := range activity.Rewards {
 			account.Tx.DepositTotal += reward.Amount
 		}
+		updateLastActivityTime(activity.Time)
 	}
 	for _, activity := range activities.StakeValidatorV1 {
 		account.Tx.WithdrawalTotal += activity.Stake
+		updateLastActivityTime(activity.Time)
 	}
 	for _, activity := range activities.TokenBurnV1 {
 		account.Tx.WithdrawalTotal += activity.Amount
+		updateLastActivityTime(activity.Time)
 	}
 	for _, activity := range activities.TransferHotspotV1 {
 		if activity.Buyer == account.Address {
@@ -529,11 +579,13 @@ func (e *Exporter) collectAccountTransactionsMetrics(wg *sync.WaitGroup, ch chan
 		} else {
 			account.Tx.DepositTotal += activity.AmountToSeller
 		}
+		updateLastActivityTime(activity.Time)
 	}
 	for _, activity := range activities.UnstakeValidatorV1 {
 		account.Tx.WithdrawalTotal += activity.StakeAmount
+		updateLastActivityTime(activity.Time)
 	}
-	account.Tx.LastUpdate = now
+	account.Tx.LastUpdate = time.Unix(lastActivityTime, 0)
 
 	ch <- prometheus.MustNewConstMetric(
 		accountDepositsHnt.Desc, accountDepositsHnt.Type, float64(account.Tx.DepositTotal)/blockchain_hnt_factor,
@@ -557,7 +609,10 @@ func (e *Exporter) collectHotspotMetrics(wg *sync.WaitGroup, ch chan<- prometheu
 
 	for _, hotspotData := range *hotspotsForAddress {
 		// collect hotspot metric requiring extra queries in a new routine
-		wg.Add(2)
+		wg.Add(5)
+		go e.collectHotspotSyncedMetrics(wg, ch, account, hotspotData)
+		go e.collectHotspotWitnessesMetrics(wg, ch, account, hotspotData)
+		go e.collectHotspotWitnessedMetrics(wg, ch, account, hotspotData)
 		go e.collectHotspotActivityMetrics(wg, ch, account, hotspotData)
 		go e.collectHotspotRewardsMetrics(wg, ch, account, hotspotData)
 
@@ -588,8 +643,68 @@ func (e *Exporter) collectHotspotMetrics(wg *sync.WaitGroup, ch chan<- prometheu
 	}
 }
 
+// collectSyncedMetrics calculate if the hotspot is in sync with the blockchain
+func (e *Exporter) collectHotspotSyncedMetrics(wg *sync.WaitGroup, ch chan<- prometheus.Metric, account *Account, hotspotData heliumapi.Hotspot) {
+	defer wg.Done()
+
+	isSynced := 0.
+	if hotspotData.Status.Online == "online" && hotspotData.Status.Timestamp != "" {
+		statusTime, err := time.Parse(time.RFC3339Nano, hotspotData.Status.Timestamp)
+		if err != nil {
+			log.Println(err)
+			return
+		}
+		heightAtUpdate, err := heliumapi.GetHeight(&statusTime)
+		if err != nil {
+			log.Println(err)
+			return
+		}
+		// if the block height and the height reported by the hotspot is within 10 blocks, we consider it in sync
+		if math.Abs(float64(hotspotData.Status.Height-heightAtUpdate)) <= 10 {
+			isSynced = 1.
+		}
+	}
+
+	ch <- prometheus.MustNewConstMetric(
+		hotspotSynced.Desc, hotspotActivity.Type, isSynced,
+		account.Address, hotspotData.Address, hotspotData.Name,
+	)
+}
+
+// collectHotspotWitnessesMetrics collect the total number witnesses of a hotspot in the last 5d
+func (e *Exporter) collectHotspotWitnessesMetrics(wg *sync.WaitGroup, ch chan<- prometheus.Metric, account *Account, hotspotData heliumapi.Hotspot) {
+	defer wg.Done()
+
+	hotspotWitnesses, err := heliumapi.GetWitnessesForHotspot(hotspotData.Address)
+	if err != nil {
+		log.Println(err)
+		return
+	}
+
+	ch <- prometheus.MustNewConstMetric(
+		hotspot5dWitnesses.Desc, hotspotActivity.Type, float64(len(*hotspotWitnesses)),
+		account.Address, hotspotData.Address, hotspotData.Name,
+	)
+}
+
+// collectHotspotWitnessedMetrics collect the total number hotspots witnessed in the last 5d
+func (e *Exporter) collectHotspotWitnessedMetrics(wg *sync.WaitGroup, ch chan<- prometheus.Metric, account *Account, hotspotData heliumapi.Hotspot) {
+	defer wg.Done()
+
+	hotspotWitnessed, err := heliumapi.GetWitnessedForHotspot(hotspotData.Address)
+	if err != nil {
+		log.Println(err)
+		return
+	}
+
+	ch <- prometheus.MustNewConstMetric(
+		hotspot5dWitnessed.Desc, hotspotActivity.Type, float64(len(*hotspotWitnessed)),
+		account.Address, hotspotData.Address, hotspotData.Name,
+	)
+}
+
 // collectHotspotActivityMetrics collect the total number of activities executed by a hotspot from the helium api
-func (e *Exporter) collectHotspotActivityMetrics(wg *sync.WaitGroup, ch chan<- prometheus.Metric, account *Account, hotspotData heliumapi.AccountHotspot) {
+func (e *Exporter) collectHotspotActivityMetrics(wg *sync.WaitGroup, ch chan<- prometheus.Metric, account *Account, hotspotData heliumapi.Hotspot) {
 	defer wg.Done()
 
 	hotspotActivityForAddress, err := heliumapi.GetHotspotActivityCounts(hotspotData.Address)
@@ -607,7 +722,7 @@ func (e *Exporter) collectHotspotActivityMetrics(wg *sync.WaitGroup, ch chan<- p
 }
 
 // collectHotspotRewardsMetrics collect the total rewards accumulated by a hotspot from the helium api
-func (e *Exporter) collectHotspotRewardsMetrics(wg *sync.WaitGroup, ch chan<- prometheus.Metric, account *Account, hotspotData heliumapi.AccountHotspot) {
+func (e *Exporter) collectHotspotRewardsMetrics(wg *sync.WaitGroup, ch chan<- prometheus.Metric, account *Account, hotspotData heliumapi.Hotspot) {
 	defer wg.Done()
 
 	hotspotRewardTotalsForAddress, err := heliumapi.GetRewardsTotalForHotspot(hotspotData.Address, &e.StartTime, nil)

heliumapi/accounts.go

@@ -110,7 +110,7 @@ func GetRewardTotalsForAccount(account string, minTime *time.Time, maxTime *time
 }
 
 // query https://docs.helium.com/api/blockchain/accounts#hotspots-for-account
-func GetHotspotsForAccount(account string) (*[]AccountHotspot, error) {
+func GetHotspotsForAccount(account string) (*[]Hotspot, error) {
 	path := "/v1/accounts/" + account + "/hotspots"
 
 	// query the api

heliumapi/activity/types.go

@@ -14,8 +14,16 @@ type Activities struct {
 	UnstakeValidatorV1 []UnstakeValidatorV1
 }
 
-type AddGatewayV1 struct {
+type BaseActivity struct {
 	Hash       string `json:"hash"`
+	Time       int64  `json:"time"`
+	StartEpoch int    `json:"start_epoch"`
+	EndEpoch   int    `json:"end_epoch"`
+	Height     int    `json:"height"`
+}
+
+type AddGatewayV1 struct {
+	BaseActivity
 	Fee        int    `json:"fee"`
 	Owner      string `json:"owner"`
 	Payer      string `json:"payer"`
@@ -24,7 +32,7 @@ type AddGatewayV1 struct {
 }
 
 type AssertLocationV1 struct {
-	Hash       string `json:"hash"`
+	BaseActivity
 	Fee        int    `json:"fee"`
 	Nonce      int    `json:"nonce"`
 	Owner      string `json:"owner"`
@@ -35,7 +43,7 @@ type AssertLocationV1 struct {
 }
 
 type AssertLocationV2 struct {
-	Hash       string `json:"hash"`
+	BaseActivity
 	Fee        int    `json:"fee"`
 	Gain       int    `json:"gain"`
 	Nonce      int    `json:"nonce"`
@@ -48,7 +56,7 @@ type AssertLocationV2 struct {
 }
 
 type PaymentV1 struct {
-	Hash   string `json:"hash"`
+	BaseActivity
 	Amount int    `json:"amount"`
 	Fee    int    `json:"fee"`
 	Nonce  int    `json:"nonce"`
@@ -57,7 +65,7 @@ type PaymentV1 struct {
 }
 
 type PaymentV2 struct {
-	Hash     string `json:"hash"`
+	BaseActivity
 	Fee      int    `json:"fee"`
 	Nonce    int    `json:"nonce"`
 	Payer    string `json:"payer"`
@@ -76,31 +84,31 @@ type reward struct {
 }
 
 type RewardsV1 struct {
-	Hash       string   `json:"hash"`
+	BaseActivity
 	StartEpoch int      `json:"start_epoch"`
 	EndEpoch   int      `json:"end_epoch"`
 	Rewards    []reward `json:"rewards"`
 }
 
 type RewardsV2 struct {
-	Hash       string   `json:"hash"`
+	BaseActivity
 	StartEpoch int      `json:"start_epoch"`
 	EndEpoch   int      `json:"end_epoch"`
 	Rewards    []reward `json:"rewards"`
 }
 
 type StakeValidatorV1 struct {
+	BaseActivity
 	Address        string `json:"address"`
 	Fee            int    `json:"fee"`
-	Hash           string `json:"hash"`
 	Owner          string `json:"owner"`
 	Stake          int    `json:"stake"`
 	OwnerSignature string `json:"owner_signature"`
 }
 
 type TokenBurnV1 struct {
+	BaseActivity
 	Fee    int    `json:"fee"`
-	Hash   string `json:"hash"`
 	Memo   string `json:"memo"`
 	Nonce  int    `json:"nonce"`
 	Payer  string `json:"payer"`
@@ -109,7 +117,7 @@ type TokenBurnV1 struct {
 }
 
 type TransferHotspotV1 struct {
-	Hash           string `json:"hash"`
+	BaseActivity
 	Fee            int    `json:"fee"`
 	Buyer          string `json:"buyer"`
 	Seller         string `json:"seller"`
@@ -119,11 +127,11 @@ type TransferHotspotV1 struct {
 }
 
 type UnstakeValidatorV1 struct {
+	BaseActivity
 	Address            string `json:"address"`
 	Owner              string `json:"owner"`
 	OwnerSignature     string `json:"owner_signature"`
 	Fee                int    `json:"fee"`
 	StakeAmount        int    `json:"stake_amount"`
 	StakeReleaseHeight int    `json:"stake_release_height"`
-	Hash               string `json:"hash"`
 }

heliumapi/block.go

@@ -0,0 +1,31 @@
+package heliumapi
+
+import (
+	"encoding/json"
+	"fmt"
+	"time"
+)
+
+// query https://docs.helium.com/api/blockchain/blocks/#height
+func GetHeight(maxTime *time.Time) (int, error) {
+	path := "/v1/blocks/height"
+	params := map[string]string{}
+	if maxTime != nil {
+		params["max_time"] = maxTime.UTC().Format("2006-01-02T15:04:05Z")
+	}
+
+	// query the api
+	respBody, err := getHeliumApi(path, &params)
+	if err != nil {
+		return -1, err
+	}
+
+	// unmarshal the response
+	respobject := HeightResp{}
+	err = json.Unmarshal(respBody, &respobject)
+	if err != nil {
+		return -1, fmt.Errorf("failed to unmarshal response from %v: %v", path, err)
+	}
+
+	return respobject.Data.Height, nil
+}

heliumapi/hotspots.go

@@ -52,3 +52,43 @@ func GetRewardsTotalForHotspot(hotspot string, minTime *time.Time, maxTime *time
 
 	return &respobject.Data, nil
 }
+
+// query https://docs.helium.com/api/blockchain/hotspots#witnesses-for-a-hotspot
+func GetWitnessesForHotspot(hotspot string) (*[]Hotspot, error) {
+	path := "/v1/hotspots/" + hotspot + "/witnesses"
+
+	// query the api
+	respBody, err := getHeliumApi(path, nil)
+	if err != nil {
+		return nil, err
+	}
+
+	// unmarshal the response
+	respobject := WitnessesResp{}
+	err = json.Unmarshal(respBody, &respobject)
+	if err != nil {
+		return nil, fmt.Errorf("failed to unmarshal response from %v: %v", path, err)
+	}
+
+	return &respobject.Data, nil
+}
+
+// query https://docs.helium.com/api/blockchain/hotspots#witnessed-for-a-hotspot
+func GetWitnessedForHotspot(hotspot string) (*[]Hotspot, error) {
+	path := "/v1/hotspots/" + hotspot + "/witnessed"
+
+	// query the api
+	respBody, err := getHeliumApi(path, nil)
+	if err != nil {
+		return nil, err
+	}
+
+	// unmarshal the response
+	respobject := WitnessedResp{}
+	err = json.Unmarshal(respBody, &respobject)
+	if err != nil {
+		return nil, fmt.Errorf("failed to unmarshal response from %v: %v", path, err)
+	}
+
+	return &respobject.Data, nil
+}

heliumapi/types.go

@@ -4,6 +4,46 @@ type AccountResp struct {
 	Data Account `json:"data"`
 }
 
+type AccountHotspotsResp struct {
+	Data []Hotspot `json:"data"`
+}
+
+type ActivityCounts map[string]int
+type ActivityCountsResp struct {
+	Data ActivityCounts
+}
+
+type RewardTotalResp struct {
+	Meta struct {
+		MinTime string `json:"min_time"`
+		MaxTime string `json:"max_time"`
+	} `json:"meta"`
+
+	Data RewardTotal `json:"data"`
+}
+
+type CurrentOraclePriceResp struct {
+	Data CurrentOraclePrice `json:"data"`
+}
+
+type BlockchainStatsResp struct {
+	Data BlockchainStats `json:"data"`
+}
+
+type WitnessesResp struct {
+	Data []Hotspot `json:"data"`
+}
+
+type HeightResp struct {
+	Data struct {
+		Height int `json:"height"`
+	} `json:"data"`
+}
+
+type WitnessedResp struct {
+	Data []Hotspot `json:"data"`
+}
+
 type Account struct {
 	Address          string `json:"address"`
 	Balance          int    `json:"balance"`
@@ -15,11 +55,7 @@ type Account struct {
 	SpeculativeNonce int    `json:"speculative_nonce"`
 }
 
-type AccountHotspotsResp struct {
-	Data []AccountHotspot `json:"data"`
-}
-
-type AccountHotspot struct {
+type Hotspot struct {
 	Lng            float64 `json:"lng"`
 	Lat            float64 `json:"lat"`
 	TimestampAdded string  `json:"timestamp_added"`
@@ -57,21 +93,6 @@ type AccountHotspot struct {
 	Address    string `json:"address"`
 }
 
-type ActivityCountsResp struct {
-	Data ActivityCounts
-}
-
-type ActivityCounts map[string]int
-
-type RewardTotalResp struct {
-	Meta struct {
-		MinTime string `json:"min_time"`
-		MaxTime string `json:"max_time"`
-	} `json:"meta"`
-
-	Data RewardTotal `json:"data"`
-}
-
 type RewardTotal struct {
 	Total  float64 `json:"total"`
 	Sum    float64 `json:"sum"`
@@ -82,19 +103,11 @@ type RewardTotal struct {
 	Avg    float64 `json:"avg"`
 }
 
-type CurrentOraclePriceResp struct {
-	Data CurrentOraclePrice `json:"data"`
-}
-
 type CurrentOraclePrice struct {
 	Price int `json:"price"`
 	Block int `json:"block"`
 }
 
-type BlockchainStatsResp struct {
-	Data BlockchainStats `json:"data"`
-}
-
 type BlockchainStats struct {
 	BlockTime struct {
 		LastDay struct {