7 changed files with 48 additions and 267 deletions
--- a/contrib/dashboard/records.yaml
+++ b/contrib/dashboard/records.yaml
@ -1,12 +0,0 @@
-groups:
-  - name: helium-blockchain-exporter-rules
-    interval: 1m
-    rules:
-      - record: hotspot:helium_hotspot_rewards_hnt:increase15m
-        expr: sum by (account, hotspot, hotspot_name) (increase(helium_hotspot_rewards_hnt_total[15m]))
-      - record: type:helium_hotspot_activity:floor_increase15m
-        expr: sum by (account, hotspot, hotspot_name, type) (floor(increase(helium_hotspot_activity_total[15m])))
-      - record: hotspot:helium_hotspot_rewards_hnt:increase1d
-        expr: sum by (account, hotspot, hotspot_name) (increase(helium_hotspot_rewards_hnt_total[1d]))
-      - record: hotspot:helium_hotspot_rewards_hnt:increase1w
-        expr: sum by (account, hotspot, hotspot_name) (increase(helium_hotspot_rewards_hnt_total[1w]))
--- a/exporter.go
+++ b/exporter.go
@ -3,7 +3,6 @@ package main
 import (
 	"flag"
 	"log"
-	"math"
 	"net/http"
 	"os"
 	"strconv"
@ -222,14 +221,6 @@ var (
 		),
 		prometheus.GaugeValue,
 	}
-	hotspotSynced = metricInfo{
-		prometheus.NewDesc(
-			prometheus.BuildFQName(namespace, "hotspot", "synced"),
-			"Whether a hotspot is synced with the blockchain.",
-			commonHotspotLabels, nil,
-		),
-		prometheus.GaugeValue,
-	}
 	hotspotRelayed = metricInfo{
 		prometheus.NewDesc(
 			prometheus.BuildFQName(namespace, "hotspot", "relayed"),
@ -254,22 +245,6 @@ var (
 		),
 		prometheus.GaugeValue,
 	}
-	hotspot5dWitnesses = metricInfo{
-		prometheus.NewDesc(
-			prometheus.BuildFQName(namespace, "hotspot", "5d_witnesses"),
-			"The number of hotspots that witnessed this hotspot in the last 5 days.",
-			commonHotspotLabels, nil,
-		),
-		prometheus.GaugeValue,
-	}
-	hotspot5dWitnessed = metricInfo{
-		prometheus.NewDesc(
-			prometheus.BuildFQName(namespace, "hotspot", "5d_witnessed"),
-			"The number of hotspots this hotspot witnessed in the last 5 days.",
-			commonHotspotLabels, nil,
-		),
-		prometheus.GaugeValue,
-	}
 	hotspotGeocodeInfo = metricInfo{
 		prometheus.NewDesc(
 			prometheus.BuildFQName(namespace, "hotspot", "geocode_info"),
@ -334,12 +309,9 @@ func (e *Exporter) Describe(ch chan<- *prometheus.Desc) {
 	ch <- accountWithdrawalsHnt.Desc

 	ch <- hotspotUp.Desc
-	ch <- hotspotSynced.Desc
 	ch <- hotspotRelayed.Desc
 	ch <- hotspotBlocks.Desc
 	ch <- hotspotRewardsScale.Desc
-	ch <- hotspot5dWitnesses.Desc
-	ch <- hotspot5dWitnessed.Desc
 	ch <- hotspotGeocodeInfo.Desc
 	ch <- hotspotAntennaInfo.Desc
 	ch <- hotspotActivity.Desc
@ -477,7 +449,6 @@ func (e *Exporter) collectAccountRewardsTotalMetrics(wg *sync.WaitGroup, ch chan
 func (e *Exporter) collectAccountTransactionsMetrics(wg *sync.WaitGroup, ch chan<- prometheus.Metric, account *Account) {
 	defer wg.Done()

-	// We optimize our queries by asking the api only the activities we car about.
 	activityTypes := []string{
 		"add_gateway_v1",
 		"assert_location_v1",
@ -492,25 +463,13 @@ func (e *Exporter) collectAccountTransactionsMetrics(wg *sync.WaitGroup, ch chan
 		"unstake_validator_v1",
 	}

-	// We can only want to allow a single instance of the routine doing
-	// calculations on the deposited and widthdrawn total.
+	// we can only want to allow a single instance of the routine doing
+	// calculations on the deposited and widthdrawn total
 	account.Tx.UpdateLock.Lock()
 	defer account.Tx.UpdateLock.Unlock()

-	// We want to keep in memory the timestamp of the last activity we
-	// received from the api. We cannot do something naive like [lastscrape, now]
-	// because the api can take a few seconds to sync with the chain and
-	// we can miss some activities by doing it that way.
-	lastActivityTime := account.Tx.LastUpdate.Unix()
-	updateLastActivityTime := func(newTime int64) {
-		if lastActivityTime < newTime {
-			lastActivityTime = newTime
-		}
-	}
-
-	// add 1 second to the last update time to avoid querying the same activity twice
-	minTime := account.Tx.LastUpdate.Add(time.Second * 1)
-	activities, err := heliumapi.GetActivityForAccount(account.Address, activityTypes, &minTime, nil)
+	now := time.Now()
+	activities, err := heliumapi.GetActivityForAccount(account.Address, activityTypes, &account.Tx.LastUpdate, &now)
 	if err != nil {
 		log.Println(err)
 		return
@ -519,15 +478,12 @@ func (e *Exporter) collectAccountTransactionsMetrics(wg *sync.WaitGroup, ch chan
 	// impl based on https://github.com/helium/hotspot-app/blob/918563fba84d1abf4554a43a4d42bb838d017bd3/src/features/wallet/root/useActivityItem.tsx#L336
 	for _, activity := range activities.AddGatewayV1 {
 		account.Tx.WithdrawalTotal += activity.StakingFee
-		updateLastActivityTime(activity.Time)
 	}
 	for _, activity := range activities.AssertLocationV1 {
 		account.Tx.WithdrawalTotal += activity.StakingFee
-		updateLastActivityTime(activity.Time)
 	}
 	for _, activity := range activities.AssertLocationV2 {
 		account.Tx.WithdrawalTotal += activity.StakingFee
-		updateLastActivityTime(activity.Time)
 	}
 	for _, activity := range activities.PaymentV1 {
 		if activity.Payer == account.Address {
@ -535,7 +491,6 @@ func (e *Exporter) collectAccountTransactionsMetrics(wg *sync.WaitGroup, ch chan
 		} else {
 			account.Tx.DepositTotal += activity.Amount
 		}
-		updateLastActivityTime(activity.Time)
 	}
 	for _, activity := range activities.PaymentV2 {
 		if activity.Payer == account.Address {
@ -551,27 +506,22 @@ func (e *Exporter) collectAccountTransactionsMetrics(wg *sync.WaitGroup, ch chan
 				}
 			}
 		}
-		updateLastActivityTime(activity.Time)
 	}
 	for _, activity := range activities.RewardsV1 {
 		for _, reward := range activity.Rewards {
 			account.Tx.DepositTotal += reward.Amount
 		}
-		updateLastActivityTime(activity.Time)
 	}
 	for _, activity := range activities.RewardsV2 {
 		for _, reward := range activity.Rewards {
 			account.Tx.DepositTotal += reward.Amount
 		}
-		updateLastActivityTime(activity.Time)
 	}
 	for _, activity := range activities.StakeValidatorV1 {
 		account.Tx.WithdrawalTotal += activity.Stake
-		updateLastActivityTime(activity.Time)
 	}
 	for _, activity := range activities.TokenBurnV1 {
 		account.Tx.WithdrawalTotal += activity.Amount
-		updateLastActivityTime(activity.Time)
 	}
 	for _, activity := range activities.TransferHotspotV1 {
 		if activity.Buyer == account.Address {
@ -579,13 +529,11 @@ func (e *Exporter) collectAccountTransactionsMetrics(wg *sync.WaitGroup, ch chan
 		} else {
 			account.Tx.DepositTotal += activity.AmountToSeller
 		}
-		updateLastActivityTime(activity.Time)
 	}
 	for _, activity := range activities.UnstakeValidatorV1 {
 		account.Tx.WithdrawalTotal += activity.StakeAmount
-		updateLastActivityTime(activity.Time)
 	}
-	account.Tx.LastUpdate = time.Unix(lastActivityTime, 0)
+	account.Tx.LastUpdate = now

 	ch <- prometheus.MustNewConstMetric(
 		accountDepositsHnt.Desc, accountDepositsHnt.Type, float64(account.Tx.DepositTotal)/blockchain_hnt_factor,
@ -609,10 +557,7 @@ func (e *Exporter) collectHotspotMetrics(wg *sync.WaitGroup, ch chan<- prometheu

 	for _, hotspotData := range *hotspotsForAddress {
 		// collect hotspot metric requiring extra queries in a new routine
-		wg.Add(5)
-		go e.collectHotspotSyncedMetrics(wg, ch, account, hotspotData)
-		go e.collectHotspotWitnessesMetrics(wg, ch, account, hotspotData)
-		go e.collectHotspotWitnessedMetrics(wg, ch, account, hotspotData)
+		wg.Add(2)
 		go e.collectHotspotActivityMetrics(wg, ch, account, hotspotData)
 		go e.collectHotspotRewardsMetrics(wg, ch, account, hotspotData)

@ -643,68 +588,8 @@ func (e *Exporter) collectHotspotMetrics(wg *sync.WaitGroup, ch chan<- prometheu
 	}
 }

-// collectSyncedMetrics calculate if the hotspot is in sync with the blockchain
-func (e *Exporter) collectHotspotSyncedMetrics(wg *sync.WaitGroup, ch chan<- prometheus.Metric, account *Account, hotspotData heliumapi.Hotspot) {
-	defer wg.Done()
-
-	isSynced := 0.
-	if hotspotData.Status.Online == "online" && hotspotData.Status.Timestamp != "" {
-		statusTime, err := time.Parse(time.RFC3339Nano, hotspotData.Status.Timestamp)
-		if err != nil {
-			log.Println(err)
-			return
-		}
-		heightAtUpdate, err := heliumapi.GetHeight(&statusTime)
-		if err != nil {
-			log.Println(err)
-			return
-		}
-		// if the block height and the height reported by the hotspot is within 10 blocks, we consider it in sync
-		if math.Abs(float64(hotspotData.Status.Height-heightAtUpdate)) <= 10 {
-			isSynced = 1.
-		}
-	}
-
-	ch <- prometheus.MustNewConstMetric(
-		hotspotSynced.Desc, hotspotActivity.Type, isSynced,
-		account.Address, hotspotData.Address, hotspotData.Name,
-	)
-}
-
-// collectHotspotWitnessesMetrics collect the total number witnesses of a hotspot in the last 5d
-func (e *Exporter) collectHotspotWitnessesMetrics(wg *sync.WaitGroup, ch chan<- prometheus.Metric, account *Account, hotspotData heliumapi.Hotspot) {
-	defer wg.Done()
-
-	hotspotWitnesses, err := heliumapi.GetWitnessesForHotspot(hotspotData.Address)
-	if err != nil {
-		log.Println(err)
-		return
-	}
-
-	ch <- prometheus.MustNewConstMetric(
-		hotspot5dWitnesses.Desc, hotspotActivity.Type, float64(len(*hotspotWitnesses)),
-		account.Address, hotspotData.Address, hotspotData.Name,
-	)
-}
-
-// collectHotspotWitnessedMetrics collect the total number hotspots witnessed in the last 5d
-func (e *Exporter) collectHotspotWitnessedMetrics(wg *sync.WaitGroup, ch chan<- prometheus.Metric, account *Account, hotspotData heliumapi.Hotspot) {
-	defer wg.Done()
-
-	hotspotWitnessed, err := heliumapi.GetWitnessedForHotspot(hotspotData.Address)
-	if err != nil {
-		log.Println(err)
-		return
-	}
-
-	ch <- prometheus.MustNewConstMetric(
-		hotspot5dWitnessed.Desc, hotspotActivity.Type, float64(len(*hotspotWitnessed)),
-		account.Address, hotspotData.Address, hotspotData.Name,
-	)
-}
-
 // collectHotspotActivityMetrics collect the total number of activities executed by a hotspot from the helium api
-func (e *Exporter) collectHotspotActivityMetrics(wg *sync.WaitGroup, ch chan<- prometheus.Metric, account *Account, hotspotData heliumapi.Hotspot) {
+func (e *Exporter) collectHotspotActivityMetrics(wg *sync.WaitGroup, ch chan<- prometheus.Metric, account *Account, hotspotData heliumapi.AccountHotspot) {
 	defer wg.Done()

 	hotspotActivityForAddress, err := heliumapi.GetHotspotActivityCounts(hotspotData.Address)
@ -722,7 +607,7 @@ func (e *Exporter) collectHotspotActivityMetrics(wg *sync.WaitGroup, ch chan<- p
 }

 // collectHotspotRewardsMetrics collect the total rewards accumulated by a hotspot from the helium api
-func (e *Exporter) collectHotspotRewardsMetrics(wg *sync.WaitGroup, ch chan<- prometheus.Metric, account *Account, hotspotData heliumapi.Hotspot) {
+func (e *Exporter) collectHotspotRewardsMetrics(wg *sync.WaitGroup, ch chan<- prometheus.Metric, account *Account, hotspotData heliumapi.AccountHotspot) {
 	defer wg.Done()

 	hotspotRewardTotalsForAddress, err := heliumapi.GetRewardsTotalForHotspot(hotspotData.Address, &e.StartTime, nil)
--- a/heliumapi/accounts.go
+++ b/heliumapi/accounts.go
@ -110,7 +110,7 @@ func GetRewardTotalsForAccount(account string, minTime *time.Time, maxTime *time
 }

 // query https://docs.helium.com/api/blockchain/accounts#hotspots-for-account
-func GetHotspotsForAccount(account string) (*[]Hotspot, error) {
+func GetHotspotsForAccount(account string) (*[]AccountHotspot, error) {
 	path := "/v1/accounts/" + account + "/hotspots"

 	// query the api
--- a/heliumapi/activity/types.go
+++ b/heliumapi/activity/types.go
@ -14,16 +14,8 @@ type Activities struct {
 	UnstakeValidatorV1 []UnstakeValidatorV1
 }

-type BaseActivity struct {
-	Hash       string `json:"hash"`
-	Time       int64  `json:"time"`
-	StartEpoch int    `json:"start_epoch"`
-	EndEpoch   int    `json:"end_epoch"`
-	Height     int    `json:"height"`
-}
-
 type AddGatewayV1 struct {
-	BaseActivity
+	Hash       string `json:"hash"`
 	Fee        int    `json:"fee"`
 	Owner      string `json:"owner"`
 	Payer      string `json:"payer"`
@ -32,7 +24,7 @@ type AddGatewayV1 struct {
 }

 type AssertLocationV1 struct {
-	BaseActivity
+	Hash       string `json:"hash"`
 	Fee        int    `json:"fee"`
 	Nonce      int    `json:"nonce"`
 	Owner      string `json:"owner"`
@ -43,7 +35,7 @@ type AssertLocationV1 struct {
 }

 type AssertLocationV2 struct {
-	BaseActivity
+	Hash       string `json:"hash"`
 	Fee        int    `json:"fee"`
 	Gain       int    `json:"gain"`
 	Nonce      int    `json:"nonce"`
@ -56,7 +48,7 @@ type AssertLocationV2 struct {
 }

 type PaymentV1 struct {
-	BaseActivity
+	Hash   string `json:"hash"`
 	Amount int    `json:"amount"`
 	Fee    int    `json:"fee"`
 	Nonce  int    `json:"nonce"`
@ -65,7 +57,7 @@ type PaymentV1 struct {
 }

 type PaymentV2 struct {
-	BaseActivity
+	Hash     string `json:"hash"`
 	Fee      int    `json:"fee"`
 	Nonce    int    `json:"nonce"`
 	Payer    string `json:"payer"`
@ -84,31 +76,31 @@ type reward struct {
 }

 type RewardsV1 struct {
-	BaseActivity
+	Hash       string   `json:"hash"`
 	StartEpoch int      `json:"start_epoch"`
 	EndEpoch   int      `json:"end_epoch"`
 	Rewards    []reward `json:"rewards"`
 }

 type RewardsV2 struct {
-	BaseActivity
+	Hash       string   `json:"hash"`
 	StartEpoch int      `json:"start_epoch"`
 	EndEpoch   int      `json:"end_epoch"`
 	Rewards    []reward `json:"rewards"`
 }

 type StakeValidatorV1 struct {
-	BaseActivity
 	Address        string `json:"address"`
 	Fee            int    `json:"fee"`
+	Hash           string `json:"hash"`
 	Owner          string `json:"owner"`
 	Stake          int    `json:"stake"`
 	OwnerSignature string `json:"owner_signature"`
 }

 type TokenBurnV1 struct {
-	BaseActivity
 	Fee    int    `json:"fee"`
+	Hash   string `json:"hash"`
 	Memo   string `json:"memo"`
 	Nonce  int    `json:"nonce"`
 	Payer  string `json:"payer"`
@ -117,7 +109,7 @@ type TokenBurnV1 struct {
 }

 type TransferHotspotV1 struct {
-	BaseActivity
+	Hash           string `json:"hash"`
 	Fee            int    `json:"fee"`
 	Buyer          string `json:"buyer"`
 	Seller         string `json:"seller"`
@ -127,11 +119,11 @@ type TransferHotspotV1 struct {
 }

 type UnstakeValidatorV1 struct {
-	BaseActivity
 	Address            string `json:"address"`
 	Owner              string `json:"owner"`
 	OwnerSignature     string `json:"owner_signature"`
 	Fee                int    `json:"fee"`
 	StakeAmount        int    `json:"stake_amount"`
 	StakeReleaseHeight int    `json:"stake_release_height"`
+	Hash               string `json:"hash"`
 }
--- a/heliumapi/block.go
+++ b/heliumapi/block.go
@ -1,31 +0,0 @@
-package heliumapi
-
-import (
-	"encoding/json"
-	"fmt"
-	"time"
-)
-
-// query https://docs.helium.com/api/blockchain/blocks/#height
-func GetHeight(maxTime *time.Time) (int, error) {
-	path := "/v1/blocks/height"
-	params := map[string]string{}
-	if maxTime != nil {
-		params["max_time"] = maxTime.UTC().Format("2006-01-02T15:04:05Z")
-	}
-
-	// query the api
-	respBody, err := getHeliumApi(path, &params)
-	if err != nil {
-		return -1, err
-	}
-
-	// unmarshal the response
-	respobject := HeightResp{}
-	err = json.Unmarshal(respBody, &respobject)
-	if err != nil {
-		return -1, fmt.Errorf("failed to unmarshal response from %v: %v", path, err)
-	}
-
-	return respobject.Data.Height, nil
-}
--- a/heliumapi/hotspots.go
+++ b/heliumapi/hotspots.go
@ -52,43 +52,3 @@ func GetRewardsTotalForHotspot(hotspot string, minTime *time.Time, maxTime *time

 	return &respobject.Data, nil
 }
-
-// query https://docs.helium.com/api/blockchain/hotspots#witnesses-for-a-hotspot
-func GetWitnessesForHotspot(hotspot string) (*[]Hotspot, error) {
-	path := "/v1/hotspots/" + hotspot + "/witnesses"
-
-	// query the api
-	respBody, err := getHeliumApi(path, nil)
-	if err != nil {
-		return nil, err
-	}
-
-	// unmarshal the response
-	respobject := WitnessesResp{}
-	err = json.Unmarshal(respBody, &respobject)
-	if err != nil {
-		return nil, fmt.Errorf("failed to unmarshal response from %v: %v", path, err)
-	}
-
-	return &respobject.Data, nil
-}
-
-// query https://docs.helium.com/api/blockchain/hotspots#witnessed-for-a-hotspot
-func GetWitnessedForHotspot(hotspot string) (*[]Hotspot, error) {
-	path := "/v1/hotspots/" + hotspot + "/witnessed"
-
-	// query the api
-	respBody, err := getHeliumApi(path, nil)
-	if err != nil {
-		return nil, err
-	}
-
-	// unmarshal the response
-	respobject := WitnessedResp{}
-	err = json.Unmarshal(respBody, &respobject)
-	if err != nil {
-		return nil, fmt.Errorf("failed to unmarshal response from %v: %v", path, err)
-	}
-
-	return &respobject.Data, nil
-}
--- a/heliumapi/types.go
+++ b/heliumapi/types.go
@ -4,46 +4,6 @@ type AccountResp struct {
 	Data Account `json:"data"`
 }

-type AccountHotspotsResp struct {
-	Data []Hotspot `json:"data"`
-}
-
-type ActivityCounts map[string]int
-type ActivityCountsResp struct {
-	Data ActivityCounts
-}
-
-type RewardTotalResp struct {
-	Meta struct {
-		MinTime string `json:"min_time"`
-		MaxTime string `json:"max_time"`
-	} `json:"meta"`
-
-	Data RewardTotal `json:"data"`
-}
-
-type CurrentOraclePriceResp struct {
-	Data CurrentOraclePrice `json:"data"`
-}
-
-type BlockchainStatsResp struct {
-	Data BlockchainStats `json:"data"`
-}
-
-type WitnessesResp struct {
-	Data []Hotspot `json:"data"`
-}
-
-type HeightResp struct {
-	Data struct {
-		Height int `json:"height"`
-	} `json:"data"`
-}
-
-type WitnessedResp struct {
-	Data []Hotspot `json:"data"`
-}
-
 type Account struct {
 	Address          string `json:"address"`
 	Balance          int    `json:"balance"`
@ -55,7 +15,11 @@ type Account struct {
 	SpeculativeNonce int    `json:"speculative_nonce"`
 }

-type Hotspot struct {
+type AccountHotspotsResp struct {
+	Data []AccountHotspot `json:"data"`
+}
+
+type AccountHotspot struct {
 	Lng            float64 `json:"lng"`
 	Lat            float64 `json:"lat"`
 	TimestampAdded string  `json:"timestamp_added"`
@ -93,6 +57,21 @@ type Hotspot struct {
 	Address    string `json:"address"`
 }

+type ActivityCountsResp struct {
+	Data ActivityCounts
+}
+
+type ActivityCounts map[string]int
+
+type RewardTotalResp struct {
+	Meta struct {
+		MinTime string `json:"min_time"`
+		MaxTime string `json:"max_time"`
+	} `json:"meta"`
+
+	Data RewardTotal `json:"data"`
+}
+
 type RewardTotal struct {
 	Total  float64 `json:"total"`
 	Sum    float64 `json:"sum"`
@ -103,11 +82,19 @@ type RewardTotal struct {
 	Avg    float64 `json:"avg"`
 }

+type CurrentOraclePriceResp struct {
+	Data CurrentOraclePrice `json:"data"`
+}
+
 type CurrentOraclePrice struct {
 	Price int `json:"price"`
 	Block int `json:"block"`
 }

+type BlockchainStatsResp struct {
+	Data BlockchainStats `json:"data"`
+}
+
 type BlockchainStats struct {
 	BlockTime struct {
 		LastDay struct {