add hotspot metrics

add account activity and rewards metrics
2021-09-15 01:04:32 -04:00 · 2021-09-14 23:34:08 -04:00
2 changed files with 347 additions and 53 deletions
--- a/helium-blockchain-exporter.go
+++ b/helium-blockchain-exporter.go
@ -5,7 +5,9 @@ import (
 	"fmt"
 	"log"
 	"net/http"
+	"strconv"
 	"strings"
+	"time"

 	"github.com/helium-blockchain-exporter/heliumapi"
 	"github.com/prometheus/client_golang/prometheus"
@ -20,7 +22,20 @@ type metricInfo struct {

 // Exporter collect metrics from the helium blockchain api and exports them as prometheus metrics.
 type Exporter struct {
-	Accounts []string
+	Accounts []Account
+}
+
+// Account represents a helium account
+type Account struct {
+	hotspots          []Hotspot
+	hash              string
+	lastRewardsUpdate time.Time
+}
+
+// Hotspot represent a helium hotspot
+type Hotspot struct {
+	hash              string
+	lastRewardsUpdate time.Time
 }

 const (
@ -47,43 +62,43 @@ var (
 	// helium stats metrics
 	statsValidators = metricInfo{
 		prometheus.NewDesc(
-			prometheus.BuildFQName(namespace, "stats", "validators"),
-			"The number of validators.",
+			prometheus.BuildFQName(namespace, "stats", "validators_total"),
+			"The total number of validators.",
 			nil, nil,
 		),
-		prometheus.GaugeValue,
+		prometheus.CounterValue,
 	}
 	statsOuis = metricInfo{
 		prometheus.NewDesc(
-			prometheus.BuildFQName(namespace, "stats", "ouis"),
-			"The number of organization unique identifiers.",
+			prometheus.BuildFQName(namespace, "stats", "ouis_total"),
+			"The total number of organization unique identifiers.",
 			nil, nil,
 		),
-		prometheus.GaugeValue,
+		prometheus.CounterValue,
 	}
 	statsHotspotsDataOnly = metricInfo{
 		prometheus.NewDesc(
-			prometheus.BuildFQName(namespace, "stats", "hotspots_dataonly"),
-			"The number of data only hotspots.",
+			prometheus.BuildFQName(namespace, "stats", "hotspots_dataonly_total"),
+			"The total number of data only hotspots.",
 			nil, nil,
 		),
-		prometheus.GaugeValue,
+		prometheus.CounterValue,
 	}
 	statsBlocks = metricInfo{
 		prometheus.NewDesc(
-			prometheus.BuildFQName(namespace, "stats", "blocks"),
-			"The height/number of blocks in the blockchain.",
+			prometheus.BuildFQName(namespace, "stats", "blocks_total"),
+			"The total height/number of blocks in the blockchain.",
 			nil, nil,
 		),
-		prometheus.GaugeValue,
+		prometheus.CounterValue,
 	}
 	statsChallenges = metricInfo{
 		prometheus.NewDesc(
-			prometheus.BuildFQName(namespace, "stats", "challenges"),
-			"The number of challenges.",
+			prometheus.BuildFQName(namespace, "stats", "challenges_total"),
+			"The total number of challenges.",
 			nil, nil,
 		),
-		prometheus.GaugeValue,
+		prometheus.CounterValue,
 	}
 	statsCities = metricInfo{
 		prometheus.NewDesc(
@ -95,8 +110,8 @@ var (
 	}
 	statsConsensusGroups = metricInfo{
 		prometheus.NewDesc(
-			prometheus.BuildFQName(namespace, "stats", "consensus_groups"),
-			"The number of consensus groups.",
+			prometheus.BuildFQName(namespace, "stats", "consensus_groups_total"),
+			"The total number of consensus groups.",
 			nil, nil,
 		),
 		prometheus.GaugeValue,
@ -111,11 +126,11 @@ var (
 	}
 	statsHotspots = metricInfo{
 		prometheus.NewDesc(
-			prometheus.BuildFQName(namespace, "stats", "hotspots"),
-			"The number of hotspots.",
+			prometheus.BuildFQName(namespace, "stats", "hotspots_total"),
+			"The total number of hotspots.",
 			nil, nil,
 		),
-		prometheus.GaugeValue,
+		prometheus.CounterValue,
 	}
 	statsTokenSupply = metricInfo{
 		prometheus.NewDesc(
@ -130,7 +145,7 @@ var (
 	accountBalanceHnt = metricInfo{
 		prometheus.NewDesc(
 			prometheus.BuildFQName(namespace, "account", "balance_hnt"),
-			"The number of HNT token owned by the account.",
+			"The number of HNT token owned by an account.",
 			commonAccountLabels, nil,
 		),
 		prometheus.GaugeValue,
@ -145,32 +160,103 @@ var (
 	}
 	accountRewardsHnt = metricInfo{
 		prometheus.NewDesc(
-			prometheus.BuildFQName(namespace, "account", "reward_hnt"),
-			"The number of HNT token rewarded to this account.",
+			prometheus.BuildFQName(namespace, "account", "rewards_hnt"),
+			"The number of HNT token rewarded to an account.",
 			commonAccountLabels, nil,
 		),
 		prometheus.GaugeValue,
 	}
-	accountDepositsHnt = metricInfo{
-		prometheus.NewDesc(
-			prometheus.BuildFQName(namespace, "account", "deposits_hnt_total"),
-			"The number of HNT token deposited to this account.",
-			commonAccountLabels, nil,
-		),
-		prometheus.CounterValue,
-	}
-	accountWithdrawalsHnt = metricInfo{
-		prometheus.NewDesc(
-			prometheus.BuildFQName(namespace, "account", "withdrawals_hnt_total"),
-			"The number of HNT token withdrawn from this account.",
-			commonAccountLabels, nil,
-		),
-		prometheus.CounterValue,
-	}
+	// accountDepositsHnt = metricInfo{
+	// 	prometheus.NewDesc(
+	// 		prometheus.BuildFQName(namespace, "account", "deposits_hnt_total"),
+	// 		"The number of HNT token deposited to this account.",
+	// 		commonAccountLabels, nil,
+	// 	),
+	// 	prometheus.CounterValue,
+	// }
+	// accountWithdrawalsHnt = metricInfo{
+	// 	prometheus.NewDesc(
+	// 		prometheus.BuildFQName(namespace, "account", "withdrawals_hnt_total"),
+	// 		"The number of HNT token withdrawn from this account.",
+	// 		commonAccountLabels, nil,
+	// 	),
+	// 	prometheus.CounterValue,
+	// }

-// helium hotspot metrics
+	// helium hotspot metrics
+	hotspotUp = metricInfo{
+		prometheus.NewDesc(
+			prometheus.BuildFQName(namespace, "hotspot", "up"),
+			"Whether a hotspot is online.",
+			commonHotspotLabels, nil,
+		),
+		prometheus.GaugeValue,
+	}
+	hotspotListening = metricInfo{
+		prometheus.NewDesc(
+			prometheus.BuildFQName(namespace, "hotspot", "listening"),
+			"Whether a hotspot is listening (not relayed).",
+			commonHotspotLabels, nil,
+		),
+		prometheus.GaugeValue,
+	}
+	hotspotBlockHeight = metricInfo{
+		prometheus.NewDesc(
+			prometheus.BuildFQName(namespace, "hotspot", "block_height"),
+			"The block height of a Hotspot. Check on the hotspot itself for the most recent data.",
+			commonHotspotLabels, nil,
+		),
+		prometheus.CounterValue,
+	}
+	hotspotRewardScale = metricInfo{
+		prometheus.NewDesc(
+			prometheus.BuildFQName(namespace, "hotspot", "reward_scale"),
+			"The reward scale of a hotspot.",
+			commonHotspotLabels, nil,
+		),
+		prometheus.GaugeValue,
+	}
+	hotspotGeocodeInfo = metricInfo{
+		prometheus.NewDesc(
+			prometheus.BuildFQName(namespace, "hotspot", "geocode_info"),
+			"Information on the location of a hotspot.",
+			append(commonHotspotLabels, "lng", "lat", "street", "state", "country", "city"), nil,
+		),
+		prometheus.GaugeValue,
+	}
+	hotspotAntennaInfo = metricInfo{
+		prometheus.NewDesc(
+			prometheus.BuildFQName(namespace, "hotspot", "antenna_info"),
+			"Information on the location of a hotspot.",
+			append(commonHotspotLabels, "gain", "elevation"), nil,
+		),
+		prometheus.GaugeValue,
+	}
+	hotspotActivity = metricInfo{
+		prometheus.NewDesc(
+			prometheus.BuildFQName(namespace, "hotspot", "activity_total"),
+			"The total number of time an activity occurred in a hotspot.",
+			append(commonHotspotLabels, "type"), nil,
+		),
+		prometheus.CounterValue,
+	}
+	hotspotRewardsHnt = metricInfo{
+		prometheus.NewDesc(
+			prometheus.BuildFQName(namespace, "hotspot", "rewards_hnt"),
+			"The number of HNT token rewarded to a hotspot.",
+			commonHotspotLabels, nil,
+		),
+		prometheus.GaugeValue,
+	}
 )

+func bool2Float64(b bool) float64 {
+	if b {
+		return 1.0
+	}
+	return 0.0
+}
+
 // Describe describes all the metrics ever exported by the helium blockchain exporter.
 // implements prometheus.Collector.
 func (e *Exporter) Describe(ch chan<- *prometheus.Desc) {
@ -190,8 +276,17 @@ func (e *Exporter) Describe(ch chan<- *prometheus.Desc) {
 	ch <- accountBalanceHnt.Desc
 	ch <- accountActivity.Desc
 	ch <- accountRewardsHnt.Desc
-	ch <- accountDepositsHnt.Desc
-	ch <- accountWithdrawalsHnt.Desc
+	// ch <- accountDepositsHnt.Desc
+	// ch <- accountWithdrawalsHnt.Desc
+
+	ch <- hotspotUp.Desc
+	ch <- hotspotListening.Desc
+	ch <- hotspotBlockHeight.Desc
+	ch <- hotspotRewardScale.Desc
+	ch <- hotspotGeocodeInfo.Desc
+	ch <- hotspotAntennaInfo.Desc
+	ch <- hotspotActivity.Desc
+	ch <- hotspotRewardsHnt.Desc
 }

 // Collect fetches the data from the helium blockchain api and delivers them as Prometheus metrics.
@ -200,7 +295,8 @@ func (e *Exporter) Collect(ch chan<- prometheus.Metric) {
 	e.collectOracleMetrics(ch)
 	e.collectStatsMetrics(ch)
 	for _, account := range e.Accounts {
-		e.collectAccountMetrics(ch, account)
+		e.collectAccountMetrics(ch, &account)
+		e.collectHotspotMetrics(ch, &account)
 	}
 }

@ -258,8 +354,20 @@ func (e *Exporter) collectStatsMetrics(ch chan<- prometheus.Metric) {
 }

 // collectStatsMetrics collect metrics in the account group from the helium api
-func (e *Exporter) collectAccountMetrics(ch chan<- prometheus.Metric, account string) {
-	accountForAddress, err := heliumapi.GetAccountForAddress(account)
+func (e *Exporter) collectAccountMetrics(ch chan<- prometheus.Metric, account *Account) {
+	accountForAddress, err := heliumapi.GetAccountForAddress(account.hash)
+	if err != nil {
+		fmt.Println(err)
+		return
+	}
+
+	accountActivityForAddress, err := heliumapi.GetActivityCountsForAccount(account.hash)
+	if err != nil {
+		fmt.Println(err)
+		return
+	}
+
+	accountRewardTotalsForAddress, err := heliumapi.GetRewardTotalsForAccount(account.hash, &account.lastRewardsUpdate, nil)
 	if err != nil {
 		fmt.Println(err)
 		return
@ -267,17 +375,80 @@ func (e *Exporter) collectAccountMetrics(ch chan<- prometheus.Metric, account st

 	ch <- prometheus.MustNewConstMetric(
 		accountBalanceHnt.Desc, accountBalanceHnt.Type, float64(accountForAddress.Data.Balance),
-		account,
+		account.hash,
 	)
+	for accType, count := range accountActivityForAddress.Data {
+		ch <- prometheus.MustNewConstMetric(
+			accountActivity.Desc, accountActivity.Type, float64(count),
+			account.hash, accType,
+		)
+	}
+	ch <- prometheus.MustNewConstMetric(
+		accountRewardsHnt.Desc, accountRewardsHnt.Type, accountRewardTotalsForAddress.Data.Sum,
+		account.hash,
+	)
+	account.lastRewardsUpdate, err = time.Parse(time.RFC3339, accountRewardTotalsForAddress.Meta.MaxTime)
+	if err != nil {
+		fmt.Printf("failed to parse time \"%v\", value of %v will be bad: %v\n", accountRewardTotalsForAddress.Meta.MaxTime, accountRewardsHnt.Desc.String(), err)
+	}
+}
+
+// collectStatsMetrics collect metrics in the hotspot group from the helium api
+func (e *Exporter) collectHotspotMetrics(ch chan<- prometheus.Metric, account *Account) {
+	hotspotsForAddress, err := heliumapi.GetHotspotsForAccount(account.hash)
+	if err != nil {
+		fmt.Println(err)
+		return
+	}
+
+	for _, hotspotData := range hotspotsForAddress.Data {
+		ch <- prometheus.MustNewConstMetric(
+			hotspotUp.Desc, hotspotUp.Type, bool2Float64(hotspotData.Status.Online == "online"),
+			account.hash, hotspotData.Address, hotspotData.Name,
+		)
+		ch <- prometheus.MustNewConstMetric(
+			hotspotListening.Desc, hotspotListening.Type, bool2Float64(len(hotspotData.Status.ListenAddrs) != 0),
+			account.hash, hotspotData.Address, hotspotData.Name,
+		)
+		ch <- prometheus.MustNewConstMetric(
+			hotspotBlockHeight.Desc, hotspotBlockHeight.Type, float64(hotspotData.Status.Height),
+			account.hash, hotspotData.Address, hotspotData.Name,
+		)
+		ch <- prometheus.MustNewConstMetric(
+			hotspotRewardScale.Desc, hotspotRewardScale.Type, float64(hotspotData.RewardScale),
+			account.hash, hotspotData.Address, hotspotData.Name,
+		)
+		ch <- prometheus.MustNewConstMetric(
+			hotspotGeocodeInfo.Desc, hotspotGeocodeInfo.Type, 1.0,
+			account.hash, hotspotData.Address, hotspotData.Name, strconv.FormatFloat(hotspotData.Lng, 'f', 6, 64), strconv.FormatFloat(hotspotData.Lat, 'f', 6, 64), hotspotData.Geocode.LongStreet, hotspotData.Geocode.LongState, hotspotData.Geocode.LongCountry, hotspotData.Geocode.LongCity,
+		)
+		ch <- prometheus.MustNewConstMetric(
+			hotspotAntennaInfo.Desc, hotspotAntennaInfo.Type, 1.0,
+			account.hash, hotspotData.Address, hotspotData.Name, strconv.Itoa(hotspotData.Gain), strconv.Itoa(hotspotData.Elevation),
+		)
+	}
 }

 // NewExporter returns an initialized Exporter
-func NewExporter(accounts []string) (*Exporter, error) {
+func NewExporter(accountHashs []string) (*Exporter, error) {
+	accounts := make([]Account, 0)
+	for _, accountHash := range accountHashs {
+		if accountHash != "" {
+			accounts = append(accounts, NewAccount(accountHash))
+		}
+	}
 	return &Exporter{
 		Accounts: accounts,
 	}, nil
 }

+func NewAccount(hash string) Account {
+	return Account{
+		hash:              hash,
+		lastRewardsUpdate: time.Now(),
+	}
+}
+
 func main() {
 	fHeliumAccounts := flag.String("accounts", "", "A comma-delimited list of helium accounts to scrape.")
 	fMetricsPath := flag.String("metricpath", "/metrics", "The metrics path")
@ -285,10 +456,7 @@ func main() {
 	fListenPort := flag.String("listenPort", "9111", "The http server listen port")
 	flag.Parse()

-	heliumAccounts := make([]string, 0)
-	if *fHeliumAccounts != "" {
-		heliumAccounts = strings.Split(*fHeliumAccounts, ",")
-	}
+	heliumAccounts := strings.Split(*fHeliumAccounts, ",")
 	serverAddr := *fListenAddress + ":" + *fListenPort

 	e, err := NewExporter(heliumAccounts)
@ -311,6 +479,6 @@ func main() {
 	})

 	http.Handle(*fMetricsPath, promhttp.HandlerFor(r, promhttp.HandlerOpts{}))
-	fmt.Printf("exporter listening on %v\n", serverAddr)
+	fmt.Printf("listening on %v\n", serverAddr)
 	http.ListenAndServe(serverAddr, nil)
 }
--- a/heliumapi/accounts.go
+++ b/heliumapi/accounts.go
@ -3,6 +3,7 @@ package heliumapi
 import (
 	"encoding/json"
 	"fmt"
+	"time"
 )

 type Account struct {
@ -18,6 +19,67 @@ type Account struct {
 	} `json:"data"`
 }

+type ActivityCounts struct {
+	Data map[string]int
+}
+
+type RewardTotal struct {
+	Meta struct {
+		MinTime string `json:"min_time"`
+		MaxTime string `json:"max_time"`
+	} `json:"meta"`
+
+	Data struct {
+		Total  float64 `json:"total"`
+		Sum    float64 `json:"sum"`
+		Stddev float64 `json:"stddev"`
+		Min    float64 `json:"min"`
+		Median float64 `json:"median"`
+		Max    float64 `json:"max"`
+		Avg    float64 `json:"avg"`
+	} `json:"data"`
+}
+
+type AccountHotspots struct {
+	Data []struct {
+		Lng            float64 `json:"lng"`
+		Lat            float64 `json:"lat"`
+		TimestampAdded string  `json:"timestamp_added"`
+		Status         struct {
+			Timestamp   string `json:"timestamp"`
+			Online      string `json:"online"`
+			ListenAddrs []string
+			Height      int
+		} `json:"status"`
+		RewardScale      float64 `json:"reward_scale"`
+		Payer            string  `json:"payer"`
+		Owner            string  `json:"owner"`
+		Nonce            int     `json:"nonce"`
+		Name             string  `json:"name"`
+		Mode             string  `json:"mode"`
+		LocationHex      string  `json:"location_hex"`
+		Location         string  `json:"location"`
+		LastPocChallenge int     `json:"last_poc_challenge"`
+		LastChangeBlock  int     `json:"last_change_block"`
+		Geocode          struct {
+			ShortStreet  string `json:"short_street"`
+			ShortState   string `json:"short_state"`
+			ShortCountry string `json:"short_country"`
+			ShortCity    string `json:"short_city"`
+			LongStreet   string `json:"long_street"`
+			LongState    string `json:"long_state"`
+			LongCountry  string `json:"long_country"`
+			LongCity     string `json:"long_city"`
+			CityID       string `json:"city_id"`
+		} `json:"geocode"`
+		Gain       int    `json:"gain"`
+		Elevation  int    `json:"elevation"`
+		BlockAdded int    `json:"block_added"`
+		Block      int    `json:"block"`
+		Address    string `json:"address"`
+	} `json:"data"`
+}
+
 func GetAccountForAddress(account string) (*Account, error) {
 	path := "/v1/accounts/" + account

@ -36,3 +98,67 @@ func GetAccountForAddress(account string) (*Account, error) {

 	return &respobject, nil
 }
+
+func GetActivityCountsForAccount(account string) (*ActivityCounts, error) {
+	path := "/v1/accounts/" + account + "/activity/count"
+
+	// query the api
+	respBody, err := getHeliumApi(path, nil)
+	if err != nil {
+		return nil, err
+	}
+
+	// unmarshal the response
+	respobject := ActivityCounts{}
+	err = json.Unmarshal(respBody, &respobject)
+	if err != nil {
+		return nil, fmt.Errorf("failed to unmarshal response from path %v: %v", path, err)
+	}
+
+	return &respobject, nil
+}
+
+func GetRewardTotalsForAccount(account string, minTime *time.Time, maxTime *time.Time) (*RewardTotal, error) {
+	path := "/v1/accounts/" + account + "/rewards/sum"
+	params := map[string]string{}
+	if minTime != nil {
+		params["min_time"] = minTime.UTC().Format("2006-01-02T15:04:05Z")
+	}
+	if maxTime != nil {
+		params["max_time"] = minTime.UTC().Format("2006-01-02T15:04:05Z")
+	}
+
+	// query the api
+	respBody, err := getHeliumApi(path, &params)
+	if err != nil {
+		return nil, err
+	}
+
+	// unmarshal the response
+	respobject := RewardTotal{}
+	err = json.Unmarshal(respBody, &respobject)
+	if err != nil {
+		return nil, fmt.Errorf("failed to unmarshal response from path %v: %v", path, err)
+	}
+
+	return &respobject, nil
+}
+
+func GetHotspotsForAccount(account string) (*AccountHotspots, error) {
+	path := "/v1/accounts/" + account + "/hotspots"
+
+	// query the api
+	respBody, err := getHeliumApi(path, nil)
+	if err != nil {
+		return nil, err
+	}
+
+	// unmarshal the response
+	respobject := AccountHotspots{}
+	err = json.Unmarshal(respBody, &respobject)
+	if err != nil {
+		return nil, fmt.Errorf("failed to unmarshal response from path %v: %v", path, err)
+	}
+
+	return &respobject, nil
+}
Author	SHA1	Message	Date
Massaki Archambault	cc251b1041	add hotspot metrics	2021-09-15 01:04:32 -04:00
Massaki Archambault	3952b7738a	add account activity and rewards metrics	2021-09-14 23:34:08 -04:00