From fd43e3751f48e404164aa88ca52b0284f709b0bf Mon Sep 17 00:00:00 2001 From: Massaki Archambault Date: Tue, 12 Oct 2021 09:19:58 -0400 Subject: [PATCH] expose hotspot block gap --- exporter.go | 69 +++++++++++++++++++------------- heliumapi/accounts.go | 17 ++++---- heliumapi/activity/activities.go | 12 +++--- heliumapi/block.go | 3 +- heliumapi/hotspots.go | 12 +++--- heliumapi/query.go | 22 +++++----- heliumapi/types.go | 7 ++++ 7 files changed, 85 insertions(+), 57 deletions(-) diff --git a/exporter.go b/exporter.go index 8e25d1b..a1bdb42 100644 --- a/exporter.go +++ b/exporter.go @@ -3,7 +3,6 @@ package main import ( "flag" "log" - "math" "net/http" "os" "strconv" @@ -117,7 +116,7 @@ var ( statsBlocks = metricInfo{ prometheus.NewDesc( prometheus.BuildFQName(namespace, "stats", "blocks"), - "The total height/number of blocks in the blockchain.", + "The height of the blockchain.", nil, nil, ), prometheus.GaugeValue, @@ -199,7 +198,7 @@ var ( accountDepositsHnt = metricInfo{ prometheus.NewDesc( prometheus.BuildFQName(namespace, "account", "deposits_hnt_total"), - "The number of HNT tokens deposited to this account.", + "The number of HNT tokens deposited to an account.", commonAccountLabels, nil, ), prometheus.CounterValue, @@ -207,7 +206,7 @@ var ( accountWithdrawalsHnt = metricInfo{ prometheus.NewDesc( prometheus.BuildFQName(namespace, "account", "withdrawals_hnt_total"), - "The number of HNT tokens withdrawn from this account.", + "The number of HNT tokens withdrawn from an account.", commonAccountLabels, nil, ), prometheus.CounterValue, @@ -222,14 +221,6 @@ var ( ), prometheus.GaugeValue, } - hotspotSynced = metricInfo{ - prometheus.NewDesc( - prometheus.BuildFQName(namespace, "hotspot", "synced"), - "Whether a hotspot is synced with the blockchain.", - commonHotspotLabels, nil, - ), - prometheus.GaugeValue, - } hotspotRelayed = metricInfo{ prometheus.NewDesc( prometheus.BuildFQName(namespace, "hotspot", "relayed"), @@ -240,11 +231,27 @@ var ( } hotspotBlocks = metricInfo{ prometheus.NewDesc( - prometheus.BuildFQName(namespace, "hotspot", "blocks_total"), + prometheus.BuildFQName(namespace, "hotspot", "blocks"), "The block height of a hotspot. Check on the hotspot itself for the most recent data.", commonHotspotLabels, nil, ), - prometheus.CounterValue, + prometheus.GaugeValue, + } + hotspotBlocksDelta = metricInfo{ + prometheus.NewDesc( + prometheus.BuildFQName(namespace, "hotspot", "blocks_delta"), + "The gap between the height of a hotspot and the height of the blockchain. A large negative gap may indicate the hotspot is out of sync. Check on the hotspot itself for the most recent data.", + commonHotspotLabels, nil, + ), + prometheus.GaugeValue, + } + hotspotStatusTimestamp = metricInfo{ + prometheus.NewDesc( + prometheus.BuildFQName(namespace, "hotspot", "status_timestamp"), + "The the last time a hotspot gossiped its status on the blockchain.", + commonHotspotLabels, nil, + ), + prometheus.GaugeValue, } hotspotRewardsScale = metricInfo{ prometheus.NewDesc( @@ -334,9 +341,10 @@ func (e *Exporter) Describe(ch chan<- *prometheus.Desc) { ch <- accountWithdrawalsHnt.Desc ch <- hotspotUp.Desc - ch <- hotspotSynced.Desc ch <- hotspotRelayed.Desc + ch <- hotspotBlocksDelta.Desc ch <- hotspotBlocks.Desc + ch <- hotspotStatusTimestamp.Desc ch <- hotspotRewardsScale.Desc ch <- hotspot5dWitnesses.Desc ch <- hotspot5dWitnessed.Desc @@ -607,10 +615,16 @@ func (e *Exporter) collectHotspotMetrics(wg *sync.WaitGroup, ch chan<- prometheu return } - for _, hotspotData := range *hotspotsForAddress { + for _, hotspotData := range hotspotsForAddress { + statusTime, err := time.Parse(time.RFC3339Nano, hotspotData.Status.Timestamp) + if err != nil { + log.Println(err) + continue + } + // collect hotspot metric requiring extra queries in a new routine wg.Add(5) - go e.collectHotspotSyncedMetrics(wg, ch, account, hotspotData) + go e.collectHotspotBlocksDeltaMetrics(wg, ch, account, hotspotData) go e.collectHotspotWitnessesMetrics(wg, ch, account, hotspotData) go e.collectHotspotWitnessedMetrics(wg, ch, account, hotspotData) go e.collectHotspotActivityMetrics(wg, ch, account, hotspotData) @@ -628,6 +642,10 @@ func (e *Exporter) collectHotspotMetrics(wg *sync.WaitGroup, ch chan<- prometheu hotspotBlocks.Desc, hotspotBlocks.Type, float64(hotspotData.Status.Height), account.Address, hotspotData.Address, hotspotData.Name, ) + ch <- prometheus.MustNewConstMetric( + hotspotStatusTimestamp.Desc, hotspotStatusTimestamp.Type, float64(statusTime.Unix()), + account.Address, hotspotData.Address, hotspotData.Name, + ) ch <- prometheus.MustNewConstMetric( hotspotRewardsScale.Desc, hotspotRewardsScale.Type, float64(hotspotData.RewardScale), account.Address, hotspotData.Address, hotspotData.Name, @@ -643,11 +661,11 @@ func (e *Exporter) collectHotspotMetrics(wg *sync.WaitGroup, ch chan<- prometheu } } -// collectSyncedMetrics calculate if the hotspot is in sync with the blockchain -func (e *Exporter) collectHotspotSyncedMetrics(wg *sync.WaitGroup, ch chan<- prometheus.Metric, account *Account, hotspotData heliumapi.Hotspot) { +// collectHotspotBlocksDeltaMetrics calculate the gap between the block height of a hotstop and the height of the chain +func (e *Exporter) collectHotspotBlocksDeltaMetrics(wg *sync.WaitGroup, ch chan<- prometheus.Metric, account *Account, hotspotData heliumapi.Hotspot) { defer wg.Done() - isSynced := 0. + delta := 0. if hotspotData.Status.Online == "online" && hotspotData.Status.Timestamp != "" { statusTime, err := time.Parse(time.RFC3339Nano, hotspotData.Status.Timestamp) if err != nil { @@ -659,14 +677,11 @@ func (e *Exporter) collectHotspotSyncedMetrics(wg *sync.WaitGroup, ch chan<- pro log.Println(err) return } - // if the block height and the height reported by the hotspot is within 10 blocks, we consider it in sync - if math.Abs(float64(hotspotData.Status.Height-heightAtUpdate)) <= 10 { - isSynced = 1. - } + delta = float64(hotspotData.Status.Height - heightAtUpdate) } ch <- prometheus.MustNewConstMetric( - hotspotSynced.Desc, hotspotActivity.Type, isSynced, + hotspotBlocksDelta.Desc, hotspotBlocksDelta.Type, delta, account.Address, hotspotData.Address, hotspotData.Name, ) } @@ -682,7 +697,7 @@ func (e *Exporter) collectHotspotWitnessesMetrics(wg *sync.WaitGroup, ch chan<- } ch <- prometheus.MustNewConstMetric( - hotspot5dWitnesses.Desc, hotspotActivity.Type, float64(len(*hotspotWitnesses)), + hotspot5dWitnesses.Desc, hotspotActivity.Type, float64(len(hotspotWitnesses)), account.Address, hotspotData.Address, hotspotData.Name, ) } @@ -698,7 +713,7 @@ func (e *Exporter) collectHotspotWitnessedMetrics(wg *sync.WaitGroup, ch chan<- } ch <- prometheus.MustNewConstMetric( - hotspot5dWitnessed.Desc, hotspotActivity.Type, float64(len(*hotspotWitnessed)), + hotspot5dWitnessed.Desc, hotspotActivity.Type, float64(len(hotspotWitnessed)), account.Address, hotspotData.Address, hotspotData.Name, ) } diff --git a/heliumapi/accounts.go b/heliumapi/accounts.go index a4a1436..15b8266 100644 --- a/heliumapi/accounts.go +++ b/heliumapi/accounts.go @@ -32,14 +32,15 @@ func GetAccountForAddress(account string) (*Account, error) { // query https://docs.helium.com/api/blockchain/accounts#activity-for-account func GetActivityForAccount(account string, filterTypes []string, minTime *time.Time, maxTime *time.Time) (*activity.Activities, error) { path := "/v1/accounts/" + account + "/activity" - params := map[string]string{ - "filter_types": strings.Join(filterTypes, ","), + params := map[string]string{} + if filterTypes != nil { + params["min_time"] = strings.Join(filterTypes, ",") } if minTime != nil { - params["min_time"] = minTime.UTC().Format("2006-01-02T15:04:05Z") + params["min_time"] = minTime.UTC().Format(timeFormat) } if maxTime != nil { - params["max_time"] = maxTime.UTC().Format("2006-01-02T15:04:05Z") + params["max_time"] = maxTime.UTC().Format(timeFormat) } // query the api @@ -87,10 +88,10 @@ func GetRewardTotalsForAccount(account string, minTime *time.Time, maxTime *time path := "/v1/accounts/" + account + "/rewards/sum" params := map[string]string{} if minTime != nil { - params["min_time"] = minTime.UTC().Format("2006-01-02T15:04:05Z") + params["min_time"] = minTime.UTC().Format(timeFormat) } if maxTime != nil { - params["max_time"] = maxTime.UTC().Format("2006-01-02T15:04:05Z") + params["max_time"] = maxTime.UTC().Format(timeFormat) } // query the api @@ -110,7 +111,7 @@ func GetRewardTotalsForAccount(account string, minTime *time.Time, maxTime *time } // query https://docs.helium.com/api/blockchain/accounts#hotspots-for-account -func GetHotspotsForAccount(account string) (*[]Hotspot, error) { +func GetHotspotsForAccount(account string) ([]Hotspot, error) { path := "/v1/accounts/" + account + "/hotspots" // query the api @@ -126,5 +127,5 @@ func GetHotspotsForAccount(account string) (*[]Hotspot, error) { return nil, fmt.Errorf("failed to unmarshal response from %v: %v", path, err) } - return &respobject.Data, nil + return respobject.Data, nil } diff --git a/heliumapi/activity/activities.go b/heliumapi/activity/activities.go index c9727c5..a924f21 100644 --- a/heliumapi/activity/activities.go +++ b/heliumapi/activity/activities.go @@ -59,17 +59,17 @@ func NewActivities(resp ActivityResp) (*Activities, error) { } activities.PaymentV2 = append(activities.PaymentV2, paymentV2) case "rewards_v1": - rewardV1 := RewardsV1{} - if err := json.Unmarshal(activityRaw, &rewardV1); err != nil { + rewardsV1 := RewardsV1{} + if err := json.Unmarshal(activityRaw, &rewardsV1); err != nil { return nil, fmt.Errorf("failed to unmarshal %v: %v", activityType.Type, err) } - activities.RewardsV1 = append(activities.RewardsV1, rewardV1) + activities.RewardsV1 = append(activities.RewardsV1, rewardsV1) case "rewards_v2": - rewardV2 := RewardsV2{} - if err := json.Unmarshal(activityRaw, &rewardV2); err != nil { + rewardsV2 := RewardsV2{} + if err := json.Unmarshal(activityRaw, &rewardsV2); err != nil { return nil, fmt.Errorf("failed to unmarshal %v: %v", activityType.Type, err) } - activities.RewardsV2 = append(activities.RewardsV2, rewardV2) + activities.RewardsV2 = append(activities.RewardsV2, rewardsV2) case "stake_validator_v1": stakeValidatorV1 := StakeValidatorV1{} if err := json.Unmarshal(activityRaw, &stakeValidatorV1); err != nil { diff --git a/heliumapi/block.go b/heliumapi/block.go index 227b66d..c455452 100644 --- a/heliumapi/block.go +++ b/heliumapi/block.go @@ -8,10 +8,11 @@ import ( // query https://docs.helium.com/api/blockchain/blocks/#height func GetHeight(maxTime *time.Time) (int, error) { + // TODO: the result of this query could be cached path := "/v1/blocks/height" params := map[string]string{} if maxTime != nil { - params["max_time"] = maxTime.UTC().Format("2006-01-02T15:04:05Z") + params["max_time"] = maxTime.UTC().Format(timeFormat) } // query the api diff --git a/heliumapi/hotspots.go b/heliumapi/hotspots.go index ab7b688..99d66b8 100644 --- a/heliumapi/hotspots.go +++ b/heliumapi/hotspots.go @@ -31,10 +31,10 @@ func GetRewardsTotalForHotspot(hotspot string, minTime *time.Time, maxTime *time path := "/v1/hotspots/" + hotspot + "/rewards/sum" params := map[string]string{} if minTime != nil { - params["min_time"] = minTime.UTC().Format("2006-01-02T15:04:05Z") + params["min_time"] = minTime.UTC().Format(timeFormat) } if maxTime != nil { - params["max_time"] = maxTime.UTC().Format("2006-01-02T15:04:05Z") + params["max_time"] = maxTime.UTC().Format(timeFormat) } // query the api @@ -54,7 +54,7 @@ func GetRewardsTotalForHotspot(hotspot string, minTime *time.Time, maxTime *time } // query https://docs.helium.com/api/blockchain/hotspots#witnesses-for-a-hotspot -func GetWitnessesForHotspot(hotspot string) (*[]Hotspot, error) { +func GetWitnessesForHotspot(hotspot string) ([]Hotspot, error) { path := "/v1/hotspots/" + hotspot + "/witnesses" // query the api @@ -70,11 +70,11 @@ func GetWitnessesForHotspot(hotspot string) (*[]Hotspot, error) { return nil, fmt.Errorf("failed to unmarshal response from %v: %v", path, err) } - return &respobject.Data, nil + return respobject.Data, nil } // query https://docs.helium.com/api/blockchain/hotspots#witnessed-for-a-hotspot -func GetWitnessedForHotspot(hotspot string) (*[]Hotspot, error) { +func GetWitnessedForHotspot(hotspot string) ([]Hotspot, error) { path := "/v1/hotspots/" + hotspot + "/witnessed" // query the api @@ -90,5 +90,5 @@ func GetWitnessedForHotspot(hotspot string) (*[]Hotspot, error) { return nil, fmt.Errorf("failed to unmarshal response from %v: %v", path, err) } - return &respobject.Data, nil + return respobject.Data, nil } diff --git a/heliumapi/query.go b/heliumapi/query.go index 26236fc..4f1772c 100644 --- a/heliumapi/query.go +++ b/heliumapi/query.go @@ -4,7 +4,9 @@ import ( "encoding/json" "fmt" "io/ioutil" + "log" "net/http" + "strings" ) var ( @@ -14,7 +16,7 @@ var ( // createGetRequest create a GET request to the helium api func createGetRequest(path string, params map[string]string) (*http.Request, error) { - req, err := http.NewRequest("GET", fmt.Sprintf("%s/%s", ApiUrl, path), nil) + req, err := http.NewRequest("GET", fmt.Sprintf("%s%s", ApiUrl, path), nil) // setup headers req.Header.Add("Accept", "application/json") @@ -47,21 +49,22 @@ func getHeliumApi(path string, params *map[string]string) ([]byte, error) { } // query the api + log.Printf("querying %v", req.URL.String()) resp, err := client.Do(req) if err != nil { - return nil, fmt.Errorf("failed to query %v: %v", req.URL.RequestURI(), err) + return nil, fmt.Errorf("failed to query %v: %v", req.URL.String(), err) } defer resp.Body.Close() // validate the response status if resp.StatusCode != 200 { - return nil, fmt.Errorf("failed to query %v: http status %v", req.URL.RequestURI(), resp.StatusCode) + return nil, fmt.Errorf("failed to query %v: http status %v", req.URL.String(), resp.StatusCode) } // read the response body body, err := ioutil.ReadAll(resp.Body) if err != nil { - return nil, fmt.Errorf("failed to read response body of %v: %v", req.URL.RequestURI(), err) + return nil, fmt.Errorf("failed to read response body of %v: %v", req.URL.String(), err) } return body, nil @@ -91,27 +94,28 @@ func getHeliumApiWithCursor(path string, params *map[string]string) ([][]byte, e // query the api resp, err := client.Do(req) if err != nil { - return nil, fmt.Errorf("failed to query %v: %v", req.URL.RequestURI(), err) + return nil, fmt.Errorf("failed to query %v: %v", req.URL.String(), err) } defer resp.Body.Close() // read the response body and add it to the result array + log.Printf("querying %v", req.URL.String()) body, err := ioutil.ReadAll(resp.Body) if err != nil { - return nil, fmt.Errorf("failed to read response body of %v: %v", req.URL.RequestURI(), err) + return nil, fmt.Errorf("failed to read response body of %v: %v", req.URL.String(), err) } res = append(res, body) // validate the response status if resp.StatusCode != 200 { - return nil, fmt.Errorf("failed to query %v: http status %v", req.URL.RequestURI(), resp.StatusCode) + return nil, fmt.Errorf("failed to query %v: http status %v", req.URL.String(), resp.StatusCode) } // parse the response body for a cursor respCursor.Cursor = "" err = json.Unmarshal(body, &respCursor) if err != nil { - return nil, fmt.Errorf("failed to unmarshal response from %v: %v", req.URL.RequestURI(), err) + return nil, fmt.Errorf("failed to unmarshal response from %v: %v", req.URL.String(), err) } // continue querying until there is no longer a cursor @@ -119,7 +123,7 @@ func getHeliumApiWithCursor(path string, params *map[string]string) ([][]byte, e params = &map[string]string{"cursor": respCursor.Cursor} req, err = createGetRequest(path, *params) if err != nil { - return nil, fmt.Errorf("failed to create query request %v: %v", req.URL.RequestURI(), err) + return nil, fmt.Errorf("failed to create query request %v: %v", req.URL.String(), err) } } else { break diff --git a/heliumapi/types.go b/heliumapi/types.go index 23bc054..2e48596 100644 --- a/heliumapi/types.go +++ b/heliumapi/types.go @@ -1,5 +1,12 @@ package heliumapi +const ( + // WORKAROUND: for some reason, helium api partially breaks when given the timezone information + // in "max_time" and "min_time", even if the example code suggest that it supports it. + // For this reason, we cannot use time.RFC3339 as the time format. + timeFormat = "2006-01-02T15:04:05Z" +) + type AccountResp struct { Data Account `json:"data"` }