badjware
/
helium-blockchain-exporter
Archived
1
0
Fork 0
Code Releases Activity

expose hotspot block gap
continuous-integration/drone/push Build is failing Details

This commit is contained in:
Massaki Archambault 2021-10-12 09:19:58 -04:00
parent 5b843a7948
commit fd43e3751f
7 changed files with 85 additions and 57 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

69
exporter.go
View File

@ -3,7 +3,6 @@ package main
import (
"flag"
"log"
"math"
"net/http"
"os"
"strconv"
@ -117,7 +116,7 @@ var (
statsBlocks = metricInfo{
prometheus.NewDesc(
prometheus.BuildFQName(namespace, "stats", "blocks"),
"The total height/number of blocks in the blockchain.",
"The height of the blockchain.",
nil, nil,
),
prometheus.GaugeValue,
@ -199,7 +198,7 @@ var (
accountDepositsHnt = metricInfo{
prometheus.NewDesc(
prometheus.BuildFQName(namespace, "account", "deposits_hnt_total"),
"The number of HNT tokens deposited to this account.",
"The number of HNT tokens deposited to an account.",
commonAccountLabels, nil,
),
prometheus.CounterValue,
@ -207,7 +206,7 @@ var (
accountWithdrawalsHnt = metricInfo{
prometheus.NewDesc(
prometheus.BuildFQName(namespace, "account", "withdrawals_hnt_total"),
"The number of HNT tokens withdrawn from this account.",
"The number of HNT tokens withdrawn from an account.",
commonAccountLabels, nil,
),
prometheus.CounterValue,
@ -222,14 +221,6 @@ var (
),
prometheus.GaugeValue,
}
hotspotSynced = metricInfo{
prometheus.NewDesc(
prometheus.BuildFQName(namespace, "hotspot", "synced"),
"Whether a hotspot is synced with the blockchain.",
commonHotspotLabels, nil,
),
prometheus.GaugeValue,
}
hotspotRelayed = metricInfo{
prometheus.NewDesc(
prometheus.BuildFQName(namespace, "hotspot", "relayed"),
@ -240,11 +231,27 @@ var (
}
hotspotBlocks = metricInfo{
prometheus.NewDesc(
prometheus.BuildFQName(namespace, "hotspot", "blocks_total"),
prometheus.BuildFQName(namespace, "hotspot", "blocks"),
"The block height of a hotspot. Check on the hotspot itself for the most recent data.",
commonHotspotLabels, nil,
),
prometheus.CounterValue,
prometheus.GaugeValue,
}
hotspotBlocksDelta = metricInfo{
prometheus.NewDesc(
prometheus.BuildFQName(namespace, "hotspot", "blocks_delta"),
"The gap between the height of a hotspot and the height of the blockchain. A large negative gap may indicate the hotspot is out of sync. Check on the hotspot itself for the most recent data.",
commonHotspotLabels, nil,
),
prometheus.GaugeValue,
}
hotspotStatusTimestamp = metricInfo{
prometheus.NewDesc(
prometheus.BuildFQName(namespace, "hotspot", "status_timestamp"),
"The the last time a hotspot gossiped its status on the blockchain.",
commonHotspotLabels, nil,
),
prometheus.GaugeValue,
}
hotspotRewardsScale = metricInfo{
prometheus.NewDesc(
@ -334,9 +341,10 @@ func (e *Exporter) Describe(ch chan<- *prometheus.Desc) {
ch <- accountWithdrawalsHnt.Desc
ch <- hotspotUp.Desc
ch <- hotspotSynced.Desc
ch <- hotspotRelayed.Desc
ch <- hotspotBlocksDelta.Desc
ch <- hotspotBlocks.Desc
ch <- hotspotStatusTimestamp.Desc
ch <- hotspotRewardsScale.Desc
ch <- hotspot5dWitnesses.Desc
ch <- hotspot5dWitnessed.Desc
@ -607,10 +615,16 @@ func (e *Exporter) collectHotspotMetrics(wg *sync.WaitGroup, ch chan<- prometheu
return
}
for _, hotspotData := range *hotspotsForAddress {
for _, hotspotData := range hotspotsForAddress {
statusTime, err := time.Parse(time.RFC3339Nano, hotspotData.Status.Timestamp)
if err != nil {
log.Println(err)
continue
}
// collect hotspot metric requiring extra queries in a new routine
wg.Add(5)
go e.collectHotspotSyncedMetrics(wg, ch, account, hotspotData)
go e.collectHotspotBlocksDeltaMetrics(wg, ch, account, hotspotData)
go e.collectHotspotWitnessesMetrics(wg, ch, account, hotspotData)
go e.collectHotspotWitnessedMetrics(wg, ch, account, hotspotData)
go e.collectHotspotActivityMetrics(wg, ch, account, hotspotData)
@ -628,6 +642,10 @@ func (e *Exporter) collectHotspotMetrics(wg *sync.WaitGroup, ch chan<- prometheu
hotspotBlocks.Desc, hotspotBlocks.Type, float64(hotspotData.Status.Height),
account.Address, hotspotData.Address, hotspotData.Name,
)
ch <- prometheus.MustNewConstMetric(
hotspotStatusTimestamp.Desc, hotspotStatusTimestamp.Type, float64(statusTime.Unix()),
account.Address, hotspotData.Address, hotspotData.Name,
)
ch <- prometheus.MustNewConstMetric(
hotspotRewardsScale.Desc, hotspotRewardsScale.Type, float64(hotspotData.RewardScale),
account.Address, hotspotData.Address, hotspotData.Name,
@ -643,11 +661,11 @@ func (e *Exporter) collectHotspotMetrics(wg *sync.WaitGroup, ch chan<- prometheu
}
}
// collectSyncedMetrics calculate if the hotspot is in sync with the blockchain
func (e *Exporter) collectHotspotSyncedMetrics(wg *sync.WaitGroup, ch chan<- prometheus.Metric, account *Account, hotspotData heliumapi.Hotspot) {
// collectHotspotBlocksDeltaMetrics calculate the gap between the block height of a hotstop and the height of the chain
func (e *Exporter) collectHotspotBlocksDeltaMetrics(wg *sync.WaitGroup, ch chan<- prometheus.Metric, account *Account, hotspotData heliumapi.Hotspot) {
defer wg.Done()
isSynced := 0.
delta := 0.
if hotspotData.Status.Online == "online" && hotspotData.Status.Timestamp != "" {
statusTime, err := time.Parse(time.RFC3339Nano, hotspotData.Status.Timestamp)
if err != nil {
@ -659,14 +677,11 @@ func (e *Exporter) collectHotspotSyncedMetrics(wg *sync.WaitGroup, ch chan<- pro
log.Println(err)
return
}
// if the block height and the height reported by the hotspot is within 10 blocks, we consider it in sync
if math.Abs(float64(hotspotData.Status.Height-heightAtUpdate)) <= 10 {
isSynced = 1.
}
delta = float64(hotspotData.Status.Height - heightAtUpdate)
}
ch <- prometheus.MustNewConstMetric(
hotspotSynced.Desc, hotspotActivity.Type, isSynced,
hotspotBlocksDelta.Desc, hotspotBlocksDelta.Type, delta,
account.Address, hotspotData.Address, hotspotData.Name,
)
}
@ -682,7 +697,7 @@ func (e *Exporter) collectHotspotWitnessesMetrics(wg *sync.WaitGroup, ch chan<-
}
ch <- prometheus.MustNewConstMetric(
hotspot5dWitnesses.Desc, hotspotActivity.Type, float64(len(*hotspotWitnesses)),
hotspot5dWitnesses.Desc, hotspotActivity.Type, float64(len(hotspotWitnesses)),
account.Address, hotspotData.Address, hotspotData.Name,
)
}
@ -698,7 +713,7 @@ func (e *Exporter) collectHotspotWitnessedMetrics(wg *sync.WaitGroup, ch chan<-
}
ch <- prometheus.MustNewConstMetric(
hotspot5dWitnessed.Desc, hotspotActivity.Type, float64(len(*hotspotWitnessed)),
hotspot5dWitnessed.Desc, hotspotActivity.Type, float64(len(hotspotWitnessed)),
account.Address, hotspotData.Address, hotspotData.Name,
)
}

17
heliumapi/accounts.go
View File

@ -32,14 +32,15 @@ func GetAccountForAddress(account string) (*Account, error) {
// query https://docs.helium.com/api/blockchain/accounts#activity-for-account
func GetActivityForAccount(account string, filterTypes []string, minTime *time.Time, maxTime *time.Time) (*activity.Activities, error) {
path := "/v1/accounts/" + account + "/activity"
params := map[string]string{
"filter_types": strings.Join(filterTypes, ","),
params := map[string]string{}
if filterTypes != nil {
params["min_time"] = strings.Join(filterTypes, ",")
}
if minTime != nil {
params["min_time"] = minTime.UTC().Format("2006-01-02T15:04:05Z")
params["min_time"] = minTime.UTC().Format(timeFormat)
}
if maxTime != nil {
params["max_time"] = maxTime.UTC().Format("2006-01-02T15:04:05Z")
params["max_time"] = maxTime.UTC().Format(timeFormat)
}
// query the api
@ -87,10 +88,10 @@ func GetRewardTotalsForAccount(account string, minTime *time.Time, maxTime *time
path := "/v1/accounts/" + account + "/rewards/sum"
params := map[string]string{}
if minTime != nil {
params["min_time"] = minTime.UTC().Format("2006-01-02T15:04:05Z")
params["min_time"] = minTime.UTC().Format(timeFormat)
}
if maxTime != nil {
params["max_time"] = maxTime.UTC().Format("2006-01-02T15:04:05Z")
params["max_time"] = maxTime.UTC().Format(timeFormat)
}
// query the api
@ -110,7 +111,7 @@ func GetRewardTotalsForAccount(account string, minTime *time.Time, maxTime *time
}
// query https://docs.helium.com/api/blockchain/accounts#hotspots-for-account
func GetHotspotsForAccount(account string) (*[]Hotspot, error) {
func GetHotspotsForAccount(account string) ([]Hotspot, error) {
path := "/v1/accounts/" + account + "/hotspots"
// query the api
@ -126,5 +127,5 @@ func GetHotspotsForAccount(account string) (*[]Hotspot, error) {
return nil, fmt.Errorf("failed to unmarshal response from %v: %v", path, err)
}
return &respobject.Data, nil
return respobject.Data, nil
}

12
heliumapi/activity/activities.go
View File

@ -59,17 +59,17 @@ func NewActivities(resp ActivityResp) (*Activities, error) {
}
activities.PaymentV2 = append(activities.PaymentV2, paymentV2)
case "rewards_v1":
rewardV1 := RewardsV1{}
if err := json.Unmarshal(activityRaw, &rewardV1); err != nil {
rewardsV1 := RewardsV1{}
if err := json.Unmarshal(activityRaw, &rewardsV1); err != nil {
return nil, fmt.Errorf("failed to unmarshal %v: %v", activityType.Type, err)
}
activities.RewardsV1 = append(activities.RewardsV1, rewardV1)
activities.RewardsV1 = append(activities.RewardsV1, rewardsV1)
case "rewards_v2":
rewardV2 := RewardsV2{}
if err := json.Unmarshal(activityRaw, &rewardV2); err != nil {
rewardsV2 := RewardsV2{}
if err := json.Unmarshal(activityRaw, &rewardsV2); err != nil {
return nil, fmt.Errorf("failed to unmarshal %v: %v", activityType.Type, err)
}
activities.RewardsV2 = append(activities.RewardsV2, rewardV2)
activities.RewardsV2 = append(activities.RewardsV2, rewardsV2)
case "stake_validator_v1":
stakeValidatorV1 := StakeValidatorV1{}
if err := json.Unmarshal(activityRaw, &stakeValidatorV1); err != nil {

3
heliumapi/block.go
View File

@ -8,10 +8,11 @@ import (
// query https://docs.helium.com/api/blockchain/blocks/#height
func GetHeight(maxTime *time.Time) (int, error) {
// TODO: the result of this query could be cached
path := "/v1/blocks/height"
params := map[string]string{}
if maxTime != nil {
params["max_time"] = maxTime.UTC().Format("2006-01-02T15:04:05Z")
params["max_time"] = maxTime.UTC().Format(timeFormat)
}
// query the api

12
heliumapi/hotspots.go
View File

@ -31,10 +31,10 @@ func GetRewardsTotalForHotspot(hotspot string, minTime *time.Time, maxTime *time
path := "/v1/hotspots/" + hotspot + "/rewards/sum"
params := map[string]string{}
if minTime != nil {
params["min_time"] = minTime.UTC().Format("2006-01-02T15:04:05Z")
params["min_time"] = minTime.UTC().Format(timeFormat)
}
if maxTime != nil {
params["max_time"] = maxTime.UTC().Format("2006-01-02T15:04:05Z")
params["max_time"] = maxTime.UTC().Format(timeFormat)
}
// query the api
@ -54,7 +54,7 @@ func GetRewardsTotalForHotspot(hotspot string, minTime *time.Time, maxTime *time
}
// query https://docs.helium.com/api/blockchain/hotspots#witnesses-for-a-hotspot
func GetWitnessesForHotspot(hotspot string) (*[]Hotspot, error) {
func GetWitnessesForHotspot(hotspot string) ([]Hotspot, error) {
path := "/v1/hotspots/" + hotspot + "/witnesses"
// query the api
@ -70,11 +70,11 @@ func GetWitnessesForHotspot(hotspot string) (*[]Hotspot, error) {
return nil, fmt.Errorf("failed to unmarshal response from %v: %v", path, err)
}
return &respobject.Data, nil
return respobject.Data, nil
}
// query https://docs.helium.com/api/blockchain/hotspots#witnessed-for-a-hotspot
func GetWitnessedForHotspot(hotspot string) (*[]Hotspot, error) {
func GetWitnessedForHotspot(hotspot string) ([]Hotspot, error) {
path := "/v1/hotspots/" + hotspot + "/witnessed"
// query the api
@ -90,5 +90,5 @@ func GetWitnessedForHotspot(hotspot string) (*[]Hotspot, error) {
return nil, fmt.Errorf("failed to unmarshal response from %v: %v", path, err)
}
return &respobject.Data, nil
return respobject.Data, nil
}

22
heliumapi/query.go
View File

@ -4,7 +4,9 @@ import (
"encoding/json"
"fmt"
"io/ioutil"
"log"
"net/http"
"strings"
)
var (
@ -14,7 +16,7 @@ var (
// createGetRequest create a GET request to the helium api
func createGetRequest(path string, params map[string]string) (*http.Request, error) {
req, err := http.NewRequest("GET", fmt.Sprintf("%s/%s", ApiUrl, path), nil)
req, err := http.NewRequest("GET", fmt.Sprintf("%s%s", ApiUrl, path), nil)
// setup headers
req.Header.Add("Accept", "application/json")
@ -47,21 +49,22 @@ func getHeliumApi(path string, params *map[string]string) ([]byte, error) {
}
// query the api
log.Printf("querying %v", req.URL.String())
resp, err := client.Do(req)
if err != nil {
return nil, fmt.Errorf("failed to query %v: %v", req.URL.RequestURI(), err)
return nil, fmt.Errorf("failed to query %v: %v", req.URL.String(), err)
}
defer resp.Body.Close()
// validate the response status
if resp.StatusCode != 200 {
return nil, fmt.Errorf("failed to query %v: http status %v", req.URL.RequestURI(), resp.StatusCode)
return nil, fmt.Errorf("failed to query %v: http status %v", req.URL.String(), resp.StatusCode)
}
// read the response body
body, err := ioutil.ReadAll(resp.Body)
if err != nil {
return nil, fmt.Errorf("failed to read response body of %v: %v", req.URL.RequestURI(), err)
return nil, fmt.Errorf("failed to read response body of %v: %v", req.URL.String(), err)
}
return body, nil
@ -91,27 +94,28 @@ func getHeliumApiWithCursor(path string, params *map[string]string) ([][]byte, e
// query the api
resp, err := client.Do(req)
if err != nil {
return nil, fmt.Errorf("failed to query %v: %v", req.URL.RequestURI(), err)
return nil, fmt.Errorf("failed to query %v: %v", req.URL.String(), err)
}
defer resp.Body.Close()
// read the response body and add it to the result array
log.Printf("querying %v", req.URL.String())
body, err := ioutil.ReadAll(resp.Body)
if err != nil {
return nil, fmt.Errorf("failed to read response body of %v: %v", req.URL.RequestURI(), err)
return nil, fmt.Errorf("failed to read response body of %v: %v", req.URL.String(), err)
}
res = append(res, body)
// validate the response status
if resp.StatusCode != 200 {
return nil, fmt.Errorf("failed to query %v: http status %v", req.URL.RequestURI(), resp.StatusCode)
return nil, fmt.Errorf("failed to query %v: http status %v", req.URL.String(), resp.StatusCode)
}
// parse the response body for a cursor
respCursor.Cursor = ""
err = json.Unmarshal(body, &respCursor)
if err != nil {
return nil, fmt.Errorf("failed to unmarshal response from %v: %v", req.URL.RequestURI(), err)
return nil, fmt.Errorf("failed to unmarshal response from %v: %v", req.URL.String(), err)
}
// continue querying until there is no longer a cursor
@ -119,7 +123,7 @@ func getHeliumApiWithCursor(path string, params *map[string]string) ([][]byte, e
params = &map[string]string{"cursor": respCursor.Cursor}
req, err = createGetRequest(path, *params)
if err != nil {
return nil, fmt.Errorf("failed to create query request %v: %v", req.URL.RequestURI(), err)
return nil, fmt.Errorf("failed to create query request %v: %v", req.URL.String(), err)
}
} else {
break

7
heliumapi/types.go
View File

@ -1,5 +1,12 @@
package heliumapi
const (
// WORKAROUND: for some reason, helium api partially breaks when given the timezone information
// in "max_time" and "min_time", even if the example code suggest that it supports it.
// For this reason, we cannot use time.RFC3339 as the time format.
timeFormat = "2006-01-02T15:04:05Z"
)
type AccountResp struct {
Data Account `json:"data"`
}