helium-blockchain-exporter/exporter.go

package main

import (
	"flag"
	"fmt"
	"log"
	"net/http"
	"os"
	"strconv"
	"strings"
	"sync"
	"time"

	"github.com/helium-blockchain-exporter/heliumapi"
	"github.com/prometheus/client_golang/prometheus"
	"github.com/prometheus/client_golang/prometheus/promhttp"
)

// metricInfo is a metric exported by the helium blockchain exporter
type metricInfo struct {
	Desc *prometheus.Desc
	Type prometheus.ValueType
}

// Exporter collect metrics from the helium blockchain api and exports them as prometheus metrics.
type Exporter struct {
	Accounts  []Account
	StartTime time.Time
}

// NewExporter returns an initialized Exporter
func NewExporter(accountAddress []string) (*Exporter, error) {
	accounts := make([]Account, 0)
	for _, accountAddress := range accountAddress {
		if accountAddress != "" {
			accounts = append(accounts, NewAccount(accountAddress))
		}
	}
	return &Exporter{
		Accounts:  accounts,
		StartTime: time.Now(),
	}, nil
}

// Account represents a helium account
type Account struct {
	Address string
	Tx      AccountTx
}

type AccountTx struct {
	DepositTotal    int
	WithdrawalTotal int
	LastUpdate      time.Time
	UpdateLock      sync.Mutex
}

func NewAccount(address string) Account {
	return Account{
		Address: address,
		Tx: AccountTx{
			DepositTotal:    0,
			WithdrawalTotal: 0,
			LastUpdate:      time.Now(),
		},
	}
}

const (
	namespace = "helium"
)

var (
	// labels
	commonAccountLabels = []string{"account"}
	commonHotspotLabels = append(commonAccountLabels, "hotspot", "hotspot_name")

	// exporter metrics

	// helium oracle metrics
	oraclePrice = metricInfo{
		prometheus.NewDesc(
			prometheus.BuildFQName(namespace, "oracle", "price_usd"),
			"The oracle price of an HNT token in USD.",
			nil, nil,
		),
		prometheus.GaugeValue,
	}

	// helium stats metrics
	statsValidators = metricInfo{
		prometheus.NewDesc(
			prometheus.BuildFQName(namespace, "stats", "validators"),
			"The total number of validators.",
			nil, nil,
		),
		prometheus.GaugeValue,
	}
	statsOuis = metricInfo{
		prometheus.NewDesc(
			prometheus.BuildFQName(namespace, "stats", "ouis"),
			"The total number of organization unique identifiers.",
			nil, nil,
		),
		prometheus.GaugeValue,
	}
	statsHotspotsDataOnly = metricInfo{
		prometheus.NewDesc(
			prometheus.BuildFQName(namespace, "stats", "hotspots_dataonly"),
			"The total number of data only hotspots.",
			nil, nil,
		),
		prometheus.GaugeValue,
	}
	statsBlocks = metricInfo{
		prometheus.NewDesc(
			prometheus.BuildFQName(namespace, "stats", "blocks"),
			"The total height/number of blocks in the blockchain.",
			nil, nil,
		),
		prometheus.GaugeValue,
	}
	statsChallenges = metricInfo{
		prometheus.NewDesc(
			prometheus.BuildFQName(namespace, "stats", "challenges"),
			"The total number of challenges.",
			nil, nil,
		),
		prometheus.GaugeValue,
	}
	statsCities = metricInfo{
		prometheus.NewDesc(
			prometheus.BuildFQName(namespace, "stats", "cities"),
			"The number of cities with at least one helium hotspot.",
			nil, nil,
		),
		prometheus.GaugeValue,
	}
	statsConsensusGroups = metricInfo{
		prometheus.NewDesc(
			prometheus.BuildFQName(namespace, "stats", "consensus_groups"),
			"The total number of consensus groups.",
			nil, nil,
		),
		prometheus.GaugeValue,
	}
	statsCountries = metricInfo{
		prometheus.NewDesc(
			prometheus.BuildFQName(namespace, "stats", "countries"),
			"The number of countries with at least on helium hotspot.",
			nil, nil,
		),
		prometheus.GaugeValue,
	}
	statsHotspots = metricInfo{
		prometheus.NewDesc(
			prometheus.BuildFQName(namespace, "stats", "hotspots"),
			"The total number of hotspots.",
			nil, nil,
		),
		prometheus.GaugeValue,
	}
	statsTokenSupply = metricInfo{
		prometheus.NewDesc(
			prometheus.BuildFQName(namespace, "stats", "token"),
			"The total supply of HNT tokens in circulation.",
			nil, nil,
		),
		prometheus.GaugeValue,
	}

	// helium account metrics
	accountBalanceHnt = metricInfo{
		prometheus.NewDesc(
			prometheus.BuildFQName(namespace, "account", "balance_hnt"),
			"The number of HNT token owned by an account.",
			commonAccountLabels, nil,
		),
		prometheus.GaugeValue,
	}
	accountActivity = metricInfo{
		prometheus.NewDesc(
			prometheus.BuildFQName(namespace, "account", "activity_total"),
			"The total number of time an activity occurred in an account.",
			append(commonAccountLabels, "type"), nil,
		),
		prometheus.CounterValue,
	}
	accountRewardsHnt = metricInfo{
		prometheus.NewDesc(
			prometheus.BuildFQName(namespace, "account", "rewards_hnt_total"),
			"The number of HNT token rewarded to an account.",
			commonAccountLabels, nil,
		),
		prometheus.CounterValue,
	}
	accountDepositsHnt = metricInfo{
		prometheus.NewDesc(
			prometheus.BuildFQName(namespace, "account", "deposits_hnt_total"),
			"The number of HNT tokens deposited to this account.",
			commonAccountLabels, nil,
		),
		prometheus.CounterValue,
	}
	accountWithdrawalsHnt = metricInfo{
		prometheus.NewDesc(
			prometheus.BuildFQName(namespace, "account", "withdrawals_hnt_total"),
			"The number of HNT tokens withdrawn from this account.",
			commonAccountLabels, nil,
		),
		prometheus.CounterValue,
	}

	// helium hotspot metrics
	hotspotUp = metricInfo{
		prometheus.NewDesc(
			prometheus.BuildFQName(namespace, "hotspot", "up"),
			"Whether a hotspot is online.",
			commonHotspotLabels, nil,
		),
		prometheus.GaugeValue,
	}
	hotspotRelayed = metricInfo{
		prometheus.NewDesc(
			prometheus.BuildFQName(namespace, "hotspot", "relayed"),
			"Whether a hotspot is relayed.",
			commonHotspotLabels, nil,
		),
		prometheus.GaugeValue,
	}
	hotspotBlocks = metricInfo{
		prometheus.NewDesc(
			prometheus.BuildFQName(namespace, "hotspot", "blocks_total"),
			"The block height of a hotspot. Check on the hotspot itself for the most recent data.",
			commonHotspotLabels, nil,
		),
		prometheus.CounterValue,
	}
	hotspotRewardsScale = metricInfo{
		prometheus.NewDesc(
			prometheus.BuildFQName(namespace, "hotspot", "rewards_scale"),
			"The reward scale of a hotspot.",
			commonHotspotLabels, nil,
		),
		prometheus.GaugeValue,
	}
	hotspotGeocodeInfo = metricInfo{
		prometheus.NewDesc(
			prometheus.BuildFQName(namespace, "hotspot", "geocode_info"),
			"Information on the location of a hotspot.",
			append(commonHotspotLabels, "lng", "lat", "street", "state", "country", "city"), nil,
		),
		prometheus.GaugeValue,
	}
	hotspotAntennaInfo = metricInfo{
		prometheus.NewDesc(
			prometheus.BuildFQName(namespace, "hotspot", "antenna_info"),
			"Information on the antenna of a hotspot.",
			append(commonHotspotLabels, "gain", "elevation"), nil,
		),
		prometheus.GaugeValue,
	}
	hotspotActivity = metricInfo{
		prometheus.NewDesc(
			prometheus.BuildFQName(namespace, "hotspot", "activity_total"),
			"The total number of time an activity occurred in a hotspot.",
			append(commonHotspotLabels, "type"), nil,
		),
		prometheus.CounterValue,
	}
	hotspotRewardsHnt = metricInfo{
		prometheus.NewDesc(
			prometheus.BuildFQName(namespace, "hotspot", "rewards_hnt_total"),
			"The number of HNT token rewarded to a hotspot.",
			commonHotspotLabels, nil,
		),
		prometheus.CounterValue,
	}
)

func bool2Float64(b bool) float64 {
	if b {
		return 1.0
	}
	return 0.0
}

// Describe describes all the metrics ever exported by the helium blockchain exporter.
// implements prometheus.Collector.
func (e *Exporter) Describe(ch chan<- *prometheus.Desc) {
	ch <- oraclePrice.Desc

	ch <- statsValidators.Desc
	ch <- statsOuis.Desc
	ch <- statsHotspotsDataOnly.Desc
	ch <- statsBlocks.Desc
	ch <- statsChallenges.Desc
	ch <- statsCities.Desc
	ch <- statsConsensusGroups.Desc
	ch <- statsCountries.Desc
	ch <- statsHotspots.Desc
	ch <- statsTokenSupply.Desc

	ch <- accountBalanceHnt.Desc
	ch <- accountActivity.Desc
	ch <- accountRewardsHnt.Desc
	ch <- accountDepositsHnt.Desc
	ch <- accountWithdrawalsHnt.Desc

	ch <- hotspotUp.Desc
	ch <- hotspotRelayed.Desc
	ch <- hotspotBlocks.Desc
	ch <- hotspotRewardsScale.Desc
	ch <- hotspotGeocodeInfo.Desc
	ch <- hotspotAntennaInfo.Desc
	ch <- hotspotActivity.Desc
	ch <- hotspotRewardsHnt.Desc
}

// Collect fetches the data from the helium blockchain api and delivers them as Prometheus metrics.
// implements prometheus.Collector.
func (e *Exporter) Collect(ch chan<- prometheus.Metric) {
	wg := new(sync.WaitGroup)

	wg.Add(2)
	go e.collectOracleMetrics(wg, ch)
	e.collectStatsMetrics(wg, ch)
	for i := range e.Accounts {
		wg.Add(5)
		go e.collectAccountMetrics(wg, ch, &e.Accounts[i])
		go e.collectAccountActivityMetrics(wg, ch, &e.Accounts[i])
		go e.collectAccountRewardsTotalMetrics(wg, ch, &e.Accounts[i])
		go e.collectAccountTransactionsMetrics(wg, ch, &e.Accounts[i])

		go e.collectHotspotMetrics(wg, ch, &e.Accounts[i])
	}
	wg.Wait()
}

// collectOracleMetrics collect metrics in the oracle group from the helium api
func (e *Exporter) collectOracleMetrics(wg *sync.WaitGroup, ch chan<- prometheus.Metric) {
	defer wg.Done()

	currentOraclePrice, err := heliumapi.GetCurrentOraclePrice()
	if err != nil {
		fmt.Println(err)
		return
	}

	ch <- prometheus.MustNewConstMetric(
		oraclePrice.Desc, oraclePrice.Type, float64(currentOraclePrice.Price)/100000000,
	)
}

// collectStatsMetrics collect metrics in the stats group from the helium api
func (e *Exporter) collectStatsMetrics(wg *sync.WaitGroup, ch chan<- prometheus.Metric) {
	defer wg.Done()

	blockchainStats, err := heliumapi.GetBlockchainStats()
	if err != nil {
		fmt.Println(err)
		return
	}

	ch <- prometheus.MustNewConstMetric(
		statsValidators.Desc, statsValidators.Type, float64(blockchainStats.Counts.Validators),
	)
	ch <- prometheus.MustNewConstMetric(
		statsOuis.Desc, statsOuis.Type, float64(blockchainStats.Counts.Ouis),
	)
	ch <- prometheus.MustNewConstMetric(
		statsHotspotsDataOnly.Desc, statsHotspotsDataOnly.Type, float64(blockchainStats.Counts.HotspotsDataonly),
	)
	ch <- prometheus.MustNewConstMetric(
		statsBlocks.Desc, statsBlocks.Type, float64(blockchainStats.Counts.Blocks),
	)
	ch <- prometheus.MustNewConstMetric(
		statsChallenges.Desc, statsChallenges.Type, float64(blockchainStats.Counts.Challenges),
	)
	ch <- prometheus.MustNewConstMetric(
		statsCities.Desc, statsCities.Type, float64(blockchainStats.Counts.Cities),
	)
	ch <- prometheus.MustNewConstMetric(
		statsConsensusGroups.Desc, statsConsensusGroups.Type, float64(blockchainStats.Counts.ConsensusGroups),
	)
	ch <- prometheus.MustNewConstMetric(
		statsCountries.Desc, statsCountries.Type, float64(blockchainStats.Counts.Countries),
	)
	ch <- prometheus.MustNewConstMetric(
		statsHotspots.Desc, statsHotspots.Type, float64(blockchainStats.Counts.Hotspots),
	)
	ch <- prometheus.MustNewConstMetric(
		statsTokenSupply.Desc, statsTokenSupply.Type, blockchainStats.TokenSupply,
	)
}

// collectStatsMetrics collect metrics in the account group from the helium api
func (e *Exporter) collectAccountMetrics(wg *sync.WaitGroup, ch chan<- prometheus.Metric, account *Account) {
	defer wg.Done()

	accountForAddress, err := heliumapi.GetAccountForAddress(account.Address)
	if err != nil {
		fmt.Println(err)
		return
	}

	ch <- prometheus.MustNewConstMetric(
		accountBalanceHnt.Desc, accountBalanceHnt.Type, float64(accountForAddress.Balance),
		account.Address,
	)
}

// collectAccountActivityMetrics collect the total number of activities executed by an account from the helium api
func (e *Exporter) collectAccountActivityMetrics(wg *sync.WaitGroup, ch chan<- prometheus.Metric, account *Account) {
	defer wg.Done()

	accountActivityForAddress, err := heliumapi.GetActivityCountsForAccount(account.Address)
	if err != nil {
		fmt.Println(err)
		return
	}

	for accType, count := range *accountActivityForAddress {
		ch <- prometheus.MustNewConstMetric(
			accountActivity.Desc, accountActivity.Type, float64(count),
			account.Address, accType,
		)
	}
}

// collectAccountRewardsTotalMetrics collect the total rewards accumulated by an account from the helium api
func (e *Exporter) collectAccountRewardsTotalMetrics(wg *sync.WaitGroup, ch chan<- prometheus.Metric, account *Account) {
	defer wg.Done()

	accountRewardTotalsForAddress, err := heliumapi.GetRewardTotalsForAccount(account.Address, &e.StartTime, nil)
	if err != nil {
		fmt.Println(err)
		return
	}

	ch <- prometheus.MustNewConstMetric(
		accountRewardsHnt.Desc, accountRewardsHnt.Type, accountRewardTotalsForAddress.Sum,
		account.Address,
	)
}

// collectAccountTransactionsMetrics collect the total deposited/withdrawn by an account from the helium api
func (e *Exporter) collectAccountTransactionsMetrics(wg *sync.WaitGroup, ch chan<- prometheus.Metric, account *Account) {
	defer wg.Done()

	// we can only ever allow a single instance of the routine doing
	// calculations on the deposited and widthdrawn total
	account.Tx.UpdateLock.Lock()
	defer account.Tx.UpdateLock.Unlock()

	now := time.Now()
	activities, err := heliumapi.GetActivityForAccount(account.Address, []string{}, &account.Tx.LastUpdate, &now)
	if err != nil {
		fmt.Println(err)
		return
	}

	// impl based on https://github.com/helium/hotspot-app/blob/918563fba84d1abf4554a43a4d42bb838d017bd3/src/features/wallet/root/useActivityItem.tsx#L336
	for _, activity := range activities.AddGatewayV1 {
		account.Tx.WithdrawalTotal += activity.StakingFee
	}
	for _, activity := range activities.AssertLocationV1 {
		account.Tx.WithdrawalTotal += activity.StakingFee
	}
	for _, activity := range activities.AssertLocationV2 {
		account.Tx.WithdrawalTotal += activity.StakingFee
	}
	for _, activity := range activities.PaymentV1 {
		if activity.Payer == account.Address {
			account.Tx.WithdrawalTotal += activity.Amount
		} else {
			account.Tx.DepositTotal += activity.Amount
		}
	}
	for _, activity := range activities.PaymentV2 {
		if activity.Payer == account.Address {
			paymentTotal := 0
			for _, payment := range activity.Payments {
				paymentTotal += payment.Amount
			}
			account.Tx.WithdrawalTotal += paymentTotal
		} else {
			for _, payment := range activity.Payments {
				if payment.Payee == account.Address {
					account.Tx.DepositTotal += payment.Amount
				}
			}
		}
	}
	for _, activity := range activities.RewardsV1 {
		for _, reward := range activity.Rewards {
			account.Tx.DepositTotal += reward.Amount
		}
	}
	for _, activity := range activities.RewardsV2 {
		for _, reward := range activity.Rewards {
			account.Tx.DepositTotal += reward.Amount
		}
	}
	for _, activity := range activities.StakeValidatorV1 {
		account.Tx.WithdrawalTotal += activity.Stake
	}
	for _, activity := range activities.TokenBurnV1 {
		account.Tx.WithdrawalTotal += activity.Amount
	}
	for _, activity := range activities.TransferHotspotV1 {
		if activity.Buyer == account.Address {
			account.Tx.WithdrawalTotal += activity.AmountToSeller
		} else {
			account.Tx.DepositTotal += activity.AmountToSeller
		}
	}
	for _, activity := range activities.UnstakeValidatorV1 {
		account.Tx.WithdrawalTotal += activity.StakeAmount
	}
	account.Tx.LastUpdate = now

	ch <- prometheus.MustNewConstMetric(
		accountDepositsHnt.Desc, accountDepositsHnt.Type, float64(account.Tx.DepositTotal),
		account.Address,
	)
	ch <- prometheus.MustNewConstMetric(
		accountWithdrawalsHnt.Desc, accountWithdrawalsHnt.Type, float64(account.Tx.WithdrawalTotal),
		account.Address,
	)
}

// collectStatsMetrics collect metrics of the hotspot of an account from the helium api
func (e *Exporter) collectHotspotMetrics(wg *sync.WaitGroup, ch chan<- prometheus.Metric, account *Account) {
	defer wg.Done()

	hotspotsForAddress, err := heliumapi.GetHotspotsForAccount(account.Address)
	if err != nil {
		fmt.Println(err)
		return
	}

	for _, hotspotData := range *hotspotsForAddress {
		// collect hotspot metric requiring extra queries in a new routine
		wg.Add(2)
		go e.collectHotspotActivityMetrics(wg, ch, account, hotspotData)
		go e.collectHotspotRewardsMetrics(wg, ch, account, hotspotData)

		ch <- prometheus.MustNewConstMetric(
			hotspotUp.Desc, hotspotUp.Type, bool2Float64(hotspotData.Status.Online == "online"),
			account.Address, hotspotData.Address, hotspotData.Name,
		)
		ch <- prometheus.MustNewConstMetric(
			hotspotRelayed.Desc, hotspotRelayed.Type, bool2Float64(len(hotspotData.Status.ListenAddrs) > 0 && strings.HasPrefix(hotspotData.Status.ListenAddrs[0], "/p2p")),
			account.Address, hotspotData.Address, hotspotData.Name,
		)
		ch <- prometheus.MustNewConstMetric(
			hotspotBlocks.Desc, hotspotBlocks.Type, float64(hotspotData.Status.Height),
			account.Address, hotspotData.Address, hotspotData.Name,
		)
		ch <- prometheus.MustNewConstMetric(
			hotspotRewardsScale.Desc, hotspotRewardsScale.Type, float64(hotspotData.RewardScale),
			account.Address, hotspotData.Address, hotspotData.Name,
		)
		ch <- prometheus.MustNewConstMetric(
			hotspotGeocodeInfo.Desc, hotspotGeocodeInfo.Type, 1.0,
			account.Address, hotspotData.Address, hotspotData.Name, strconv.FormatFloat(hotspotData.Lng, 'f', 6, 64), strconv.FormatFloat(hotspotData.Lat, 'f', 6, 64), hotspotData.Geocode.LongStreet, hotspotData.Geocode.LongState, hotspotData.Geocode.LongCountry, hotspotData.Geocode.LongCity,
		)
		ch <- prometheus.MustNewConstMetric(
			hotspotAntennaInfo.Desc, hotspotAntennaInfo.Type, 1.0,
			account.Address, hotspotData.Address, hotspotData.Name, strconv.Itoa(hotspotData.Gain), strconv.Itoa(hotspotData.Elevation),
		)
	}
}

// collectHotspotActivityMetrics collect the total number of activities executed by a hotspot from the helium api
func (e *Exporter) collectHotspotActivityMetrics(wg *sync.WaitGroup, ch chan<- prometheus.Metric, account *Account, hotspotData heliumapi.AccountHotspot) {
	defer wg.Done()

	hotspotActivityForAddress, err := heliumapi.GetHotspotActivityCount(hotspotData.Address)
	if err != nil {
		fmt.Println(err)
		return
	}

	for accType, count := range *hotspotActivityForAddress {
		ch <- prometheus.MustNewConstMetric(
			hotspotActivity.Desc, hotspotActivity.Type, float64(count),
			account.Address, hotspotData.Address, hotspotData.Name, accType,
		)
	}
}

// collectHotspotRewardsMetrics collect the total rewards accumulated by a hotspot from the helium api
func (e *Exporter) collectHotspotRewardsMetrics(wg *sync.WaitGroup, ch chan<- prometheus.Metric, account *Account, hotspotData heliumapi.AccountHotspot) {
	defer wg.Done()

	hotspotRewardTotalsForAddress, err := heliumapi.GetRewardsTotalForHotspot(hotspotData.Address, &e.StartTime, nil)
	if err != nil {
		fmt.Println(err)
		return
	}

	ch <- prometheus.MustNewConstMetric(
		hotspotRewardsHnt.Desc, hotspotRewardsHnt.Type, hotspotRewardTotalsForAddress.Sum,
		account.Address, hotspotData.Address, hotspotData.Name,
	)
}

func main() {
	fApiUrl := flag.String("apiUrl", "https://api.helium.io", "The helium api url")
	fHeliumAccounts := flag.String("accounts", "", "A comma-delimited list of helium accounts to scrape (optional)")
	fMetricsPath := flag.String("metricpath", "/metrics", "The metrics path")
	fListenAddress := flag.String("listenAddress", "0.0.0.0", "The http server listen address")
	fListenPort := flag.String("listenPort", "9865", "The http server listen port")
	flag.Parse()

	heliumapi.ApiUrl = *fApiUrl
	heliumAccounts := strings.Split(*fHeliumAccounts, ",")
	serverAddr := *fListenAddress + ":" + *fListenPort

	e, err := NewExporter(heliumAccounts)
	if err != nil {
		log.Fatalf("failed to start exporter: %s", err.Error())
	}

	r := prometheus.NewRegistry()
	r.MustRegister(e)

	// setup http route
	http.HandleFunc("/", func(w http.ResponseWriter, r *http.Request) {
		_, _ = w.Write([]byte(`<html>
             <head><title>Helium blockchain exporter</title></head>
             <body>
             <h1>Helium blockchain exporter</h1>
             <p><a href='` + *fMetricsPath + `'>Metrics</a></p>
             </body>
             </html>`))
	})

	http.Handle(*fMetricsPath, promhttp.HandlerFor(r, promhttp.HandlerOpts{}))
	fmt.Printf("listening on %v\n", serverAddr)
	if err = http.ListenAndServe(serverAddr, nil); err != nil {
		fmt.Println(err)
		os.Exit(1)
	}
}