badjware
/
helium-blockchain-exporter
Archived
1
0
Fork 0
Code Releases Activity

make each new query to the helium api be in its own goroutine

This commit is contained in:
Massaki Archambault 2021-09-25 23:38:16 -04:00
parent b4598ce495
commit 20eddb8fc3
6 changed files with 317 additions and 257 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

296
exporter.go
View File

@ -7,6 +7,7 @@ import (
"net/http"
"strconv"
"strings"
"sync"
"time"
"github.com/helium-blockchain-exporter/heliumapi"
@ -46,6 +47,13 @@ type Account struct {
Tx AccountTx
}
type AccountTx struct {
DepositTotal int
WithdrawalTotal int
LastUpdate time.Time
UpdateLock sync.Mutex
}
func NewAccount(address string) Account {
return Account{
Address: address,
@ -57,12 +65,6 @@ func NewAccount(address string) Account {
}
}
type AccountTx struct {
DepositTotal int
WithdrawalTotal int
LastUpdate time.Time
}
const (
namespace = "helium"
)
@ -317,16 +319,27 @@ func (e *Exporter) Describe(ch chan<- *prometheus.Desc) {
// Collect fetches the data from the helium blockchain api and delivers them as Prometheus metrics.
// implements prometheus.Collector.
func (e *Exporter) Collect(ch chan<- prometheus.Metric) {
e.collectOracleMetrics(ch)
e.collectStatsMetrics(ch)
wg := new(sync.WaitGroup)
wg.Add(2)
go e.collectOracleMetrics(wg, ch)
e.collectStatsMetrics(wg, ch)
for i := range e.Accounts {
e.collectAccountMetrics(ch, &e.Accounts[i])
e.collectHotspotMetrics(ch, &e.Accounts[i])
wg.Add(5)
go e.collectAccountMetrics(wg, ch, &e.Accounts[i])
go e.collectAccountActivityMetrics(wg, ch, &e.Accounts[i])
go e.collectAccountRewardsTotalMetrics(wg, ch, &e.Accounts[i])
go e.collectAccountTransactionsMetrics(wg, ch, &e.Accounts[i])
go e.collectHotspotMetrics(wg, ch, &e.Accounts[i])
}
wg.Wait()
}
// collectOracleMetrics collect metrics in the oracle group from the helium api
func (e *Exporter) collectOracleMetrics(ch chan<- prometheus.Metric) {
func (e *Exporter) collectOracleMetrics(wg *sync.WaitGroup, ch chan<- prometheus.Metric) {
defer wg.Done()
currentOraclePrice, err := heliumapi.GetCurrentOraclePrice()
if err != nil {
fmt.Println(err)
@ -339,7 +352,9 @@ func (e *Exporter) collectOracleMetrics(ch chan<- prometheus.Metric) {
}
// collectStatsMetrics collect metrics in the stats group from the helium api
func (e *Exporter) collectStatsMetrics(ch chan<- prometheus.Metric) {
func (e *Exporter) collectStatsMetrics(wg *sync.WaitGroup, ch chan<- prometheus.Metric) {
defer wg.Done()
blockchainStats, err := heliumapi.GetBlockchainStats()
if err != nil {
fmt.Println(err)
@ -379,48 +394,145 @@ func (e *Exporter) collectStatsMetrics(ch chan<- prometheus.Metric) {
}
// collectStatsMetrics collect metrics in the account group from the helium api
func (e *Exporter) collectAccountMetrics(ch chan<- prometheus.Metric, account *Account) {
func (e *Exporter) collectAccountMetrics(wg *sync.WaitGroup, ch chan<- prometheus.Metric, account *Account) {
defer wg.Done()
accountForAddress, err := heliumapi.GetAccountForAddress(account.Address)
if err != nil {
fmt.Println(err)
return
}
accountActivityForAddress, err := heliumapi.GetActivityCountsForAccount(account.Address)
if err != nil {
fmt.Println(err)
return
}
accountRewardTotalsForAddress, err := heliumapi.GetRewardTotalsForAccount(account.Address, &e.StartTime, nil)
if err != nil {
fmt.Println(err)
return
}
err = account.collectTransactionMetrics(ch)
if err != nil {
fmt.Println(err)
return
}
ch <- prometheus.MustNewConstMetric(
accountBalanceHnt.Desc, accountBalanceHnt.Type, float64(accountForAddress.Data.Balance),
account.Address,
)
}
// collectAccountActivityMetrics collect the total number of activities executed by an account from the helium api
func (e *Exporter) collectAccountActivityMetrics(wg *sync.WaitGroup, ch chan<- prometheus.Metric, account *Account) {
defer wg.Done()
accountActivityForAddress, err := heliumapi.GetActivityCountsForAccount(account.Address)
if err != nil {
fmt.Println(err)
return
}
for accType, count := range accountActivityForAddress.Data {
ch <- prometheus.MustNewConstMetric(
accountActivity.Desc, accountActivity.Type, float64(count),
account.Address, accType,
)
}
}
// collectAccountRewardsTotalMetrics collect the total rewards accumulated by an account from the helium api
func (e *Exporter) collectAccountRewardsTotalMetrics(wg *sync.WaitGroup, ch chan<- prometheus.Metric, account *Account) {
defer wg.Done()
accountRewardTotalsForAddress, err := heliumapi.GetRewardTotalsForAccount(account.Address, &e.StartTime, nil)
if err != nil {
fmt.Println(err)
return
}
ch <- prometheus.MustNewConstMetric(
accountRewardsHnt.Desc, accountRewardsHnt.Type, accountRewardTotalsForAddress.Data.Sum,
account.Address,
)
}
// collectStatsMetrics collect metrics in the hotspot group from the helium api
func (e *Exporter) collectHotspotMetrics(ch chan<- prometheus.Metric, account *Account) {
// collectAccountTransactionsMetrics collect the total deposited/withdrawn by an account from the helium api
func (e *Exporter) collectAccountTransactionsMetrics(wg *sync.WaitGroup, ch chan<- prometheus.Metric, account *Account) {
defer wg.Done()
// we can only ever allow a single instance of the routine doing
// calculations on the deposited and widthdrawn total
account.Tx.UpdateLock.Lock()
defer account.Tx.UpdateLock.Unlock()
now := time.Now()
activities, err := heliumapi.GetActivityForAccount(account.Address, []string{}, &account.Tx.LastUpdate, &now)
if err != nil {
fmt.Println(err)
return
}
// logic based on https://github.com/helium/hotspot-app/blob/918563fba84d1abf4554a43a4d42bb838d017bd3/src/features/wallet/root/useActivityItem.tsx#L336
for _, activity := range activities.AddGatewayV1 {
account.Tx.WithdrawalTotal += activity.StakingFee
}
for _, activity := range activities.AssertLocationV1 {
account.Tx.WithdrawalTotal += activity.StakingFee
}
for _, activity := range activities.AssertLocationV2 {
account.Tx.WithdrawalTotal += activity.StakingFee
}
for _, activity := range activities.PaymentV1 {
if activity.Payer == account.Address {
account.Tx.WithdrawalTotal += activity.Amount
} else {
account.Tx.DepositTotal += activity.Amount
}
}
for _, activity := range activities.PaymentV2 {
if activity.Payer == account.Address {
paymentTotal := 0
for _, payment := range activity.Payments {
paymentTotal += payment.Amount
}
account.Tx.WithdrawalTotal += paymentTotal
} else {
for _, payment := range activity.Payments {
if payment.Payee == account.Address {
account.Tx.DepositTotal += payment.Amount
}
}
}
}
for _, activity := range activities.RewardsV1 {
for _, reward := range activity.Rewards {
account.Tx.DepositTotal += reward.Amount
}
}
for _, activity := range activities.RewardsV2 {
for _, reward := range activity.Rewards {
account.Tx.DepositTotal += reward.Amount
}
}
for _, activity := range activities.StakeValidatorV1 {
account.Tx.WithdrawalTotal += activity.Stake
}
for _, activity := range activities.TokenBurnV1 {
account.Tx.WithdrawalTotal += activity.Amount
}
for _, activity := range activities.TransferHotspotV1 {
if activity.Buyer == account.Address {
account.Tx.WithdrawalTotal += activity.AmountToSeller
} else {
account.Tx.DepositTotal += activity.AmountToSeller
}
}
for _, activity := range activities.UnstakeValidatorV1 {
account.Tx.WithdrawalTotal += activity.StakeAmount
}
account.Tx.LastUpdate = now
ch <- prometheus.MustNewConstMetric(
accountDepositsHnt.Desc, accountDepositsHnt.Type, float64(account.Tx.DepositTotal),
account.Address,
)
ch <- prometheus.MustNewConstMetric(
accountWithdrawalsHnt.Desc, accountWithdrawalsHnt.Type, float64(account.Tx.WithdrawalTotal),
account.Address,
)
}
// collectStatsMetrics collect metrics of the hotspot of an account from the helium api
func (e *Exporter) collectHotspotMetrics(wg *sync.WaitGroup, ch chan<- prometheus.Metric, account *Account) {
defer wg.Done()
hotspotsForAddress, err := heliumapi.GetHotspotsForAccount(account.Address)
if err != nil {
fmt.Println(err)
@ -428,17 +540,10 @@ func (e *Exporter) collectHotspotMetrics(ch chan<- prometheus.Metric, account *A
}
for _, hotspotData := range hotspotsForAddress.Data {
hotspotActivityForAddress, err := heliumapi.GetHotspotActivityCount(hotspotData.Address)
if err != nil {
fmt.Println(err)
return
}
hotspotRewardTotalsForAddress, err := heliumapi.GetRewardsTotalForHotspot(hotspotData.Address, &e.StartTime, nil)
if err != nil {
fmt.Println(err)
return
}
// collect hotspot metric requiring extra queries in a new routine
wg.Add(2)
go e.collectHotspotActivityMetrics(wg, ch, account, hotspotData)
go e.collectHotspotRewardsMetrics(wg, ch, account, hotspotData)
ch <- prometheus.MustNewConstMetric(
hotspotUp.Desc, hotspotUp.Type, bool2Float64(hotspotData.Status.Online == "online"),
@ -464,97 +569,42 @@ func (e *Exporter) collectHotspotMetrics(ch chan<- prometheus.Metric, account *A
hotspotAntennaInfo.Desc, hotspotAntennaInfo.Type, 1.0,
account.Address, hotspotData.Address, hotspotData.Name, strconv.Itoa(hotspotData.Gain), strconv.Itoa(hotspotData.Elevation),
)
}
}
// collectHotspotActivityMetrics collect the total number of activities executed by a hotspot from the helium api
func (e *Exporter) collectHotspotActivityMetrics(wg *sync.WaitGroup, ch chan<- prometheus.Metric, account *Account, hotspotData heliumapi.AccountHotspot) {
defer wg.Done()
hotspotActivityForAddress, err := heliumapi.GetHotspotActivityCount(hotspotData.Address)
if err != nil {
fmt.Println(err)
return
}
for accType, count := range hotspotActivityForAddress.Data {
ch <- prometheus.MustNewConstMetric(
hotspotActivity.Desc, hotspotActivity.Type, float64(count),
account.Address, hotspotData.Address, hotspotData.Name, accType,
)
}
}
// collectHotspotRewardsMetrics collect the total rewards accumulated by a hotspot from the helium api
func (e *Exporter) collectHotspotRewardsMetrics(wg *sync.WaitGroup, ch chan<- prometheus.Metric, account *Account, hotspotData heliumapi.AccountHotspot) {
defer wg.Done()
hotspotRewardTotalsForAddress, err := heliumapi.GetRewardsTotalForHotspot(hotspotData.Address, &e.StartTime, nil)
if err != nil {
fmt.Println(err)
return
}
ch <- prometheus.MustNewConstMetric(
hotspotRewardsHnt.Desc, hotspotRewardsHnt.Type, hotspotRewardTotalsForAddress.Data.Sum,
account.Address, hotspotData.Address, hotspotData.Name,
)
}
}
func (a *Account) collectTransactionMetrics(ch chan<- prometheus.Metric) error {
now := time.Now()
activities, err := heliumapi.GetActivityForAccount(a.Address, []string{}, &a.Tx.LastUpdate, &now)
if err != nil {
return err
}
// logic based on https://github.com/helium/hotspot-app/blob/918563fba84d1abf4554a43a4d42bb838d017bd3/src/features/wallet/root/useActivityItem.tsx#L336
for _, activity := range activities.AddGatewayV1 {
a.Tx.WithdrawalTotal += activity.StakingFee
}
for _, activity := range activities.AssertLocationV1 {
a.Tx.WithdrawalTotal += activity.StakingFee
}
for _, activity := range activities.AssertLocationV2 {
a.Tx.WithdrawalTotal += activity.StakingFee
}
for _, activity := range activities.PaymentV1 {
if activity.Payer == a.Address {
a.Tx.WithdrawalTotal += activity.Amount
} else {
a.Tx.DepositTotal += activity.Amount
}
}
for _, activity := range activities.PaymentV2 {
if activity.Payer == a.Address {
paymentTotal := 0
for _, payment := range activity.Payments {
paymentTotal += payment.Amount
}
a.Tx.WithdrawalTotal += paymentTotal
} else {
for _, payment := range activity.Payments {
if payment.Payee == a.Address {
a.Tx.DepositTotal += payment.Amount
}
}
}
}
for _, activity := range activities.RewardsV1 {
for _, reward := range activity.Rewards {
a.Tx.DepositTotal += reward.Amount
}
}
for _, activity := range activities.RewardsV2 {
for _, reward := range activity.Rewards {
a.Tx.DepositTotal += reward.Amount
}
}
for _, activity := range activities.StakeValidatorV1 {
a.Tx.WithdrawalTotal += activity.Stake
}
for _, activity := range activities.TokenBurnV1 {
a.Tx.WithdrawalTotal += activity.Amount
}
for _, activity := range activities.TransferHotspotV1 {
if activity.Buyer == a.Address {
a.Tx.WithdrawalTotal += activity.AmountToSeller
} else {
a.Tx.DepositTotal += activity.AmountToSeller
}
}
for _, activity := range activities.UnstakeValidatorV1 {
a.Tx.WithdrawalTotal += activity.StakeAmount
}
a.Tx.LastUpdate = now
ch <- prometheus.MustNewConstMetric(
accountDepositsHnt.Desc, accountDepositsHnt.Type, float64(a.Tx.DepositTotal),
a.Address,
)
ch <- prometheus.MustNewConstMetric(
accountWithdrawalsHnt.Desc, accountWithdrawalsHnt.Type, float64(a.Tx.WithdrawalTotal),
a.Address,
)
return nil
}
func main() {
fHeliumAccounts := flag.String("accounts", "", "A comma-delimited list of helium accounts to scrape (optional)")

16
heliumapi/accounts.go
View File

@ -8,7 +8,7 @@ import (
"github.com/helium-blockchain-exporter/heliumapi/activity"
)
func GetAccountForAddress(account string) (*Account, error) {
func GetAccountForAddress(account string) (*AccountResp, error) {
path := "/v1/accounts/" + account
// query the api
@ -18,7 +18,7 @@ func GetAccountForAddress(account string) (*Account, error) {
}
// unmarshal the response
respobject := Account{}
respobject := AccountResp{}
err = json.Unmarshal(respBody, &respobject)
if err != nil {
return nil, fmt.Errorf("failed to unmarshal response from path %v: %v", path, err)
@ -57,7 +57,7 @@ func GetActivityForAccount(account string, filterTypes []string, minTime *time.T
return activity.NewActivities(combinedResp)
}
func GetActivityCountsForAccount(account string) (*ActivityCounts, error) {
func GetActivityCountsForAccount(account string) (*ActivityCountsResp, error) {
path := "/v1/accounts/" + account + "/activity/count"
// query the api
@ -67,7 +67,7 @@ func GetActivityCountsForAccount(account string) (*ActivityCounts, error) {
}
// unmarshal the response
respobject := ActivityCounts{}
respobject := ActivityCountsResp{}
err = json.Unmarshal(respBody, &respobject)
if err != nil {
return nil, fmt.Errorf("failed to unmarshal response from path %v: %v", path, err)
@ -76,7 +76,7 @@ func GetActivityCountsForAccount(account string) (*ActivityCounts, error) {
return &respobject, nil
}
func GetRewardTotalsForAccount(account string, minTime *time.Time, maxTime *time.Time) (*RewardTotal, error) {
func GetRewardTotalsForAccount(account string, minTime *time.Time, maxTime *time.Time) (*RewardTotalResp, error) {
path := "/v1/accounts/" + account + "/rewards/sum"
params := map[string]string{}
if minTime != nil {
@ -93,7 +93,7 @@ func GetRewardTotalsForAccount(account string, minTime *time.Time, maxTime *time
}
// unmarshal the response
respobject := RewardTotal{}
respobject := RewardTotalResp{}
err = json.Unmarshal(respBody, &respobject)
if err != nil {
return nil, fmt.Errorf("failed to unmarshal response from path %v: %v", path, err)
@ -102,7 +102,7 @@ func GetRewardTotalsForAccount(account string, minTime *time.Time, maxTime *time
return &respobject, nil
}
func GetHotspotsForAccount(account string) (*AccountHotspots, error) {
func GetHotspotsForAccount(account string) (*AccountHotspotsResp, error) {
path := "/v1/accounts/" + account + "/hotspots"
// query the api
@ -112,7 +112,7 @@ func GetHotspotsForAccount(account string) (*AccountHotspots, error) {
}
// unmarshal the response
respobject := AccountHotspots{}
respobject := AccountHotspotsResp{}
err = json.Unmarshal(respBody, &respobject)
if err != nil {
return nil, fmt.Errorf("failed to unmarshal response from path %v: %v", path, err)

8
heliumapi/hotspots.go
View File

@ -6,7 +6,7 @@ import (
"time"
)
func GetHotspotActivityCount(hotspot string) (*ActivityCounts, error) {
func GetHotspotActivityCount(hotspot string) (*ActivityCountsResp, error) {
path := "/v1/hotspots/" + hotspot + "/activity/count"
// query the api
@ -16,7 +16,7 @@ func GetHotspotActivityCount(hotspot string) (*ActivityCounts, error) {
}
// unmarshal the response
respobject := ActivityCounts{}
respobject := ActivityCountsResp{}
err = json.Unmarshal(respBody, &respobject)
if err != nil {
return nil, fmt.Errorf("failed to unmarshal response from path %v: %v", path, err)
@ -25,7 +25,7 @@ func GetHotspotActivityCount(hotspot string) (*ActivityCounts, error) {
return &respobject, nil
}
func GetRewardsTotalForHotspot(hotspot string, minTime *time.Time, maxTime *time.Time) (*RewardTotal, error) {
func GetRewardsTotalForHotspot(hotspot string, minTime *time.Time, maxTime *time.Time) (*RewardTotalResp, error) {
path := "/v1/hotspots/" + hotspot + "/rewards/sum"
params := map[string]string{}
if minTime != nil {
@ -42,7 +42,7 @@ func GetRewardsTotalForHotspot(hotspot string, minTime *time.Time, maxTime *time
}
// unmarshal the response
respobject := RewardTotal{}
respobject := RewardTotalResp{}
err = json.Unmarshal(respBody, &respobject)
if err != nil {
return nil, fmt.Errorf("failed to unmarshal response from path %v: %v", path, err)

4
heliumapi/oracleprices.go
View File

@ -5,7 +5,7 @@ import (
"fmt"
)
func GetCurrentOraclePrice() (*CurrentOraclePrice, error) {
func GetCurrentOraclePrice() (*CurrentOraclePriceResp, error) {
const path = "/v1/oracle/prices/current"
// query the api
@ -15,7 +15,7 @@ func GetCurrentOraclePrice() (*CurrentOraclePrice, error) {
}
// unmarshal the response
respobject := CurrentOraclePrice{}
respobject := CurrentOraclePriceResp{}
err = json.Unmarshal(respBody, &respobject)
if err != nil {
return nil, fmt.Errorf("failed to unmarshal response from path %v: %v", path, err)

4
heliumapi/stats.go
View File

@ -5,7 +5,7 @@ import (
"fmt"
)
func GetBlockchainStats() (*BlockchainStats, error) {
func GetBlockchainStats() (*BlockchainStatsResp, error) {
const path = "/v1/stats"
// query the api
@ -15,7 +15,7 @@ func GetBlockchainStats() (*BlockchainStats, error) {
}
// unmarshal the response
respobject := BlockchainStats{}
respobject := BlockchainStatsResp{}
err = json.Unmarshal(respBody, &respobject)
if err != nil {
return nil, fmt.Errorf("failed to unmarshal response from path %v: %v", path, err)

36
heliumapi/types.go
View File

@ -1,7 +1,10 @@
package heliumapi
type AccountResp struct {
Data Account `json:"data"`
}
type Account struct {
Data struct {
Address string `json:"address"`
Balance int `json:"balance"`
Block int `json:"block"`
@ -10,11 +13,13 @@ type Account struct {
SECBalance int `json:"sec_balance"`
SECNonce int `json:"sec_nonce"`
SpeculativeNonce int `json:"speculative_nonce"`
} `json:"data"`
}
type AccountHotspots struct {
Data []struct {
type AccountHotspotsResp struct {
Data []AccountHotspot `json:"data"`
}
type AccountHotspot struct {
Lng float64 `json:"lng"`
Lat float64 `json:"lat"`
TimestampAdded string `json:"timestamp_added"`
@ -50,20 +55,22 @@ type AccountHotspots struct {
BlockAdded int `json:"block_added"`
Block int `json:"block"`
Address string `json:"address"`
} `json:"data"`
}
type ActivityCounts struct {
type ActivityCountsResp struct {
Data map[string]int
}
type RewardTotal struct {
type RewardTotalResp struct {
Meta struct {
MinTime string `json:"min_time"`
MaxTime string `json:"max_time"`
} `json:"meta"`
Data struct {
Data RewardTotal `json:"data"`
}
type RewardTotal struct {
Total float64 `json:"total"`
Sum float64 `json:"sum"`
Stddev float64 `json:"stddev"`
@ -71,18 +78,22 @@ type RewardTotal struct {
Median float64 `json:"median"`
Max float64 `json:"max"`
Avg float64 `json:"avg"`
} `json:"data"`
}
type CurrentOraclePriceResp struct {
Data CurrentOraclePrice `json:"data"`
}
type CurrentOraclePrice struct {
Data struct {
Price int `json:"price"`
Block int `json:"block"`
} `json:"data"`
}
type BlockchainStatsResp struct {
Data BlockchainStats `json:"data"`
}
type BlockchainStats struct {
Data struct {
BlockTime struct {
LastDay struct {
Avg float64 `json:"avg"`
@ -136,5 +147,4 @@ type BlockchainStats struct {
} `json:"last_week"`
} `json:"election_times"`
TokenSupply float64 `json:"token_supply"`
} `json:"data"`
}