diff --git a/contrib/dashboard/records.yaml b/contrib/dashboard/records.yaml new file mode 100644 index 0000000..25e0688 --- /dev/null +++ b/contrib/dashboard/records.yaml @@ -0,0 +1,12 @@ +groups: + - name: helium-blockchain-exporter-rules + interval: 1m + rules: + - record: hotspot:helium_hotspot_rewards_hnt:increase15m + expr: sum by (account, hotspot, hotspot_name) (increase(helium_hotspot_rewards_hnt_total[15m])) + - record: type:helium_hotspot_activity:floor_increase15m + expr: sum by (account, hotspot, hotspot_name, type) (floor(increase(helium_hotspot_activity_total[15m]))) + - record: hotspot:helium_hotspot_rewards_hnt:increase1d + expr: sum by (account, hotspot, hotspot_name) (increase(helium_hotspot_rewards_hnt_total[1d])) + - record: hotspot:helium_hotspot_rewards_hnt:increase1w + expr: sum by (account, hotspot, hotspot_name) (increase(helium_hotspot_rewards_hnt_total[1w])) diff --git a/exporter.go b/exporter.go index 8852aef..df017f5 100644 --- a/exporter.go +++ b/exporter.go @@ -449,6 +449,7 @@ func (e *Exporter) collectAccountRewardsTotalMetrics(wg *sync.WaitGroup, ch chan func (e *Exporter) collectAccountTransactionsMetrics(wg *sync.WaitGroup, ch chan<- prometheus.Metric, account *Account) { defer wg.Done() + // We optimize our queries by asking the api only the activities we car about. activityTypes := []string{ "add_gateway_v1", "assert_location_v1", @@ -463,13 +464,25 @@ func (e *Exporter) collectAccountTransactionsMetrics(wg *sync.WaitGroup, ch chan "unstake_validator_v1", } - // we can only want to allow a single instance of the routine doing - // calculations on the deposited and widthdrawn total + // We can only want to allow a single instance of the routine doing + // calculations on the deposited and widthdrawn total. account.Tx.UpdateLock.Lock() defer account.Tx.UpdateLock.Unlock() - now := time.Now() - activities, err := heliumapi.GetActivityForAccount(account.Address, activityTypes, &account.Tx.LastUpdate, &now) + // We want to keep in memory the timestamp of the last activity we + // received from the api. We cannot do something naive like [lastscrape, now] + // because the api can take a few seconds to sync with the chain and + // we can miss some activities by doing it that way. + lastActivityTime := account.Tx.LastUpdate.Unix() + updateLastActivityTime := func(newTime int64) { + if lastActivityTime < newTime { + lastActivityTime = newTime + } + } + + // add 1 second to the last update time to avoid querying the same activity twice + minTime := account.Tx.LastUpdate.Add(time.Second * 1) + activities, err := heliumapi.GetActivityForAccount(account.Address, activityTypes, &minTime, nil) if err != nil { log.Println(err) return @@ -478,12 +491,15 @@ func (e *Exporter) collectAccountTransactionsMetrics(wg *sync.WaitGroup, ch chan // impl based on https://github.com/helium/hotspot-app/blob/918563fba84d1abf4554a43a4d42bb838d017bd3/src/features/wallet/root/useActivityItem.tsx#L336 for _, activity := range activities.AddGatewayV1 { account.Tx.WithdrawalTotal += activity.StakingFee + updateLastActivityTime(activity.Time) } for _, activity := range activities.AssertLocationV1 { account.Tx.WithdrawalTotal += activity.StakingFee + updateLastActivityTime(activity.Time) } for _, activity := range activities.AssertLocationV2 { account.Tx.WithdrawalTotal += activity.StakingFee + updateLastActivityTime(activity.Time) } for _, activity := range activities.PaymentV1 { if activity.Payer == account.Address { @@ -491,6 +507,7 @@ func (e *Exporter) collectAccountTransactionsMetrics(wg *sync.WaitGroup, ch chan } else { account.Tx.DepositTotal += activity.Amount } + updateLastActivityTime(activity.Time) } for _, activity := range activities.PaymentV2 { if activity.Payer == account.Address { @@ -506,22 +523,27 @@ func (e *Exporter) collectAccountTransactionsMetrics(wg *sync.WaitGroup, ch chan } } } + updateLastActivityTime(activity.Time) } for _, activity := range activities.RewardsV1 { for _, reward := range activity.Rewards { account.Tx.DepositTotal += reward.Amount } + updateLastActivityTime(activity.Time) } for _, activity := range activities.RewardsV2 { for _, reward := range activity.Rewards { account.Tx.DepositTotal += reward.Amount } + updateLastActivityTime(activity.Time) } for _, activity := range activities.StakeValidatorV1 { account.Tx.WithdrawalTotal += activity.Stake + updateLastActivityTime(activity.Time) } for _, activity := range activities.TokenBurnV1 { account.Tx.WithdrawalTotal += activity.Amount + updateLastActivityTime(activity.Time) } for _, activity := range activities.TransferHotspotV1 { if activity.Buyer == account.Address { @@ -529,11 +551,13 @@ func (e *Exporter) collectAccountTransactionsMetrics(wg *sync.WaitGroup, ch chan } else { account.Tx.DepositTotal += activity.AmountToSeller } + updateLastActivityTime(activity.Time) } for _, activity := range activities.UnstakeValidatorV1 { account.Tx.WithdrawalTotal += activity.StakeAmount + updateLastActivityTime(activity.Time) } - account.Tx.LastUpdate = now + account.Tx.LastUpdate = time.Unix(lastActivityTime, 0) ch <- prometheus.MustNewConstMetric( accountDepositsHnt.Desc, accountDepositsHnt.Type, float64(account.Tx.DepositTotal)/blockchain_hnt_factor, diff --git a/heliumapi/activity/types.go b/heliumapi/activity/types.go index 737f0cf..d4f9737 100644 --- a/heliumapi/activity/types.go +++ b/heliumapi/activity/types.go @@ -14,8 +14,16 @@ type Activities struct { UnstakeValidatorV1 []UnstakeValidatorV1 } -type AddGatewayV1 struct { +type BaseActivity struct { Hash string `json:"hash"` + Time int64 `json:"time"` + StartEpoch int `json:"start_epoch"` + EndEpoch int `json:"end_epoch"` + Height int `json:"height"` +} + +type AddGatewayV1 struct { + BaseActivity Fee int `json:"fee"` Owner string `json:"owner"` Payer string `json:"payer"` @@ -24,7 +32,7 @@ type AddGatewayV1 struct { } type AssertLocationV1 struct { - Hash string `json:"hash"` + BaseActivity Fee int `json:"fee"` Nonce int `json:"nonce"` Owner string `json:"owner"` @@ -35,7 +43,7 @@ type AssertLocationV1 struct { } type AssertLocationV2 struct { - Hash string `json:"hash"` + BaseActivity Fee int `json:"fee"` Gain int `json:"gain"` Nonce int `json:"nonce"` @@ -48,7 +56,7 @@ type AssertLocationV2 struct { } type PaymentV1 struct { - Hash string `json:"hash"` + BaseActivity Amount int `json:"amount"` Fee int `json:"fee"` Nonce int `json:"nonce"` @@ -57,7 +65,7 @@ type PaymentV1 struct { } type PaymentV2 struct { - Hash string `json:"hash"` + BaseActivity Fee int `json:"fee"` Nonce int `json:"nonce"` Payer string `json:"payer"` @@ -76,31 +84,31 @@ type reward struct { } type RewardsV1 struct { - Hash string `json:"hash"` + BaseActivity StartEpoch int `json:"start_epoch"` EndEpoch int `json:"end_epoch"` Rewards []reward `json:"rewards"` } type RewardsV2 struct { - Hash string `json:"hash"` + BaseActivity StartEpoch int `json:"start_epoch"` EndEpoch int `json:"end_epoch"` Rewards []reward `json:"rewards"` } type StakeValidatorV1 struct { + BaseActivity Address string `json:"address"` Fee int `json:"fee"` - Hash string `json:"hash"` Owner string `json:"owner"` Stake int `json:"stake"` OwnerSignature string `json:"owner_signature"` } type TokenBurnV1 struct { + BaseActivity Fee int `json:"fee"` - Hash string `json:"hash"` Memo string `json:"memo"` Nonce int `json:"nonce"` Payer string `json:"payer"` @@ -109,7 +117,7 @@ type TokenBurnV1 struct { } type TransferHotspotV1 struct { - Hash string `json:"hash"` + BaseActivity Fee int `json:"fee"` Buyer string `json:"buyer"` Seller string `json:"seller"` @@ -119,11 +127,11 @@ type TransferHotspotV1 struct { } type UnstakeValidatorV1 struct { + BaseActivity Address string `json:"address"` Owner string `json:"owner"` OwnerSignature string `json:"owner_signature"` Fee int `json:"fee"` StakeAmount int `json:"stake_amount"` StakeReleaseHeight int `json:"stake_release_height"` - Hash string `json:"hash"` }