base activity query min_time on response from the api instead of last refresh

contrib/dashboard/records.yaml

@@ -0,0 +1,12 @@
+groups:
+  - name: helium-blockchain-exporter-rules
+    interval: 1m
+    rules:
+      - record: hotspot:helium_hotspot_rewards_hnt:increase15m
+        expr: sum by (account, hotspot, hotspot_name) (increase(helium_hotspot_rewards_hnt_total[15m]))
+      - record: type:helium_hotspot_activity:floor_increase15m
+        expr: sum by (account, hotspot, hotspot_name, type) (floor(increase(helium_hotspot_activity_total[15m])))
+      - record: hotspot:helium_hotspot_rewards_hnt:increase1d
+        expr: sum by (account, hotspot, hotspot_name) (increase(helium_hotspot_rewards_hnt_total[1d]))
+      - record: hotspot:helium_hotspot_rewards_hnt:increase1w
+        expr: sum by (account, hotspot, hotspot_name) (increase(helium_hotspot_rewards_hnt_total[1w]))

exporter.go

@@ -449,6 +449,7 @@ func (e *Exporter) collectAccountRewardsTotalMetrics(wg *sync.WaitGroup, ch chan
 func (e *Exporter) collectAccountTransactionsMetrics(wg *sync.WaitGroup, ch chan<- prometheus.Metric, account *Account) {
 	defer wg.Done()
 
+	// We optimize our queries by asking the api only the activities we car about.
 	activityTypes := []string{
 		"add_gateway_v1",
 		"assert_location_v1",
@@ -463,13 +464,25 @@ func (e *Exporter) collectAccountTransactionsMetrics(wg *sync.WaitGroup, ch chan
 		"unstake_validator_v1",
 	}
 
-	// we can only want to allow a single instance of the routine doing
+	// We can only want to allow a single instance of the routine doing
-	// calculations on the deposited and widthdrawn total
+	// calculations on the deposited and widthdrawn total.
 	account.Tx.UpdateLock.Lock()
 	defer account.Tx.UpdateLock.Unlock()
 
-	now := time.Now()
+	// We want to keep in memory the timestamp of the last activity we
-	activities, err := heliumapi.GetActivityForAccount(account.Address, activityTypes, &account.Tx.LastUpdate, &now)
+	// received from the api. We cannot do something naive like [lastscrape, now]
+	// because the api can take a few seconds to sync with the chain and
+	// we can miss some activities by doing it that way.
+	lastActivityTime := account.Tx.LastUpdate.Unix()
+	updateLastActivityTime := func(newTime int64) {
+		if lastActivityTime < newTime {
+			lastActivityTime = newTime
+		}
+	}
+
+	// add 1 second to the last update time to avoid querying the same activity twice
+	minTime := account.Tx.LastUpdate.Add(time.Second * 1)
+	activities, err := heliumapi.GetActivityForAccount(account.Address, activityTypes, &minTime, nil)
 	if err != nil {
 		log.Println(err)
 		return
@@ -478,12 +491,15 @@ func (e *Exporter) collectAccountTransactionsMetrics(wg *sync.WaitGroup, ch chan
 	// impl based on https://github.com/helium/hotspot-app/blob/918563fba84d1abf4554a43a4d42bb838d017bd3/src/features/wallet/root/useActivityItem.tsx#L336
 	for _, activity := range activities.AddGatewayV1 {
 		account.Tx.WithdrawalTotal += activity.StakingFee
+		updateLastActivityTime(activity.Time)
 	}
 	for _, activity := range activities.AssertLocationV1 {
 		account.Tx.WithdrawalTotal += activity.StakingFee
+		updateLastActivityTime(activity.Time)
 	}
 	for _, activity := range activities.AssertLocationV2 {
 		account.Tx.WithdrawalTotal += activity.StakingFee
+		updateLastActivityTime(activity.Time)
 	}
 	for _, activity := range activities.PaymentV1 {
 		if activity.Payer == account.Address {
@@ -491,6 +507,7 @@ func (e *Exporter) collectAccountTransactionsMetrics(wg *sync.WaitGroup, ch chan
 		} else {
 			account.Tx.DepositTotal += activity.Amount
 		}
+		updateLastActivityTime(activity.Time)
 	}
 	for _, activity := range activities.PaymentV2 {
 		if activity.Payer == account.Address {
@@ -506,22 +523,27 @@ func (e *Exporter) collectAccountTransactionsMetrics(wg *sync.WaitGroup, ch chan
 				}
 			}
 		}
+		updateLastActivityTime(activity.Time)
 	}
 	for _, activity := range activities.RewardsV1 {
 		for _, reward := range activity.Rewards {
 			account.Tx.DepositTotal += reward.Amount
 		}
+		updateLastActivityTime(activity.Time)
 	}
 	for _, activity := range activities.RewardsV2 {
 		for _, reward := range activity.Rewards {
 			account.Tx.DepositTotal += reward.Amount
 		}
+		updateLastActivityTime(activity.Time)
 	}
 	for _, activity := range activities.StakeValidatorV1 {
 		account.Tx.WithdrawalTotal += activity.Stake
+		updateLastActivityTime(activity.Time)
 	}
 	for _, activity := range activities.TokenBurnV1 {
 		account.Tx.WithdrawalTotal += activity.Amount
+		updateLastActivityTime(activity.Time)
 	}
 	for _, activity := range activities.TransferHotspotV1 {
 		if activity.Buyer == account.Address {
@@ -529,11 +551,13 @@ func (e *Exporter) collectAccountTransactionsMetrics(wg *sync.WaitGroup, ch chan
 		} else {
 			account.Tx.DepositTotal += activity.AmountToSeller
 		}
+		updateLastActivityTime(activity.Time)
 	}
 	for _, activity := range activities.UnstakeValidatorV1 {
 		account.Tx.WithdrawalTotal += activity.StakeAmount
+		updateLastActivityTime(activity.Time)
 	}
-	account.Tx.LastUpdate = now
+	account.Tx.LastUpdate = time.Unix(lastActivityTime, 0)
 
 	ch <- prometheus.MustNewConstMetric(
 		accountDepositsHnt.Desc, accountDepositsHnt.Type, float64(account.Tx.DepositTotal)/blockchain_hnt_factor,

heliumapi/activity/types.go

@@ -14,8 +14,16 @@ type Activities struct {
 	UnstakeValidatorV1 []UnstakeValidatorV1
 }
 
-type AddGatewayV1 struct {
+type BaseActivity struct {
 	Hash       string `json:"hash"`
+	Time       int64  `json:"time"`
+	StartEpoch int    `json:"start_epoch"`
+	EndEpoch   int    `json:"end_epoch"`
+	Height     int    `json:"height"`
+}
+
+type AddGatewayV1 struct {
+	BaseActivity
 	Fee        int    `json:"fee"`
 	Owner      string `json:"owner"`
 	Payer      string `json:"payer"`
@@ -24,7 +32,7 @@ type AddGatewayV1 struct {
 }
 
 type AssertLocationV1 struct {
-	Hash       string `json:"hash"`
+	BaseActivity
 	Fee        int    `json:"fee"`
 	Nonce      int    `json:"nonce"`
 	Owner      string `json:"owner"`
@@ -35,7 +43,7 @@ type AssertLocationV1 struct {
 }
 
 type AssertLocationV2 struct {
-	Hash       string `json:"hash"`
+	BaseActivity
 	Fee        int    `json:"fee"`
 	Gain       int    `json:"gain"`
 	Nonce      int    `json:"nonce"`
@@ -48,7 +56,7 @@ type AssertLocationV2 struct {
 }
 
 type PaymentV1 struct {
-	Hash   string `json:"hash"`
+	BaseActivity
 	Amount int    `json:"amount"`
 	Fee    int    `json:"fee"`
 	Nonce  int    `json:"nonce"`
@@ -57,7 +65,7 @@ type PaymentV1 struct {
 }
 
 type PaymentV2 struct {
-	Hash     string `json:"hash"`
+	BaseActivity
 	Fee      int    `json:"fee"`
 	Nonce    int    `json:"nonce"`
 	Payer    string `json:"payer"`
@@ -76,31 +84,31 @@ type reward struct {
 }
 
 type RewardsV1 struct {
-	Hash       string   `json:"hash"`
+	BaseActivity
 	StartEpoch int      `json:"start_epoch"`
 	EndEpoch   int      `json:"end_epoch"`
 	Rewards    []reward `json:"rewards"`
 }
 
 type RewardsV2 struct {
-	Hash       string   `json:"hash"`
+	BaseActivity
 	StartEpoch int      `json:"start_epoch"`
 	EndEpoch   int      `json:"end_epoch"`
 	Rewards    []reward `json:"rewards"`
 }
 
 type StakeValidatorV1 struct {
+	BaseActivity
 	Address        string `json:"address"`
 	Fee            int    `json:"fee"`
-	Hash           string `json:"hash"`
 	Owner          string `json:"owner"`
 	Stake          int    `json:"stake"`
 	OwnerSignature string `json:"owner_signature"`
 }
 
 type TokenBurnV1 struct {
+	BaseActivity
 	Fee    int    `json:"fee"`
-	Hash   string `json:"hash"`
 	Memo   string `json:"memo"`
 	Nonce  int    `json:"nonce"`
 	Payer  string `json:"payer"`
@@ -109,7 +117,7 @@ type TokenBurnV1 struct {
 }
 
 type TransferHotspotV1 struct {
-	Hash           string `json:"hash"`
+	BaseActivity
 	Fee            int    `json:"fee"`
 	Buyer          string `json:"buyer"`
 	Seller         string `json:"seller"`
@@ -119,11 +127,11 @@ type TransferHotspotV1 struct {
 }
 
 type UnstakeValidatorV1 struct {
+	BaseActivity
 	Address            string `json:"address"`
 	Owner              string `json:"owner"`
 	OwnerSignature     string `json:"owner_signature"`
 	Fee                int    `json:"fee"`
 	StakeAmount        int    `json:"stake_amount"`
 	StakeReleaseHeight int    `json:"stake_release_height"`
-	Hash               string `json:"hash"`
 }