helium-blockchain-exporter/heliumapi/query.go

package heliumapi

import (
	"context"
	"encoding/json"
	"fmt"
	"io/ioutil"
	"net/http"
	"time"

	"golang.org/x/time/rate"
)

var (
	// those are set via flags on init
	ApiUrl     string
	ApiTimeout time.Duration
	Limiter    *rate.Limiter

	client = &http.Client{}
)

// createGetRequest create a GET request to the helium api
func createGetRequest(path string, params map[string]string) (*http.Request, *context.CancelFunc, error) {
	ctx, ctxCancel := context.WithTimeout(context.Background(), ApiTimeout)
	req, err := http.NewRequestWithContext(ctx, "GET", fmt.Sprintf("%s%s", ApiUrl, path), nil)

	// setup headers
	req.Header.Add("Accept", "application/json")
	req.Header.Add("User-Agent", "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/95.0.4638.54 Safari/537.36")

	// setup query param if there are any
	if params != nil {
		query := req.URL.Query()
		for k, v := range params {
			query.Add(k, v)
		}
		if err != nil {
			defer ctxCancel()
			return nil, nil, err
		}
		req.URL.RawQuery = query.Encode()
	}

	return req, &ctxCancel, err
}

// getHeliumApi query a regular helium api endpoint
func getHeliumApi(path string, params *map[string]string) ([]byte, error) {
	// if params is nil, set it to an empty map
	if params == nil {
		params = &map[string]string{}
	}

	// ensure we respect the rate limit
	if err := Limiter.Wait(context.Background()); err != nil {
		return nil, err
	}

	// create the request
	req, ctxCancel, err := createGetRequest(path, *params)
	if err != nil {
		return nil, fmt.Errorf("failed to create query request %v: %v", path, err)
	}
	defer (*ctxCancel)()

	// query the api
	// log.Printf("querying %v", req.URL.String())
	resp, err := client.Do(req)
	if err != nil {
		return nil, fmt.Errorf("failed to query %v: %v", req.URL.String(), err)
	}
	defer resp.Body.Close()

	// validate the response status
	if resp.StatusCode != 200 {
		return nil, fmt.Errorf("failed to query %v: http status %v", req.URL.String(), resp.StatusCode)
	}

	// read the response body
	body, err := ioutil.ReadAll(resp.Body)
	if err != nil {
		return nil, fmt.Errorf("failed to read response body of %v: %v", req.URL.String(), err)
	}

	return body, nil
}

// getHeliumApiWithCursor query a helium api with a cursor
// It will continue to query the api until all the cursors are exhausted. Combine all the
// responses in a single object.
func getHeliumApiWithCursor(path string, params *map[string]string) ([][]byte, error) {
	// if params is nil, set it to an empty map
	if params == nil {
		params = &map[string]string{}
	}

	type ResponseWithCursor struct {
		Cursor string `json:"cursor"`
	}

	// create the initial request
	res := [][]byte{}
	respCursor := ResponseWithCursor{}
	req, ctxCancel, err := createGetRequest(path, *params)
	if err != nil {
		return nil, fmt.Errorf("failed to create query request %v: %v", path, err)
	}
	defer (*ctxCancel)()

	for {
		// ensure we respect the rate limit
		if err := Limiter.Wait(context.Background()); err != nil {
			return nil, err
		}

		// query the api
		resp, err := client.Do(req)
		if err != nil {
			return nil, fmt.Errorf("failed to query %v: %v", req.URL.String(), err)
		}
		defer resp.Body.Close()

		// read the response body and add it to the result array
		// log.Printf("querying %v", req.URL.String())
		body, err := ioutil.ReadAll(resp.Body)
		if err != nil {
			return nil, fmt.Errorf("failed to read response body of %v: %v", req.URL.String(), err)
		}
		res = append(res, body)

		// validate the response status
		if resp.StatusCode != 200 {
			return nil, fmt.Errorf("failed to query %v: http status %v", req.URL.String(), resp.StatusCode)
		}

		// parse the response body for a cursor
		respCursor.Cursor = ""
		if err := json.Unmarshal(body, &respCursor); err != nil {
			return nil, fmt.Errorf("failed to unmarshal response from %v: %v", req.URL.String(), err)
		}

		// continue querying until there is no longer a cursor
		if respCursor.Cursor != "" {
			params = &map[string]string{"cursor": respCursor.Cursor}
			req, ctxCancel, err = createGetRequest(path, *params)
			if err != nil {
				return nil, fmt.Errorf("failed to create query request %v: %v", req.URL.String(), err)
			}
			defer (*ctxCancel)()
		} else {
			break
		}
	}

	return res, nil
}