package heliumapi import ( "context" "encoding/json" "fmt" "io/ioutil" "net/http" "time" "golang.org/x/time/rate" ) var ( // those are set via flags on init ApiUrl string ApiTimeout time.Duration Limiter *rate.Limiter client = &http.Client{} ) // createGetRequest create a GET request to the helium api func createGetRequest(path string, params map[string]string) (*http.Request, *context.CancelFunc, error) { ctx, ctxCancel := context.WithTimeout(context.Background(), ApiTimeout) req, err := http.NewRequestWithContext(ctx, "GET", fmt.Sprintf("%s%s", ApiUrl, path), nil) // setup headers req.Header.Add("Accept", "application/json") req.Header.Add("User-Agent", "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/95.0.4638.54 Safari/537.36") // setup query param if there are any if params != nil { query := req.URL.Query() for k, v := range params { query.Add(k, v) } if err != nil { defer ctxCancel() return nil, nil, err } req.URL.RawQuery = query.Encode() } return req, &ctxCancel, err } // getHeliumApi query a regular helium api endpoint func getHeliumApi(path string, params *map[string]string) ([]byte, error) { // if params is nil, set it to an empty map if params == nil { params = &map[string]string{} } // ensure we respect the rate limit if err := Limiter.Wait(context.Background()); err != nil { return nil, err } // create the request req, ctxCancel, err := createGetRequest(path, *params) if err != nil { return nil, fmt.Errorf("failed to create query request %v: %v", path, err) } defer (*ctxCancel)() // query the api // log.Printf("querying %v", req.URL.String()) resp, err := client.Do(req) if err != nil { return nil, fmt.Errorf("failed to query %v: %v", req.URL.String(), err) } defer resp.Body.Close() // validate the response status if resp.StatusCode != 200 { return nil, fmt.Errorf("failed to query %v: http status %v", req.URL.String(), resp.StatusCode) } // read the response body body, err := ioutil.ReadAll(resp.Body) if err != nil { return nil, fmt.Errorf("failed to read response body of %v: %v", req.URL.String(), err) } return body, nil } // getHeliumApiWithCursor query a helium api with a cursor // It will continue to query the api until all the cursors are exhausted. Combine all the // responses in a single object. func getHeliumApiWithCursor(path string, params *map[string]string) ([][]byte, error) { // if params is nil, set it to an empty map if params == nil { params = &map[string]string{} } type ResponseWithCursor struct { Cursor string `json:"cursor"` } // create the initial request res := [][]byte{} respCursor := ResponseWithCursor{} req, ctxCancel, err := createGetRequest(path, *params) if err != nil { return nil, fmt.Errorf("failed to create query request %v: %v", path, err) } defer (*ctxCancel)() for { // ensure we respect the rate limit if err := Limiter.Wait(context.Background()); err != nil { return nil, err } // query the api resp, err := client.Do(req) if err != nil { return nil, fmt.Errorf("failed to query %v: %v", req.URL.String(), err) } defer resp.Body.Close() // read the response body and add it to the result array // log.Printf("querying %v", req.URL.String()) body, err := ioutil.ReadAll(resp.Body) if err != nil { return nil, fmt.Errorf("failed to read response body of %v: %v", req.URL.String(), err) } res = append(res, body) // validate the response status if resp.StatusCode != 200 { return nil, fmt.Errorf("failed to query %v: http status %v", req.URL.String(), resp.StatusCode) } // parse the response body for a cursor respCursor.Cursor = "" if err := json.Unmarshal(body, &respCursor); err != nil { return nil, fmt.Errorf("failed to unmarshal response from %v: %v", req.URL.String(), err) } // continue querying until there is no longer a cursor if respCursor.Cursor != "" { params = &map[string]string{"cursor": respCursor.Cursor} req, ctxCancel, err = createGetRequest(path, *params) if err != nil { return nil, fmt.Errorf("failed to create query request %v: %v", req.URL.String(), err) } defer (*ctxCancel)() } else { break } } return res, nil }