.python-version

@@ -1 +0,0 @@
-3.10

ecommerce_exporter/cli.py

@@ -1,19 +1,13 @@
 import argparse
 import os
 import time
-import logging
 
 import yaml
 
+from httpx import RequestError
 from prometheus_client import start_http_server, Gauge, Counter
 
-from ecommerce_exporter.scrape_target import ScrapeTarget
-
-logging.basicConfig(
-    format=os.environ.get('LOG_FORMAT', '[%(asctime)s] [%(levelname)-8s] %(message)s'),
-    level=os.environ.get('LOG_LEVEL', 'INFO')
-)
-logger = logging.getLogger(__name__)
+from ecommerce_exporter.scrape_target import ScrapeError, ScrapeTarget
 
 ECOMMERCE_SCRAPE_TARGET_VALUE = Gauge(
     'ecommerce_scrape_target_value',
@@ -49,7 +43,7 @@ def main():
         '--user-agent',
         help='The user-agent to spoof. (default: %(default)s)',
         type=str,
-        default='Mozilla/5.0 (X11; Linux x86_64; rv:131.0) Gecko/20100101 Firefox/131.0',
+        default='Mozilla/5.0 (X11; Linux x86_64; rv:106.0) Gecko/20100101 Firefox/106.0',
     )
     parser.add_argument(
         '-p', '--listen-port',
@@ -70,19 +64,18 @@ def main():
     # setup the headers for each scrape targets
     for scrape_target in scrape_targets:
         scrape_target.headers = {
-            'accept': '*/*',
-            'user-agent': args.user_agent,
+            'Accept': '*/*',
+            'User-Agent': args.user_agent,
         }
 
     # start the http server to server the prometheus metrics
-    logger.info("serving metrics on http://%s:%s/metrics", args.listen_address, args.listen_port)
     start_http_server(args.listen_port, args.listen_address)
 
     # start the main loop
     while True:
         for scrape_target in scrape_targets:
             try:
-                logger.info("Starting scrape. product: '%s', target '%s'", scrape_target.product_name, scrape_target.target_name)
+                print("Starting scrape. product: '%s', target '%s'" % (scrape_target.product_name, scrape_target.target_name))
                 value = scrape_target.query_target()
                 ECOMMERCE_SCRAPE_TARGET_VALUE.labels(
                     product_name=scrape_target.product_name,
@@ -92,10 +85,8 @@ def main():
                     product_name=scrape_target.product_name,
                     target_name=scrape_target.target_name,
                 ).inc()
-            except KeyboardInterrupt:
-                return
-            except Exception as e:
-                logger.error("Failed to scrape! product: '%s', target: '%s', message: '%s'" , scrape_target.product_name, scrape_target.target_name, e)
+            except (RequestError, ScrapeError) as e:
+                print("Failed to scrape! product: '%s', target: '%s', message: '%s'" % (scrape_target.product_name, scrape_target.target_name, e))
                 ECOMMERCE_SCRAPE_TARGET_FAILURE.labels(
                     product_name=scrape_target.product_name,
                     target_name=scrape_target.target_name,
@@ -105,7 +96,7 @@ def main():
 
 def parse_config(config_filename):
     result = []
-    logger.info('Loading configurations from %s', config_filename)
+    print('Loading configurations from %s' % config_filename)
     with open(config_filename, 'r') as f:
         config = yaml.safe_load(f)
 

ecommerce_exporter/scrape_target.py

@@ -6,9 +6,7 @@ from urllib.parse import urlparse
 import httpx
 import parsel
 import pyjq
-import logging
 
-logger = logging.getLogger(__name__)
 class ScrapeTarget:
     def __init__(self, product_name, url, selector, target_name=None, regex=None, parser=None):
         self.product_name = product_name
@@ -18,10 +16,6 @@ class ScrapeTarget:
         self.regex = re.compile(regex if regex else r'[0-9]+(\.[0-9]{2})?')
         self.parser = parser if parser else 'html'
         self.headers = {}
-        self.client = httpx.Client(
-            follow_redirects=True,
-            http2=True,
-        )
 
         # sanity check
         valid_parsers = ('html', 'json')
@@ -31,23 +25,21 @@ class ScrapeTarget:
     def query_target(self):
         # some sites get suspicious if we talk to them in HTTP/1.1 (maybe because it doesn't match our user-agent?)
         # we use httpx to have HTTP2 support and circumvent that issue
-        query_response = self.client.get(
-            self.url,
+        query_response = httpx.get(
+            url=self.url,
             headers=self.headers,
-        )
-        logger.info('Status: %s', query_response.status_code)
-        query_response_text = query_response.text
-        logger.debug('Response: %s', query_response_text)
+            follow_redirects=True,
+        ).text
 
         # parse the response and match the selector
         selector_match = ''
         if self.parser == 'html':
             # parse response as html
-            selector = parsel.Selector(text=query_response_text)
+            selector = parsel.Selector(text=query_response)
             selector_match = selector.css(self.selector).get()
         elif self.parser == 'json':
             # parse response as json
-            query_response_json = json.loads(query_response_text)
+            query_response_json = json.loads(query_response)
             selector_match = str(pyjq.first(self.selector, query_response_json))
         else:
             raise ScrapeError('Invalid parser!')

setup.cfg

@@ -16,7 +16,7 @@ setup_requires =
   setuptools_scm
 install_requires=
   PyYAML~=6.0
-  httpx[http2]~=0.23.0
+  httpx~=0.23.0
   parsel~=1.6.0
   pyjq~=2.6.0
   prometheus-client~=0.15.0