1
0
Fork 0

json parser

This commit is contained in:
Massaki Archambault 2022-10-30 14:20:59 -04:00
parent 4a1f6d052c
commit bda9b43318
3 changed files with 51 additions and 16 deletions

View File

@ -1,4 +1,5 @@
import argparse
import time
import yaml
@ -14,6 +15,12 @@ def main():
type=str,
default='dealwatch.yml',
)
parser.add_argument(
'-i', '--interval',
help='The target scrape interval, in minutes. (default: %(default)s)',
type=float,
default=10,
)
parser.add_argument(
'--user-agent',
help='The user-agent to spoof. (default: %(default)s)',
@ -46,8 +53,11 @@ def main():
# start the http server to server the prometheus metrics
start_http_server(args.listen_port, args.listen_address)
# start the main loop
while True:
for scrape_target in scrape_targets:
print(scrape_target.query_target())
time.sleep(args.interval * 60)
def parse_config(config_filename):
result = []
@ -70,6 +80,7 @@ def parse_config(config_filename):
url=get_field_or_die(target, 'url'),
selector=get_field_or_die(target, 'selector'),
regex=target.get('regex'),
parser=target.get('parser'),
))
return result

View File

@ -1,15 +1,19 @@
from email import parser
import json
import re
import httpx
import parsel
import pyjq
class ScrapeTarget:
def __init__(self, product_name, target_name, url, selector, regex=None):
def __init__(self, product_name, target_name, url, selector, regex=None, parser=None):
self.product_name = product_name
self.target_name = target_name
self.url = url
self.selector = selector+'::text'
self.selector = selector
self.regex = re.compile(regex if regex else r'[0-9]+(\.[0-9]{2})?')
self.parser = parser if parser else 'html'
self.headers = {}
def query_target(self):
@ -21,16 +25,35 @@ class ScrapeTarget:
headers=self.headers,
follow_redirects=True,
).text
selector = parsel.Selector(text=query_response)
# Match the selector
# parse the response and match the selector
selector_match = ''
if self.parser == 'html':
# parse response as html
selector = parsel.Selector(text=query_response)
selector_match = selector.css(self.selector).get()
if selector_match:
# Match the regex
elif self.parser == 'json':
# parse response as json
query_response_json = json.loads(query_response)
selector_match = str(pyjq.first(self.selector, query_response_json))
else:
# TODO: better error handling
print('invalid parser!')
return None
if not selector_match:
# TODO: better error handling
print('no selector_match!')
return None
# match the regex
regex_match = self.regex.search(selector_match)
if regex_match:
str_result = regex_match.group(0)
# Convert the reult to float
# convert the result to float
float_result = float(str_result)
return float_result
else:
# TODO: better error handling
print('no regex match!')
return None

View File

@ -13,6 +13,7 @@ install_requires=
PyYAML~=6.0
httpx~=0.23.0
parsel~=1.6.0
pyjq~=2.6.0
prometheus-client~=0.15.0
[options.entry_points]