From ac9d8db1e8c393bfe5f9f9e0b8d362493ac035ae Mon Sep 17 00:00:00 2001 From: Tobias Gruetzmacher Date: Sun, 3 Nov 2019 20:17:27 +0100 Subject: [PATCH] Make sure user agent is in all HTTP requests --- dosagelib/util.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/dosagelib/util.py b/dosagelib/util.py index 20bf21949..b3970f00c 100644 --- a/dosagelib/util.py +++ b/dosagelib/util.py @@ -64,6 +64,7 @@ def requests_session(): retry = Retry(MaxRetries, backoff_factor=RetryBackoffFactor) s.mount('http://', HTTPAdapter(max_retries=retry)) s.mount('https://', HTTPAdapter(max_retries=retry)) + s.headers.update({'User-Agent': UserAgent}) return s @@ -275,12 +276,11 @@ def get_robotstxt_parser(url, session=None): def urlopen(url, session, referrer=None, max_content_bytes=None, - allow_errors=(), useragent=UserAgent, **kwargs): + allow_errors=(), **kwargs): """Open an URL and return the response object.""" out.debug(u'Open URL %s' % url) if 'headers' not in kwargs: kwargs['headers'] = {} - kwargs['headers']['User-Agent'] = useragent if referrer: kwargs['headers']['Referer'] = referrer out.debug(u'Sending headers %s' % kwargs['headers'], level=3)