dmoonfire/dosage
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Activity

Make sure user agent is in all HTTP requests

Browse source
This commit is contained in:
Tobias Gruetzmacher 2019-11-03 20:17:27 +01:00
parent d243017aff
commit ac9d8db1e8
1 changed files with 2 additions and 2 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

4
dosagelib/util.py
View file

@ -64,6 +64,7 @@ def requests_session():
retry = Retry(MaxRetries, backoff_factor=RetryBackoffFactor) retry = Retry(MaxRetries, backoff_factor=RetryBackoffFactor)
s.mount('http://', HTTPAdapter(max_retries=retry)) s.mount('http://', HTTPAdapter(max_retries=retry))
s.mount('https://', HTTPAdapter(max_retries=retry)) s.mount('https://', HTTPAdapter(max_retries=retry))
s.headers.update({'User-Agent': UserAgent})
return s return s
@ -275,12 +276,11 @@ def get_robotstxt_parser(url, session=None):
def urlopen(url, session, referrer=None, max_content_bytes=None, def urlopen(url, session, referrer=None, max_content_bytes=None,
allow_errors=(), useragent=UserAgent, **kwargs): allow_errors=(), **kwargs):
"""Open an URL and return the response object.""" """Open an URL and return the response object."""
out.debug(u'Open URL %s' % url) out.debug(u'Open URL %s' % url)
if 'headers' not in kwargs: if 'headers' not in kwargs:
kwargs['headers'] = {} kwargs['headers'] = {}
kwargs['headers']['User-Agent'] = useragent
if referrer: if referrer:
kwargs['headers']['Referer'] = referrer kwargs['headers']['Referer'] = referrer
out.debug(u'Sending headers %s' % kwargs['headers'], level=3) out.debug(u'Sending headers %s' % kwargs['headers'], level=3)