Work around a bug in python requests.

This commit is contained in:
Bastian Kleineidam 2013-02-11 19:52:59 +01:00
parent 14f0a6fe78
commit a35c54525d

View file

@ -101,6 +101,10 @@ def getPageContent(url, max_content_bytes=MaxContentBytes, session=None):
# read page data # read page data
page = urlopen(url, max_content_bytes=max_content_bytes, session=session) page = urlopen(url, max_content_bytes=max_content_bytes, session=session)
data = page.text data = page.text
if not data:
# sometimes the python requests library is wonky - try again
page = urlopen(url, max_content_bytes=max_content_bytes, session=session)
data = page.text
# determine base URL # determine base URL
baseUrl = None baseUrl = None
match = baseSearch.search(data) match = baseSearch.search(data)