dmoonfire/dosage
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Activity

Code cleanup.

Browse source
This commit is contained in:
Bastian Kleineidam 2012-09-27 21:24:28 +02:00
parent 1333be7225
commit f3365f6a5e
1 changed files with 7 additions and 10 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

17
dosagelib/util.py
View file

@ -132,29 +132,26 @@ def normaliseURL(url):
pu[2] = '/' + '/'.join(segments) pu[2] = '/' + '/'.join(segments)
return urlparse.urlunparse(pu) return urlparse.urlunparse(pu)
def urlopen(url, referrer=None, retries=5, retry_wait_seconds=10):
def urlopen(url, referrer=None, retries=5): assert retries >= 0, 'invalid retry value %r' % retries
assert retry_wait_seconds > 0, 'invalid retry seconds value %r' % retry_wait_seconds
# Work around urllib2 brokenness # Work around urllib2 brokenness
url = normaliseURL(url) url = normaliseURL(url)
req = urllib2.Request(url) req = urllib2.Request(url)
if referrer: if referrer:
req.add_header('Referer', referrer) req.add_header('Referer', referrer)
req.add_header('User-Agent', UserAgent) req.add_header('User-Agent', UserAgent)
tries = 0 tries = 0
while 1: while True:
try: try:
urlobj = urllib2.urlopen(req) return urllib2.urlopen(req)
break
except IOError: except IOError:
out.write('URL retrieval failed, sleeping %d seconds and retrying (%d)' % (2**tries, tries), 2) out.write('URL retrieval failed; waiting %d seconds and retrying (%d)' % (retry_wait_seconds, tries), 2)
time.sleep(2**tries) time.sleep(retry_wait_seconds)
tries += 1 tries += 1
if tries >= retries: if tries >= retries:
raise raise
return urlobj
def get_columns (fp): def get_columns (fp):
"""Return number of columns for given file.""" """Return number of columns for given file."""