dmoonfire/dosage
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Activity

Code cleanup.

Browse source
This commit is contained in:
Bastian Kleineidam 2012-09-27 21:24:28 +02:00
parent 1333be7225
commit f3365f6a5e
1 changed files with 7 additions and 10 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

17
dosagelib/util.py
View file

@ -132,29 +132,26 @@ def normaliseURL(url):
pu[2] = '/' + '/'.join(segments)
return urlparse.urlunparse(pu)
def urlopen(url, referrer=None, retries=5):
def urlopen(url, referrer=None, retries=5, retry_wait_seconds=10):
assert retries >= 0, 'invalid retry value %r' % retries
assert retry_wait_seconds > 0, 'invalid retry seconds value %r' % retry_wait_seconds
# Work around urllib2 brokenness
url = normaliseURL(url)
req = urllib2.Request(url)
if referrer:
req.add_header('Referer', referrer)
req.add_header('User-Agent', UserAgent)
tries = 0
while 1:
while True:
try:
urlobj = urllib2.urlopen(req)
break
return urllib2.urlopen(req)
except IOError:
out.write('URL retrieval failed, sleeping %d seconds and retrying (%d)' % (2**tries, tries), 2)
time.sleep(2**tries)
out.write('URL retrieval failed; waiting %d seconds and retrying (%d)' % (retry_wait_seconds, tries), 2)
time.sleep(retry_wait_seconds)
tries += 1
if tries >= retries:
raise
return urlobj
def get_columns (fp):
"""Return number of columns for given file."""