From f3365f6a5e428d5841cb6ae4688d3214d15f5c95 Mon Sep 17 00:00:00 2001 From: Bastian Kleineidam Date: Thu, 27 Sep 2012 21:24:28 +0200 Subject: [PATCH] Code cleanup. --- dosagelib/util.py | 17 +++++++---------- 1 file changed, 7 insertions(+), 10 deletions(-) diff --git a/dosagelib/util.py b/dosagelib/util.py index de3e17384..2feb3d277 100644 --- a/dosagelib/util.py +++ b/dosagelib/util.py @@ -132,29 +132,26 @@ def normaliseURL(url): pu[2] = '/' + '/'.join(segments) return urlparse.urlunparse(pu) - -def urlopen(url, referrer=None, retries=5): +def urlopen(url, referrer=None, retries=5, retry_wait_seconds=10): + assert retries >= 0, 'invalid retry value %r' % retries + assert retry_wait_seconds > 0, 'invalid retry seconds value %r' % retry_wait_seconds # Work around urllib2 brokenness url = normaliseURL(url) req = urllib2.Request(url) if referrer: req.add_header('Referer', referrer) req.add_header('User-Agent', UserAgent) - tries = 0 - while 1: + while True: try: - urlobj = urllib2.urlopen(req) - break + return urllib2.urlopen(req) except IOError: - out.write('URL retrieval failed, sleeping %d seconds and retrying (%d)' % (2**tries, tries), 2) - time.sleep(2**tries) + out.write('URL retrieval failed; waiting %d seconds and retrying (%d)' % (retry_wait_seconds, tries), 2) + time.sleep(retry_wait_seconds) tries += 1 if tries >= retries: raise - return urlobj - def get_columns (fp): """Return number of columns for given file."""