From 78f44e9d9cc6feb1a7d386ff76f152ee303919e8 Mon Sep 17 00:00:00 2001 From: Bastian Kleineidam Date: Thu, 11 Oct 2012 19:53:10 +0200 Subject: [PATCH] Improve URL retrieval. --- dosagelib/util.py | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/dosagelib/util.py b/dosagelib/util.py index e70dd101f..8183f7eab 100644 --- a/dosagelib/util.py +++ b/dosagelib/util.py @@ -160,7 +160,8 @@ def normaliseURL(url): pu[2] = '/' + '/'.join(segments) return urlparse.urlunparse(pu) -def urlopen(url, referrer=None, retries=5, retry_wait_seconds=10): +def urlopen(url, referrer=None, retries=3, retry_wait_seconds=5): + out.write('Open URL %s' % url, 2) assert retries >= 0, 'invalid retry value %r' % retries assert retry_wait_seconds > 0, 'invalid retry seconds value %r' % retry_wait_seconds # Work around urllib2 brokenness @@ -173,8 +174,9 @@ def urlopen(url, referrer=None, retries=5, retry_wait_seconds=10): while True: try: return urllib2.urlopen(req) - except IOError: - out.write('URL retrieval failed; waiting %d seconds and retrying (%d)' % (retry_wait_seconds, tries), 2) + except IOError, msg: + out.write('URL retrieval failed: %s' % msg) + out.write('waiting %d seconds and retrying (%d)' % (retry_wait_seconds, tries), 2) time.sleep(retry_wait_seconds) tries += 1 if tries >= retries: