dmoonfire/dosage
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Activity

Improve URL retrieval.

Browse source
This commit is contained in:
Bastian Kleineidam 2012-10-11 19:53:10 +02:00
parent 65d881eee5
commit 78f44e9d9c
1 changed files with 5 additions and 3 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

8
dosagelib/util.py
View file

@ -160,7 +160,8 @@ def normaliseURL(url):
pu[2] = '/' + '/'.join(segments) pu[2] = '/' + '/'.join(segments)
return urlparse.urlunparse(pu) return urlparse.urlunparse(pu)
def urlopen(url, referrer=None, retries=5, retry_wait_seconds=10): def urlopen(url, referrer=None, retries=3, retry_wait_seconds=5):
out.write('Open URL %s' % url, 2)
assert retries >= 0, 'invalid retry value %r' % retries assert retries >= 0, 'invalid retry value %r' % retries
assert retry_wait_seconds > 0, 'invalid retry seconds value %r' % retry_wait_seconds assert retry_wait_seconds > 0, 'invalid retry seconds value %r' % retry_wait_seconds
# Work around urllib2 brokenness # Work around urllib2 brokenness
@ -173,8 +174,9 @@ def urlopen(url, referrer=None, retries=5, retry_wait_seconds=10):
while True: while True:
try: try:
return urllib2.urlopen(req) return urllib2.urlopen(req)
except IOError: except IOError, msg:
out.write('URL retrieval failed; waiting %d seconds and retrying (%d)' % (retry_wait_seconds, tries), 2) out.write('URL retrieval failed: %s' % msg)
out.write('waiting %d seconds and retrying (%d)' % (retry_wait_seconds, tries), 2)
time.sleep(retry_wait_seconds) time.sleep(retry_wait_seconds)
tries += 1 tries += 1
if tries >= retries: if tries >= retries: