dmoonfire/dosage
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Activity

Add option to wait before downloading.

Browse source
This commit is contained in:
Bastian Kleineidam 2013-03-08 06:46:50 +01:00
parent 0ee0822e00
commit 4c344765ff
1 changed files with 6 additions and 0 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

6
dosagelib/scraper.py
View file

@ -2,6 +2,7 @@
# Copyright (C) 2004-2005 Tristan Seligmann and Jonathan Jacobs # Copyright (C) 2004-2005 Tristan Seligmann and Jonathan Jacobs
# Copyright (C) 2012-2013 Bastian Kleineidam # Copyright (C) 2012-2013 Bastian Kleineidam
import requests import requests
import time
from . import loader from . import loader
from .util import fetchUrl, fetchUrls, getPageContent from .util import fetchUrl, fetchUrls, getPageContent
from .comic import ComicStrip from .comic import ComicStrip
@ -41,6 +42,9 @@ class _BasicScraper(object):
# usually the index format help # usually the index format help
help = '' help = ''
# wait time before downloading any pages or images
waitSeconds = 0
# HTTP session storing cookies # HTTP session storing cookies
session = requests.session() session = requests.session()
@ -133,6 +137,8 @@ class _BasicScraper(object):
out.warn("Already seen previous URL %r" % prevUrl) out.warn("Already seen previous URL %r" % prevUrl)
break break
url = prevUrl url = prevUrl
if self.waitSeconds:
time.sleep(self.waitSeconds)
def getPrevUrl(self, url, data, baseUrl): def getPrevUrl(self, url, data, baseUrl):
"""Find previous URL.""" """Find previous URL."""