From 14510478777d6aa26d91efe860ae2a62302645ec Mon Sep 17 00:00:00 2001 From: Bastian Kleineidam Date: Tue, 5 Feb 2013 19:51:46 +0100 Subject: [PATCH] Rename latestUrl in url --- dosagelib/helpers.py | 12 ++++++------ dosagelib/plugins/b.py | 2 +- dosagelib/plugins/clonemanga.py | 3 ++- dosagelib/plugins/e.py | 2 +- dosagelib/plugins/f.py | 8 ++++---- dosagelib/plugins/g.py | 7 ++++--- dosagelib/plugins/h.py | 2 +- dosagelib/plugins/i.py | 2 +- dosagelib/plugins/j.py | 2 +- dosagelib/plugins/k.py | 2 +- dosagelib/plugins/m.py | 2 +- dosagelib/plugins/n.py | 2 +- dosagelib/plugins/nuklearpower.py | 2 +- dosagelib/plugins/num.py | 2 +- dosagelib/plugins/o.py | 2 +- dosagelib/plugins/pensandtales.py | 2 +- dosagelib/plugins/petitesymphony.py | 2 +- dosagelib/plugins/q.py | 2 +- dosagelib/plugins/r.py | 2 +- dosagelib/plugins/s.py | 2 +- dosagelib/plugins/snafu.py | 2 +- dosagelib/plugins/t.py | 2 +- dosagelib/plugins/u.py | 2 +- dosagelib/plugins/v.py | 2 +- dosagelib/plugins/w.py | 2 +- dosagelib/plugins/webcomiceu.py | 2 +- dosagelib/plugins/webcomicnation.py | 2 +- dosagelib/plugins/wlpcomics.py | 2 +- dosagelib/plugins/x.py | 2 +- dosagelib/plugins/y.py | 2 +- dosagelib/plugins/z.py | 2 +- dosagelib/scraper.py | 4 ++-- scripts/mktestpage.py | 20 ++++---------------- 33 files changed, 49 insertions(+), 59 deletions(-) diff --git a/dosagelib/helpers.py b/dosagelib/helpers.py index 9e35a4da1..209d36764 100644 --- a/dosagelib/helpers.py +++ b/dosagelib/helpers.py @@ -24,17 +24,17 @@ def regexNamer(regex): return _namer -def bounceStarter(latestUrl, nextSearch): +def bounceStarter(url, nextSearch): """Get start URL by "bouncing" back and forth one time.""" @classmethod def _starter(cls): """Get bounced start URL.""" - url = fetchUrl(latestUrl, cls.prevSearch, session=cls.session) - if not url: - raise ValueError("could not find prevSearch pattern %r in %s" % (cls.prevSearch.pattern, latestUrl)) - url2 = fetchUrl(url, nextSearch, session=cls.session) + url1 = fetchUrl(url, cls.prevSearch, session=cls.session) + if not url1: + raise ValueError("could not find prevSearch pattern %r in %s" % (cls.prevSearch.pattern, url)) + url2 = fetchUrl(url1, nextSearch, session=cls.session) if not url2: - raise ValueError("could not find nextSearch pattern %r in %s" % (nextSearch.pattern, url)) + raise ValueError("could not find nextSearch pattern %r in %s" % (nextSearch.pattern, url1)) return url2 return _starter diff --git a/dosagelib/plugins/b.py b/dosagelib/plugins/b.py index 93fa52f3c..f9229ff81 100644 --- a/dosagelib/plugins/b.py +++ b/dosagelib/plugins/b.py @@ -1,6 +1,6 @@ # -*- coding: iso-8859-1 -*- # Copyright (C) 2004-2005 Tristan Seligmann and Jonathan Jacobs -# Copyright (C) 2012 Bastian Kleineidam +# Copyright (C) 2012-2013 Bastian Kleineidam from re import compile diff --git a/dosagelib/plugins/clonemanga.py b/dosagelib/plugins/clonemanga.py index 8a50f27eb..00f191c31 100644 --- a/dosagelib/plugins/clonemanga.py +++ b/dosagelib/plugins/clonemanga.py @@ -1,6 +1,6 @@ # -*- coding: iso-8859-1 -*- # Copyright (C) 2004-2005 Tristan Seligmann and Jonathan Jacobs -# Copyright (C) 2012 Bastian Kleineidam +# Copyright (C) 2012-2013 Bastian Kleineidam from re import compile from ..scraper import make_scraper from ..util import tagre, getQueryParams, fetchUrl @@ -44,6 +44,7 @@ def add(name, shortName, imageFolder=None, lastStrip=None): prevSearch=_prevSearch, help='Index format: n', namer=namer, + url=baseUrl, ) if lastStrip is None: attrs['starter'] = _starter diff --git a/dosagelib/plugins/e.py b/dosagelib/plugins/e.py index 1442db73e..dba8e66c6 100644 --- a/dosagelib/plugins/e.py +++ b/dosagelib/plugins/e.py @@ -1,6 +1,6 @@ # -*- coding: iso-8859-1 -*- # Copyright (C) 2004-2005 Tristan Seligmann and Jonathan Jacobs -# Copyright (C) 2012 Bastian Kleineidam +# Copyright (C) 2012-2013 Bastian Kleineidam from re import compile, IGNORECASE diff --git a/dosagelib/plugins/f.py b/dosagelib/plugins/f.py index 59deb08ed..5ff397156 100644 --- a/dosagelib/plugins/f.py +++ b/dosagelib/plugins/f.py @@ -1,6 +1,6 @@ # -*- coding: iso-8859-1 -*- # Copyright (C) 2004-2005 Tristan Seligmann and Jonathan Jacobs -# Copyright (C) 2012 Bastian Kleineidam +# Copyright (C) 2012-2013 Bastian Kleineidam from re import compile, IGNORECASE, MULTILINE @@ -121,11 +121,11 @@ class Fallen(_BasicScraper): class FredoAndPidjin(_BasicScraper): - homepage = 'http://www.pidjin.net/' + url = 'http://www.pidjin.net/' stripUrl = None help = 'Index format: yyyy/mm/dd/name' imageSearch = compile(tagre('img', 'src', '(http://cdn\.pidjin\.net/wp-content/uploads/\d+/\d+/[^"]+\.png)')) multipleImagesPerStrip = True prevSearch = compile(tagre('a', 'href', '([^"]+)')+"Prev") - starter = indirectStarter(homepage, - compile(tagre('a', 'href', "("+homepage+r'\d\d\d\d/\d\d/\d\d/[^"]+/)'))) + starter = indirectStarter(url, + compile(tagre('a', 'href', "("+url+r'\d\d\d\d/\d\d/\d\d/[^"]+/)'))) diff --git a/dosagelib/plugins/g.py b/dosagelib/plugins/g.py index 45f0cfa74..0e14ac5e8 100644 --- a/dosagelib/plugins/g.py +++ b/dosagelib/plugins/g.py @@ -1,6 +1,6 @@ # -*- coding: iso-8859-1 -*- # Copyright (C) 2004-2005 Tristan Seligmann and Jonathan Jacobs -# Copyright (C) 2012 Bastian Kleineidam +# Copyright (C) 2012-2013 Bastian Kleineidam from re import compile @@ -68,9 +68,10 @@ class Goats(_BasicScraper): class GoneWithTheBlastwave(_BasicScraper): - starter = indirectStarter('http://www.blastwave-comic.com/index.php?p=comic&nro=1', + url = 'http://www.blastwave-comic.com/index.php?p=comic&nro=1' + starter = indirectStarter(url, compile(r'href="(index.php\?p=comic&nro=\d+)"> %(content)s - + + {%% endblock content %%} """ @@ -153,10 +141,10 @@ def get_testinfo(filename, modified): key, entry = get_testentry(line) keys.append(key) update_testentry(key, entry, testinfo) + if num_tests % 5 == 0: + print(num_tests, end=" ", file=sys.stderr) elif add_error and line.startswith(" E "): entry["error"] = line[3:].strip() - if num_tests % 5 == 0: - print(num_tests, end=" ", file=sys.stderr) orphan_entries(keys, testinfo) save_result(testinfo, json_file) return testinfo