dmoonfire/dosage
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Activity

Rename latestUrl in url

Browse source
This commit is contained in:
Bastian Kleineidam 2013-02-05 19:51:46 +01:00
parent 7f78bea1af
commit 1451047877
33 changed files with 49 additions and 59 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

12
dosagelib/helpers.py
View file

@ -24,17 +24,17 @@ def regexNamer(regex):
return _namer return _namer
def bounceStarter(latestUrl, nextSearch): def bounceStarter(url, nextSearch):
"""Get start URL by "bouncing" back and forth one time.""" """Get start URL by "bouncing" back and forth one time."""
@classmethod @classmethod
def _starter(cls): def _starter(cls):
"""Get bounced start URL.""" """Get bounced start URL."""
url = fetchUrl(latestUrl, cls.prevSearch, session=cls.session) url1 = fetchUrl(url, cls.prevSearch, session=cls.session)
if not url: if not url1:
raise ValueError("could not find prevSearch pattern %r in %s" % (cls.prevSearch.pattern, latestUrl)) raise ValueError("could not find prevSearch pattern %r in %s" % (cls.prevSearch.pattern, url))
url2 = fetchUrl(url, nextSearch, session=cls.session) url2 = fetchUrl(url1, nextSearch, session=cls.session)
if not url2: if not url2:
raise ValueError("could not find nextSearch pattern %r in %s" % (nextSearch.pattern, url)) raise ValueError("could not find nextSearch pattern %r in %s" % (nextSearch.pattern, url1))
return url2 return url2
return _starter return _starter

2
dosagelib/plugins/b.py
View file

@ -1,6 +1,6 @@
# -*- coding: iso-8859-1 -*- # -*- coding: iso-8859-1 -*-
# Copyright (C) 2004-2005 Tristan Seligmann and Jonathan Jacobs # Copyright (C) 2004-2005 Tristan Seligmann and Jonathan Jacobs
# Copyright (C) 2012 Bastian Kleineidam # Copyright (C) 2012-2013 Bastian Kleineidam
from re import compile from re import compile

3
dosagelib/plugins/clonemanga.py
View file

@ -1,6 +1,6 @@
# -*- coding: iso-8859-1 -*- # -*- coding: iso-8859-1 -*-
# Copyright (C) 2004-2005 Tristan Seligmann and Jonathan Jacobs # Copyright (C) 2004-2005 Tristan Seligmann and Jonathan Jacobs
# Copyright (C) 2012 Bastian Kleineidam # Copyright (C) 2012-2013 Bastian Kleineidam
from re import compile from re import compile
from ..scraper import make_scraper from ..scraper import make_scraper
from ..util import tagre, getQueryParams, fetchUrl from ..util import tagre, getQueryParams, fetchUrl
@ -44,6 +44,7 @@ def add(name, shortName, imageFolder=None, lastStrip=None):
prevSearch=_prevSearch, prevSearch=_prevSearch,
help='Index format: n', help='Index format: n',
namer=namer, namer=namer,
url=baseUrl,
) )
if lastStrip is None: if lastStrip is None:
attrs['starter'] = _starter attrs['starter'] = _starter

2
dosagelib/plugins/e.py
View file

@ -1,6 +1,6 @@
# -*- coding: iso-8859-1 -*- # -*- coding: iso-8859-1 -*-
# Copyright (C) 2004-2005 Tristan Seligmann and Jonathan Jacobs # Copyright (C) 2004-2005 Tristan Seligmann and Jonathan Jacobs
# Copyright (C) 2012 Bastian Kleineidam # Copyright (C) 2012-2013 Bastian Kleineidam
from re import compile, IGNORECASE from re import compile, IGNORECASE

8
dosagelib/plugins/f.py
View file

@ -1,6 +1,6 @@
# -*- coding: iso-8859-1 -*- # -*- coding: iso-8859-1 -*-
# Copyright (C) 2004-2005 Tristan Seligmann and Jonathan Jacobs # Copyright (C) 2004-2005 Tristan Seligmann and Jonathan Jacobs
# Copyright (C) 2012 Bastian Kleineidam # Copyright (C) 2012-2013 Bastian Kleineidam
from re import compile, IGNORECASE, MULTILINE from re import compile, IGNORECASE, MULTILINE
@ -121,11 +121,11 @@ class Fallen(_BasicScraper):
class FredoAndPidjin(_BasicScraper): class FredoAndPidjin(_BasicScraper):
homepage = 'http://www.pidjin.net/' url = 'http://www.pidjin.net/'
stripUrl = None stripUrl = None
help = 'Index format: yyyy/mm/dd/name' help = 'Index format: yyyy/mm/dd/name'
imageSearch = compile(tagre('img', 'src', '(http://cdn\.pidjin\.net/wp-content/uploads/\d+/\d+/[^"]+\.png)')) imageSearch = compile(tagre('img', 'src', '(http://cdn\.pidjin\.net/wp-content/uploads/\d+/\d+/[^"]+\.png)'))
multipleImagesPerStrip = True multipleImagesPerStrip = True
prevSearch = compile(tagre('a', 'href', '([^"]+)')+"Prev</a>") prevSearch = compile(tagre('a', 'href', '([^"]+)')+"Prev</a>")
starter = indirectStarter(homepage, starter = indirectStarter(url,
compile(tagre('a', 'href', "("+homepage+r'\d\d\d\d/\d\d/\d\d/[^"]+/)'))) compile(tagre('a', 'href', "("+url+r'\d\d\d\d/\d\d/\d\d/[^"]+/)')))

7
dosagelib/plugins/g.py
View file

@ -1,6 +1,6 @@
# -*- coding: iso-8859-1 -*- # -*- coding: iso-8859-1 -*-
# Copyright (C) 2004-2005 Tristan Seligmann and Jonathan Jacobs # Copyright (C) 2004-2005 Tristan Seligmann and Jonathan Jacobs
# Copyright (C) 2012 Bastian Kleineidam # Copyright (C) 2012-2013 Bastian Kleineidam
from re import compile from re import compile
@ -68,9 +68,10 @@ class Goats(_BasicScraper):
class GoneWithTheBlastwave(_BasicScraper): class GoneWithTheBlastwave(_BasicScraper):
starter = indirectStarter('http://www.blastwave-comic.com/index.php?p=comic&nro=1', url = 'http://www.blastwave-comic.com/index.php?p=comic&nro=1'
starter = indirectStarter(url,
compile(r'href="(index.php\?p=comic&amp;nro=\d+)"><img src="images/page/default/latest')) compile(r'href="(index.php\?p=comic&amp;nro=\d+)"><img src="images/page/default/latest'))
stripUrl = 'http://www.blastwave-comic.com/index.php?p=comic&nro=%s' stripUrl = url[:-1] + '%s'
imageSearch = compile(r'<img.+src=".+(/comics/.+?)"') imageSearch = compile(r'<img.+src=".+(/comics/.+?)"')
prevSearch = compile(r'href="(index.php\?p=comic&amp;nro=\d+)"><img src="images/page/default/previous') prevSearch = compile(r'href="(index.php\?p=comic&amp;nro=\d+)"><img src="images/page/default/previous')
help = 'Index format: n' help = 'Index format: n'

2
dosagelib/plugins/h.py
View file

@ -1,4 +1,4 @@
# Copyright (C) 2012 Bastian Kleineidam # Copyright (C) 2012-2013 Bastian Kleineidam
from re import compile from re import compile
from ..scraper import _BasicScraper from ..scraper import _BasicScraper

2
dosagelib/plugins/i.py
View file

@ -1,6 +1,6 @@
# -*- coding: iso-8859-1 -*- # -*- coding: iso-8859-1 -*-
# Copyright (C) 2004-2005 Tristan Seligmann and Jonathan Jacobs # Copyright (C) 2004-2005 Tristan Seligmann and Jonathan Jacobs
# Copyright (C) 2012 Bastian Kleineidam # Copyright (C) 2012-2013 Bastian Kleineidam
from re import compile from re import compile
from ..scraper import _BasicScraper from ..scraper import _BasicScraper

2
dosagelib/plugins/j.py
View file

@ -1,6 +1,6 @@
# -*- coding: iso-8859-1 -*- # -*- coding: iso-8859-1 -*-
# Copyright (C) 2004-2005 Tristan Seligmann and Jonathan Jacobs # Copyright (C) 2004-2005 Tristan Seligmann and Jonathan Jacobs
# Copyright (C) 2012 Bastian Kleineidam # Copyright (C) 2012-2013 Bastian Kleineidam
from re import compile from re import compile
from ..scraper import _BasicScraper from ..scraper import _BasicScraper

2
dosagelib/plugins/k.py
View file

@ -1,6 +1,6 @@
# -*- coding: iso-8859-1 -*- # -*- coding: iso-8859-1 -*-
# Copyright (C) 2004-2005 Tristan Seligmann and Jonathan Jacobs # Copyright (C) 2004-2005 Tristan Seligmann and Jonathan Jacobs
# Copyright (C) 2012 Bastian Kleineidam # Copyright (C) 2012-2013 Bastian Kleineidam
from re import compile, IGNORECASE from re import compile, IGNORECASE
from ..scraper import _BasicScraper from ..scraper import _BasicScraper

2
dosagelib/plugins/m.py
View file

@ -1,6 +1,6 @@
# -*- coding: iso-8859-1 -*- # -*- coding: iso-8859-1 -*-
# Copyright (C) 2004-2005 Tristan Seligmann and Jonathan Jacobs # Copyright (C) 2004-2005 Tristan Seligmann and Jonathan Jacobs
# Copyright (C) 2012 Bastian Kleineidam # Copyright (C) 2012-2013 Bastian Kleineidam
from re import compile, IGNORECASE from re import compile, IGNORECASE

2
dosagelib/plugins/n.py
View file

@ -1,6 +1,6 @@
# -*- coding: iso-8859-1 -*- # -*- coding: iso-8859-1 -*-
# Copyright (C) 2004-2005 Tristan Seligmann and Jonathan Jacobs # Copyright (C) 2004-2005 Tristan Seligmann and Jonathan Jacobs
# Copyright (C) 2012 Bastian Kleineidam # Copyright (C) 2012-2013 Bastian Kleineidam
from re import compile from re import compile
from ..scraper import _BasicScraper from ..scraper import _BasicScraper

2
dosagelib/plugins/nuklearpower.py
View file

@ -1,6 +1,6 @@
# -*- coding: iso-8859-1 -*- # -*- coding: iso-8859-1 -*-
# Copyright (C) 2004-2005 Tristan Seligmann and Jonathan Jacobs # Copyright (C) 2004-2005 Tristan Seligmann and Jonathan Jacobs
# Copyright (C) 2012 Bastian Kleineidam # Copyright (C) 2012-2013 Bastian Kleineidam
from re import compile from re import compile
from ..scraper import make_scraper from ..scraper import make_scraper

2
dosagelib/plugins/num.py
View file

@ -1,6 +1,6 @@
# -*- coding: iso-8859-1 -*- # -*- coding: iso-8859-1 -*-
# Copyright (C) 2004-2005 Tristan Seligmann and Jonathan Jacobs # Copyright (C) 2004-2005 Tristan Seligmann and Jonathan Jacobs
# Copyright (C) 2012 Bastian Kleineidam # Copyright (C) 2012-2013 Bastian Kleineidam
from re import compile from re import compile

2
dosagelib/plugins/o.py
View file

@ -1,6 +1,6 @@
# -*- coding: iso-8859-1 -*- # -*- coding: iso-8859-1 -*-
# Copyright (C) 2004-2005 Tristan Seligmann and Jonathan Jacobs # Copyright (C) 2004-2005 Tristan Seligmann and Jonathan Jacobs
# Copyright (C) 2012 Bastian Kleineidam # Copyright (C) 2012-2013 Bastian Kleineidam
from re import compile from re import compile
from ..scraper import _BasicScraper from ..scraper import _BasicScraper

2
dosagelib/plugins/pensandtales.py
View file

@ -1,6 +1,6 @@
# -*- coding: iso-8859-1 -*- # -*- coding: iso-8859-1 -*-
# Copyright (C) 2004-2005 Tristan Seligmann and Jonathan Jacobs # Copyright (C) 2004-2005 Tristan Seligmann and Jonathan Jacobs
# Copyright (C) 2012 Bastian Kleineidam # Copyright (C) 2012-2013 Bastian Kleineidam
from re import compile from re import compile
from ..scraper import make_scraper from ..scraper import make_scraper
from ..util import tagre from ..util import tagre

2
dosagelib/plugins/petitesymphony.py
View file

@ -1,6 +1,6 @@
# -*- coding: iso-8859-1 -*- # -*- coding: iso-8859-1 -*-
# Copyright (C) 2004-2005 Tristan Seligmann and Jonathan Jacobs # Copyright (C) 2004-2005 Tristan Seligmann and Jonathan Jacobs
# Copyright (C) 2012 Bastian Kleineidam # Copyright (C) 2012-2013 Bastian Kleineidam
from re import compile from re import compile
from ..scraper import make_scraper from ..scraper import make_scraper

2
dosagelib/plugins/q.py
View file

@ -1,6 +1,6 @@
# -*- coding: iso-8859-1 -*- # -*- coding: iso-8859-1 -*-
# Copyright (C) 2004-2005 Tristan Seligmann and Jonathan Jacobs # Copyright (C) 2004-2005 Tristan Seligmann and Jonathan Jacobs
# Copyright (C) 2012 Bastian Kleineidam # Copyright (C) 2012-2013 Bastian Kleineidam
from re import compile from re import compile
from ..scraper import _BasicScraper from ..scraper import _BasicScraper

2
dosagelib/plugins/r.py
View file

@ -1,6 +1,6 @@
# -*- coding: iso-8859-1 -*- # -*- coding: iso-8859-1 -*-
# Copyright (C) 2004-2005 Tristan Seligmann and Jonathan Jacobs # Copyright (C) 2004-2005 Tristan Seligmann and Jonathan Jacobs
# Copyright (C) 2012 Bastian Kleineidam # Copyright (C) 2012-2013 Bastian Kleineidam
from re import compile from re import compile
from ..scraper import _BasicScraper from ..scraper import _BasicScraper

2
dosagelib/plugins/s.py
View file

@ -1,6 +1,6 @@
# -*- coding: iso-8859-1 -*- # -*- coding: iso-8859-1 -*-
# Copyright (C) 2004-2005 Tristan Seligmann and Jonathan Jacobs # Copyright (C) 2004-2005 Tristan Seligmann and Jonathan Jacobs
# Copyright (C) 2012 Bastian Kleineidam # Copyright (C) 2012-2013 Bastian Kleineidam
from re import compile, MULTILINE, IGNORECASE, sub from re import compile, MULTILINE, IGNORECASE, sub
from os.path import splitext from os.path import splitext

2
dosagelib/plugins/snafu.py
View file

@ -1,6 +1,6 @@
# -*- coding: iso-8859-1 -*- # -*- coding: iso-8859-1 -*-
# Copyright (C) 2004-2005 Tristan Seligmann and Jonathan Jacobs # Copyright (C) 2004-2005 Tristan Seligmann and Jonathan Jacobs
# Copyright (C) 2012 Bastian Kleineidam # Copyright (C) 2012-2013 Bastian Kleineidam
from re import compile from re import compile
from ..scraper import make_scraper from ..scraper import make_scraper

2
dosagelib/plugins/t.py
View file

@ -1,6 +1,6 @@
# -*- coding: iso-8859-1 -*- # -*- coding: iso-8859-1 -*-
# Copyright (C) 2004-2005 Tristan Seligmann and Jonathan Jacobs # Copyright (C) 2004-2005 Tristan Seligmann and Jonathan Jacobs
# Copyright (C) 2012 Bastian Kleineidam # Copyright (C) 2012-2013 Bastian Kleineidam
from re import compile, IGNORECASE from re import compile, IGNORECASE
from ..scraper import _BasicScraper from ..scraper import _BasicScraper

2
dosagelib/plugins/u.py
View file

@ -1,6 +1,6 @@
# -*- coding: iso-8859-1 -*- # -*- coding: iso-8859-1 -*-
# Copyright (C) 2004-2005 Tristan Seligmann and Jonathan Jacobs # Copyright (C) 2004-2005 Tristan Seligmann and Jonathan Jacobs
# Copyright (C) 2012 Bastian Kleineidam # Copyright (C) 2012-2013 Bastian Kleineidam
from re import compile from re import compile

2
dosagelib/plugins/v.py
View file

@ -1,6 +1,6 @@
# -*- coding: iso-8859-1 -*- # -*- coding: iso-8859-1 -*-
# Copyright (C) 2004-2005 Tristan Seligmann and Jonathan Jacobs # Copyright (C) 2004-2005 Tristan Seligmann and Jonathan Jacobs
# Copyright (C) 2012 Bastian Kleineidam # Copyright (C) 2012-2013 Bastian Kleineidam
from re import compile from re import compile

2
dosagelib/plugins/w.py
View file

@ -1,6 +1,6 @@
# -*- coding: iso-8859-1 -*- # -*- coding: iso-8859-1 -*-
# Copyright (C) 2004-2005 Tristan Seligmann and Jonathan Jacobs # Copyright (C) 2004-2005 Tristan Seligmann and Jonathan Jacobs
# Copyright (C) 2012 Bastian Kleineidam # Copyright (C) 2012-2013 Bastian Kleineidam
from re import compile, IGNORECASE from re import compile, IGNORECASE

2
dosagelib/plugins/webcomiceu.py
View file

@ -1,5 +1,5 @@
# -*- coding: iso-8859-1 -*- # -*- coding: iso-8859-1 -*-
# Copyright (C) 2012 Bastian Kleineidam # Copyright (C) 2012-2013 Bastian Kleineidam
from re import compile from re import compile
from ..scraper import make_scraper from ..scraper import make_scraper

2
dosagelib/plugins/webcomicnation.py
View file

@ -1,6 +1,6 @@
# -*- coding: iso-8859-1 -*- # -*- coding: iso-8859-1 -*-
# Copyright (C) 2004-2005 Tristan Seligmann and Jonathan Jacobs # Copyright (C) 2004-2005 Tristan Seligmann and Jonathan Jacobs
# Copyright (C) 2012 Bastian Kleineidam # Copyright (C) 2012-2013 Bastian Kleineidam
from re import compile, IGNORECASE, DOTALL from re import compile, IGNORECASE, DOTALL
from ..scraper import make_scraper from ..scraper import make_scraper

2
dosagelib/plugins/wlpcomics.py
View file

@ -1,6 +1,6 @@
# -*- coding: iso-8859-1 -*- # -*- coding: iso-8859-1 -*-
# Copyright (C) 2004-2005 Tristan Seligmann and Jonathan Jacobs # Copyright (C) 2004-2005 Tristan Seligmann and Jonathan Jacobs
# Copyright (C) 2012 Bastian Kleineidam # Copyright (C) 2012-2013 Bastian Kleineidam
from re import compile from re import compile
from ..util import tagre from ..util import tagre

2
dosagelib/plugins/x.py
View file

@ -1,6 +1,6 @@
# -*- coding: iso-8859-1 -*- # -*- coding: iso-8859-1 -*-
# Copyright (C) 2004-2005 Tristan Seligmann and Jonathan Jacobs # Copyright (C) 2004-2005 Tristan Seligmann and Jonathan Jacobs
# Copyright (C) 2012 Bastian Kleineidam # Copyright (C) 2012-2013 Bastian Kleineidam
from re import compile from re import compile

2
dosagelib/plugins/y.py
View file

@ -1,6 +1,6 @@
# -*- coding: iso-8859-1 -*- # -*- coding: iso-8859-1 -*-
# Copyright (C) 2004-2005 Tristan Seligmann and Jonathan Jacobs # Copyright (C) 2004-2005 Tristan Seligmann and Jonathan Jacobs
# Copyright (C) 2012 Bastian Kleineidam # Copyright (C) 2012-2013 Bastian Kleineidam
from re import compile from re import compile
from ..scraper import _BasicScraper from ..scraper import _BasicScraper

2
dosagelib/plugins/z.py
View file

@ -1,6 +1,6 @@
# -*- coding: iso-8859-1 -*- # -*- coding: iso-8859-1 -*-
# Copyright (C) 2004-2005 Tristan Seligmann and Jonathan Jacobs # Copyright (C) 2004-2005 Tristan Seligmann and Jonathan Jacobs
# Copyright (C) 2012 Bastian Kleineidam # Copyright (C) 2012-2013 Bastian Kleineidam
from re import compile from re import compile
from ..scraper import _BasicScraper from ..scraper import _BasicScraper

4
dosagelib/scraper.py
View file

@ -11,8 +11,8 @@ from .output import out
class _BasicScraper(object): class _BasicScraper(object):
'''Base class with scrape functions for comics. '''Base class with scrape functions for comics.
@type latestUrl: C{string} @type url: C{string}
@cvar latestUrl: The URL for the latest comic strip. @cvar url: The URL for the comic strip.
@type stripUrl: C{string} @type stripUrl: C{string}
@cvar stripUrl: A string that is interpolated with the strip index @cvar stripUrl: A string that is interpolated with the strip index
to yield the URL for a particular strip. to yield the URL for a particular strip.

20
scripts/mktestpage.py
View file

@ -36,13 +36,7 @@ description: a list of comic strips supported by Dosage
<div id="comics"> <div id="comics">
%(content)s %(content)s
</div> </div>
<script type="text/javascript"> <script type="text/javascript" src="https://apis.google.com/js/plusone.js"></script>
(function() {
var po = document.createElement('script'); po.type = 'text/javascript'; po.async = true;
po.src = 'https://apis.google.com/js/plusone.js';
var s = document.getElementsByTagName('script')[0]; s.parentNode.insertBefore(po, s);
})();
</script>
<script> <script>
window.onload = function() { window.onload = function() {
var wall = new Masonry(document.getElementById('comics'), { var wall = new Masonry(document.getElementById('comics'), {
@ -81,13 +75,7 @@ title: Dosage comic %(name)s
</tr> </tr>
</table> </table>
<div class="g-plusone" data-size="standard" data-annotation="inline" data-width="300"></div> <div class="g-plusone" data-size="standard" data-annotation="inline" data-width="300"></div>
<script type="text/javascript"> <script type="text/javascript" src="https://apis.google.com/js/plusone.js"></script>
(function() {
var po = document.createElement('script'); po.type = 'text/javascript'; po.async = true;
po.src = 'https://apis.google.com/js/plusone.js';
var s = document.getElementsByTagName('script')[0]; s.parentNode.insertBefore(po, s);
})();
</script>
</section> </section>
{%% endblock content %%} {%% endblock content %%}
""" """
@ -153,10 +141,10 @@ def get_testinfo(filename, modified):
key, entry = get_testentry(line) key, entry = get_testentry(line)
keys.append(key) keys.append(key)
update_testentry(key, entry, testinfo) update_testentry(key, entry, testinfo)
elif add_error and line.startswith(" E "):
entry["error"] = line[3:].strip()
if num_tests % 5 == 0: if num_tests % 5 == 0:
print(num_tests, end=" ", file=sys.stderr) print(num_tests, end=" ", file=sys.stderr)
elif add_error and line.startswith(" E "):
entry["error"] = line[3:].strip()
orphan_entries(keys, testinfo) orphan_entries(keys, testinfo)
save_result(testinfo, json_file) save_result(testinfo, json_file)
return testinfo return testinfo