dmoonfire/dosage
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Activity

Rename latestUrl in url

Browse source
This commit is contained in:
Bastian Kleineidam 2013-02-05 19:51:46 +01:00
parent 7f78bea1af
commit 1451047877
33 changed files with 49 additions and 59 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

12
dosagelib/helpers.py
View file

@ -24,17 +24,17 @@ def regexNamer(regex):
return _namer
def bounceStarter(latestUrl, nextSearch):
def bounceStarter(url, nextSearch):
"""Get start URL by "bouncing" back and forth one time."""
@classmethod
def _starter(cls):
"""Get bounced start URL."""
url = fetchUrl(latestUrl, cls.prevSearch, session=cls.session)
if not url:
raise ValueError("could not find prevSearch pattern %r in %s" % (cls.prevSearch.pattern, latestUrl))
url2 = fetchUrl(url, nextSearch, session=cls.session)
url1 = fetchUrl(url, cls.prevSearch, session=cls.session)
if not url1:
raise ValueError("could not find prevSearch pattern %r in %s" % (cls.prevSearch.pattern, url))
url2 = fetchUrl(url1, nextSearch, session=cls.session)
if not url2:
raise ValueError("could not find nextSearch pattern %r in %s" % (nextSearch.pattern, url))
raise ValueError("could not find nextSearch pattern %r in %s" % (nextSearch.pattern, url1))
return url2
return _starter

2
dosagelib/plugins/b.py
View file

@ -1,6 +1,6 @@
# -*- coding: iso-8859-1 -*-
# Copyright (C) 2004-2005 Tristan Seligmann and Jonathan Jacobs
# Copyright (C) 2012 Bastian Kleineidam
# Copyright (C) 2012-2013 Bastian Kleineidam
from re import compile

3
dosagelib/plugins/clonemanga.py
View file

@ -1,6 +1,6 @@
# -*- coding: iso-8859-1 -*-
# Copyright (C) 2004-2005 Tristan Seligmann and Jonathan Jacobs
# Copyright (C) 2012 Bastian Kleineidam
# Copyright (C) 2012-2013 Bastian Kleineidam
from re import compile
from ..scraper import make_scraper
from ..util import tagre, getQueryParams, fetchUrl
@ -44,6 +44,7 @@ def add(name, shortName, imageFolder=None, lastStrip=None):
prevSearch=_prevSearch,
help='Index format: n',
namer=namer,
url=baseUrl,
)
if lastStrip is None:
attrs['starter'] = _starter

2
dosagelib/plugins/e.py
View file

@ -1,6 +1,6 @@
# -*- coding: iso-8859-1 -*-
# Copyright (C) 2004-2005 Tristan Seligmann and Jonathan Jacobs
# Copyright (C) 2012 Bastian Kleineidam
# Copyright (C) 2012-2013 Bastian Kleineidam
from re import compile, IGNORECASE

8
dosagelib/plugins/f.py
View file

@ -1,6 +1,6 @@
# -*- coding: iso-8859-1 -*-
# Copyright (C) 2004-2005 Tristan Seligmann and Jonathan Jacobs
# Copyright (C) 2012 Bastian Kleineidam
# Copyright (C) 2012-2013 Bastian Kleineidam
from re import compile, IGNORECASE, MULTILINE
@ -121,11 +121,11 @@ class Fallen(_BasicScraper):
class FredoAndPidjin(_BasicScraper):
homepage = 'http://www.pidjin.net/'
url = 'http://www.pidjin.net/'
stripUrl = None
help = 'Index format: yyyy/mm/dd/name'
imageSearch = compile(tagre('img', 'src', '(http://cdn\.pidjin\.net/wp-content/uploads/\d+/\d+/[^"]+\.png)'))
multipleImagesPerStrip = True
prevSearch = compile(tagre('a', 'href', '([^"]+)')+"Prev</a>")
starter = indirectStarter(homepage,
compile(tagre('a', 'href', "("+homepage+r'\d\d\d\d/\d\d/\d\d/[^"]+/)')))
starter = indirectStarter(url,
compile(tagre('a', 'href', "("+url+r'\d\d\d\d/\d\d/\d\d/[^"]+/)')))

7
dosagelib/plugins/g.py
View file

@ -1,6 +1,6 @@
# -*- coding: iso-8859-1 -*-
# Copyright (C) 2004-2005 Tristan Seligmann and Jonathan Jacobs
# Copyright (C) 2012 Bastian Kleineidam
# Copyright (C) 2012-2013 Bastian Kleineidam
from re import compile
@ -68,9 +68,10 @@ class Goats(_BasicScraper):
class GoneWithTheBlastwave(_BasicScraper):
starter = indirectStarter('http://www.blastwave-comic.com/index.php?p=comic&nro=1',
url = 'http://www.blastwave-comic.com/index.php?p=comic&nro=1'
starter = indirectStarter(url,
compile(r'href="(index.php\?p=comic&amp;nro=\d+)"><img src="images/page/default/latest'))
stripUrl = 'http://www.blastwave-comic.com/index.php?p=comic&nro=%s'
stripUrl = url[:-1] + '%s'
imageSearch = compile(r'<img.+src=".+(/comics/.+?)"')
prevSearch = compile(r'href="(index.php\?p=comic&amp;nro=\d+)"><img src="images/page/default/previous')
help = 'Index format: n'

2
dosagelib/plugins/h.py
View file

@ -1,4 +1,4 @@
# Copyright (C) 2012 Bastian Kleineidam
# Copyright (C) 2012-2013 Bastian Kleineidam
from re import compile
from ..scraper import _BasicScraper

2
dosagelib/plugins/i.py
View file

@ -1,6 +1,6 @@
# -*- coding: iso-8859-1 -*-
# Copyright (C) 2004-2005 Tristan Seligmann and Jonathan Jacobs
# Copyright (C) 2012 Bastian Kleineidam
# Copyright (C) 2012-2013 Bastian Kleineidam
from re import compile
from ..scraper import _BasicScraper

2
dosagelib/plugins/j.py
View file

@ -1,6 +1,6 @@
# -*- coding: iso-8859-1 -*-
# Copyright (C) 2004-2005 Tristan Seligmann and Jonathan Jacobs
# Copyright (C) 2012 Bastian Kleineidam
# Copyright (C) 2012-2013 Bastian Kleineidam
from re import compile
from ..scraper import _BasicScraper

2
dosagelib/plugins/k.py
View file

@ -1,6 +1,6 @@
# -*- coding: iso-8859-1 -*-
# Copyright (C) 2004-2005 Tristan Seligmann and Jonathan Jacobs
# Copyright (C) 2012 Bastian Kleineidam
# Copyright (C) 2012-2013 Bastian Kleineidam
from re import compile, IGNORECASE
from ..scraper import _BasicScraper

2
dosagelib/plugins/m.py
View file

@ -1,6 +1,6 @@
# -*- coding: iso-8859-1 -*-
# Copyright (C) 2004-2005 Tristan Seligmann and Jonathan Jacobs
# Copyright (C) 2012 Bastian Kleineidam
# Copyright (C) 2012-2013 Bastian Kleineidam
from re import compile, IGNORECASE

2
dosagelib/plugins/n.py
View file

@ -1,6 +1,6 @@
# -*- coding: iso-8859-1 -*-
# Copyright (C) 2004-2005 Tristan Seligmann and Jonathan Jacobs
# Copyright (C) 2012 Bastian Kleineidam
# Copyright (C) 2012-2013 Bastian Kleineidam
from re import compile
from ..scraper import _BasicScraper

2
dosagelib/plugins/nuklearpower.py
View file

@ -1,6 +1,6 @@
# -*- coding: iso-8859-1 -*-
# Copyright (C) 2004-2005 Tristan Seligmann and Jonathan Jacobs
# Copyright (C) 2012 Bastian Kleineidam
# Copyright (C) 2012-2013 Bastian Kleineidam
from re import compile
from ..scraper import make_scraper

2
dosagelib/plugins/num.py
View file

@ -1,6 +1,6 @@
# -*- coding: iso-8859-1 -*-
# Copyright (C) 2004-2005 Tristan Seligmann and Jonathan Jacobs
# Copyright (C) 2012 Bastian Kleineidam
# Copyright (C) 2012-2013 Bastian Kleineidam
from re import compile

2
dosagelib/plugins/o.py
View file

@ -1,6 +1,6 @@
# -*- coding: iso-8859-1 -*-
# Copyright (C) 2004-2005 Tristan Seligmann and Jonathan Jacobs
# Copyright (C) 2012 Bastian Kleineidam
# Copyright (C) 2012-2013 Bastian Kleineidam
from re import compile
from ..scraper import _BasicScraper

2
dosagelib/plugins/pensandtales.py
View file

@ -1,6 +1,6 @@
# -*- coding: iso-8859-1 -*-
# Copyright (C) 2004-2005 Tristan Seligmann and Jonathan Jacobs
# Copyright (C) 2012 Bastian Kleineidam
# Copyright (C) 2012-2013 Bastian Kleineidam
from re import compile
from ..scraper import make_scraper
from ..util import tagre

2
dosagelib/plugins/petitesymphony.py
View file

@ -1,6 +1,6 @@
# -*- coding: iso-8859-1 -*-
# Copyright (C) 2004-2005 Tristan Seligmann and Jonathan Jacobs
# Copyright (C) 2012 Bastian Kleineidam
# Copyright (C) 2012-2013 Bastian Kleineidam
from re import compile
from ..scraper import make_scraper

2
dosagelib/plugins/q.py
View file

@ -1,6 +1,6 @@
# -*- coding: iso-8859-1 -*-
# Copyright (C) 2004-2005 Tristan Seligmann and Jonathan Jacobs
# Copyright (C) 2012 Bastian Kleineidam
# Copyright (C) 2012-2013 Bastian Kleineidam
from re import compile
from ..scraper import _BasicScraper

2
dosagelib/plugins/r.py
View file

@ -1,6 +1,6 @@
# -*- coding: iso-8859-1 -*-
# Copyright (C) 2004-2005 Tristan Seligmann and Jonathan Jacobs
# Copyright (C) 2012 Bastian Kleineidam
# Copyright (C) 2012-2013 Bastian Kleineidam
from re import compile
from ..scraper import _BasicScraper

2
dosagelib/plugins/s.py
View file

@ -1,6 +1,6 @@
# -*- coding: iso-8859-1 -*-
# Copyright (C) 2004-2005 Tristan Seligmann and Jonathan Jacobs
# Copyright (C) 2012 Bastian Kleineidam
# Copyright (C) 2012-2013 Bastian Kleineidam
from re import compile, MULTILINE, IGNORECASE, sub
from os.path import splitext

2
dosagelib/plugins/snafu.py
View file

@ -1,6 +1,6 @@
# -*- coding: iso-8859-1 -*-
# Copyright (C) 2004-2005 Tristan Seligmann and Jonathan Jacobs
# Copyright (C) 2012 Bastian Kleineidam
# Copyright (C) 2012-2013 Bastian Kleineidam
from re import compile
from ..scraper import make_scraper

2
dosagelib/plugins/t.py
View file

@ -1,6 +1,6 @@
# -*- coding: iso-8859-1 -*-
# Copyright (C) 2004-2005 Tristan Seligmann and Jonathan Jacobs
# Copyright (C) 2012 Bastian Kleineidam
# Copyright (C) 2012-2013 Bastian Kleineidam
from re import compile, IGNORECASE
from ..scraper import _BasicScraper

2
dosagelib/plugins/u.py
View file

@ -1,6 +1,6 @@
# -*- coding: iso-8859-1 -*-
# Copyright (C) 2004-2005 Tristan Seligmann and Jonathan Jacobs
# Copyright (C) 2012 Bastian Kleineidam
# Copyright (C) 2012-2013 Bastian Kleineidam
from re import compile

2
dosagelib/plugins/v.py
View file

@ -1,6 +1,6 @@
# -*- coding: iso-8859-1 -*-
# Copyright (C) 2004-2005 Tristan Seligmann and Jonathan Jacobs
# Copyright (C) 2012 Bastian Kleineidam
# Copyright (C) 2012-2013 Bastian Kleineidam
from re import compile

2
dosagelib/plugins/w.py
View file

@ -1,6 +1,6 @@
# -*- coding: iso-8859-1 -*-
# Copyright (C) 2004-2005 Tristan Seligmann and Jonathan Jacobs
# Copyright (C) 2012 Bastian Kleineidam
# Copyright (C) 2012-2013 Bastian Kleineidam
from re import compile, IGNORECASE

2
dosagelib/plugins/webcomiceu.py
View file

@ -1,5 +1,5 @@
# -*- coding: iso-8859-1 -*-
# Copyright (C) 2012 Bastian Kleineidam
# Copyright (C) 2012-2013 Bastian Kleineidam
from re import compile
from ..scraper import make_scraper

2
dosagelib/plugins/webcomicnation.py
View file

@ -1,6 +1,6 @@
# -*- coding: iso-8859-1 -*-
# Copyright (C) 2004-2005 Tristan Seligmann and Jonathan Jacobs
# Copyright (C) 2012 Bastian Kleineidam
# Copyright (C) 2012-2013 Bastian Kleineidam
from re import compile, IGNORECASE, DOTALL
from ..scraper import make_scraper

2
dosagelib/plugins/wlpcomics.py
View file

@ -1,6 +1,6 @@
# -*- coding: iso-8859-1 -*-
# Copyright (C) 2004-2005 Tristan Seligmann and Jonathan Jacobs
# Copyright (C) 2012 Bastian Kleineidam
# Copyright (C) 2012-2013 Bastian Kleineidam
from re import compile
from ..util import tagre

2
dosagelib/plugins/x.py
View file

@ -1,6 +1,6 @@
# -*- coding: iso-8859-1 -*-
# Copyright (C) 2004-2005 Tristan Seligmann and Jonathan Jacobs
# Copyright (C) 2012 Bastian Kleineidam
# Copyright (C) 2012-2013 Bastian Kleineidam
from re import compile

2
dosagelib/plugins/y.py
View file

@ -1,6 +1,6 @@
# -*- coding: iso-8859-1 -*-
# Copyright (C) 2004-2005 Tristan Seligmann and Jonathan Jacobs
# Copyright (C) 2012 Bastian Kleineidam
# Copyright (C) 2012-2013 Bastian Kleineidam
from re import compile
from ..scraper import _BasicScraper

2
dosagelib/plugins/z.py
View file

@ -1,6 +1,6 @@
# -*- coding: iso-8859-1 -*-
# Copyright (C) 2004-2005 Tristan Seligmann and Jonathan Jacobs
# Copyright (C) 2012 Bastian Kleineidam
# Copyright (C) 2012-2013 Bastian Kleineidam
from re import compile
from ..scraper import _BasicScraper

4
dosagelib/scraper.py
View file

@ -11,8 +11,8 @@ from .output import out
class _BasicScraper(object):
'''Base class with scrape functions for comics.
@type latestUrl: C{string}
@cvar latestUrl: The URL for the latest comic strip.
@type url: C{string}
@cvar url: The URL for the comic strip.
@type stripUrl: C{string}
@cvar stripUrl: A string that is interpolated with the strip index
to yield the URL for a particular strip.

20
scripts/mktestpage.py
View file

@ -36,13 +36,7 @@ description: a list of comic strips supported by Dosage
<div id="comics">
%(content)s
</div>
<script type="text/javascript">
(function() {
var po = document.createElement('script'); po.type = 'text/javascript'; po.async = true;
po.src = 'https://apis.google.com/js/plusone.js';
var s = document.getElementsByTagName('script')[0]; s.parentNode.insertBefore(po, s);
})();
</script>
<script type="text/javascript" src="https://apis.google.com/js/plusone.js"></script>
<script>
window.onload = function() {
var wall = new Masonry(document.getElementById('comics'), {
@ -81,13 +75,7 @@ title: Dosage comic %(name)s
</tr>
</table>
<div class="g-plusone" data-size="standard" data-annotation="inline" data-width="300"></div>
<script type="text/javascript">
(function() {
var po = document.createElement('script'); po.type = 'text/javascript'; po.async = true;
po.src = 'https://apis.google.com/js/plusone.js';
var s = document.getElementsByTagName('script')[0]; s.parentNode.insertBefore(po, s);
})();
</script>
<script type="text/javascript" src="https://apis.google.com/js/plusone.js"></script>
</section>
{%% endblock content %%}
"""
@ -153,10 +141,10 @@ def get_testinfo(filename, modified):
key, entry = get_testentry(line)
keys.append(key)
update_testentry(key, entry, testinfo)
if num_tests % 5 == 0:
print(num_tests, end=" ", file=sys.stderr)
elif add_error and line.startswith(" E "):
entry["error"] = line[3:].strip()
if num_tests % 5 == 0:
print(num_tests, end=" ", file=sys.stderr)
orphan_entries(keys, testinfo)
save_result(testinfo, json_file)
return testinfo