Another round of comic module fixes.

dosagelib/plugins/c.py

@@ -10,7 +10,7 @@ from re import compile, escape
 from ..scraper import _BasicScraper, _ParserScraper
 from ..helpers import bounceStarter, indirectStarter
 from ..util import tagre
-from .common import _WordPressScraper, xpath_class
+from .common import _TumblrScraper, _WordPressScraper, xpath_class
 
 
 class Caggage(_BasicScraper):
@@ -207,24 +207,14 @@ class CigarroAndCerveja(_ParserScraper):
     prevSearch = '//a[contains(text()," Prev")]',
 
 
-class Collar6(_ParserScraper):
+class Collar6(_TumblrScraper):
     url = 'http://collar6.tumblr.com/'
     firstStripUrl = url + 'post/138117470810/the-very-first-strip-from-when-i-thought-it-was'
     imageSearch = '//figure[@class="photo-hires-item"]//img'
     prevSearch = '//a[@class="previous-button"]'
     latestSearch = '//li[@class="timestamp"]/a'
-    starter = indirectStarter
     adult = True
 
-    def namer(self, image_url, page_url):
-        # tumblr URLs: http://host/post/num/name
-        #              0    1 2    3    4   5
-        parts = page_url.split('/')
-        if len(parts) > 5:
-            return '%s_%s' % (parts[4], parts[5])
-        else:
-            return parts[4]
-
 
 class Comedity(_BasicScraper):
     url = 'http://www.comedity.com/'

dosagelib/plugins/common.py

@@ -6,6 +6,7 @@
 from __future__ import absolute_import, division, print_function
 
 from ..scraper import _ParserScraper
+from ..helpers import indirectStarter
 
 # Common base classes for comics with the same structure (same hosting
 # software, for example) go here. Since those are shared by many modules,
@@ -35,3 +36,20 @@ class _WPNaviIn(_WordPressScraper):
 class _ComicControlScraper(_ParserScraper):
     imageSearch = '//img[@id="cc-comic"]'
     prevSearch = '//a[@rel="prev"]'
+
+
+class _TumblrScraper(_ParserScraper):
+    starter = indirectStarter
+
+    def namer(self, image_url, page_url):
+        # tumblr URLs: http://host/post/num/name
+        #              0    1 2    3    4   5
+        parts = page_url.split('/')
+        if len(parts) > 5:
+            return '%s_%s' % (parts[4], parts[5])
+        else:
+            return parts[4]
+
+    def shouldSkipUrl(self, url, data):
+        """Reblogged stuff is iframed"""
+        return data.xpath('//div[@id="post"]//iframe')

dosagelib/plugins/f.py

@@ -46,16 +46,6 @@ class FauxPas(_BasicScraper):
     help = 'Index format: nnn'
 
 
-class FilibusterCartoons(_BasicScraper):
-    url = 'http://www.filibustercartoons.com/'
-    rurl = escape(url)
-    stripUrl = url + 'index.php/%s'
-    firstStripUrl = stripUrl % '2001/06/28/poor-jean'
-    imageSearch = compile(tagre("img", "src", r'(%scomics/[^"]+)' % rurl))
-    prevSearch = compile(tagre("a", "href", r'(%s[^"]+)' % rurl, after="prev"))
-    help = 'Index format: yyyy/mm/dd/name'
-
-
 class FireflyCross(_WordPressScraper):
     url = 'http://www.fireflycross.pensandtales.com/'
     firstStripUrl = url + '?comic=05062002'
@@ -101,14 +91,13 @@ class Flipside(_BasicScraper):
     help = 'Index format: nnnn'
 
 
-class FonFlatter(_BasicScraper):
+class FonFlatter(_ParserScraper):
     url = 'http://www.fonflatter.de/'
-    rurl = escape(url)
     stripUrl = url + '%s/'
     firstStripUrl = stripUrl % '2005/09/20/01-begegnung-mit-batman'
     lang = 'de'
-    imageSearch = compile(r'src="(%s\d+/fred_\d+-\d+-\d+[^"]+)' % rurl)
+    imageSearch = r'//img[re:test(@src, "/fred_\d+")]'
-    prevSearch = compile(tagre("a", "href", r'(%s[^"]+)' % rurl, after="prev"))
+    prevSearch = '//a[@rel="prev"]'
     help = 'Index format: yyyy/mm/dd/number-stripname'
 
     def shouldSkipUrl(self, url, data):

dosagelib/plugins/j.py

@@ -10,6 +10,7 @@ from re import compile, escape
 from ..scraper import _BasicScraper
 from ..util import tagre
 from ..helpers import indirectStarter
+from .common import _ComicControlScraper
 
 
 class JackCannon(_BasicScraper):
@@ -51,13 +52,8 @@ class JoeAndMonkey(_BasicScraper):
     help = 'Index format: nnn'
 
 
-class JohnnyWander(_BasicScraper):
+class JohnnyWander(_ComicControlScraper):
     url = 'http://www.johnnywander.com/'
-    stripUrl = url + 'comics/%s'
-    firstStripUrl = stripUrl % '423'
-    imageSearch = compile(tagre("img", "src", r'(http://www\.johnnywander\.com/files/comics/[^"]+)'))
-    prevSearch = compile(tagre("a", "href", r'(/comics/\d+)') + r'prev')
-    help = 'Index format: nnn'
 
 
 class JustAnotherEscape(_BasicScraper):

dosagelib/plugins/t.py

@@ -4,11 +4,13 @@
 # Copyright (C) 2015-2016 Tobias Gruetzmacher
 
 from __future__ import absolute_import, division, print_function
+
 from re import compile, escape, IGNORECASE
+
 from ..scraper import _BasicScraper, _ParserScraper
 from ..helpers import indirectStarter
 from ..util import tagre
-from .common import _WordPressScraper
+from .common import _ComicControlScraper, _TumblrScraper, _WordPressScraper
 
 
 class TheBrads(_BasicScraper):
@@ -50,9 +52,8 @@ class TheGentlemansArmchair(_WordPressScraper):
 
 
 class TheLandscaper(_BasicScraper):
-    url = 'http://landscaper.visual-assault.net/comic/latest'
+    stripUrl = 'http://landscaper.visual-assault.net/comic/%s'
-    rurl = escape(url)
+    url = stripUrl % 'latest'
-    stripUrl = url + 'comic/%s'
     firstStripUrl = stripUrl % '1'
     imageSearch = compile(tagre("img", "src",
                                 r'(/comics/comic/comic_page/[^"]+)'))
@@ -65,14 +66,11 @@ class TheMelvinChronicles(_WordPressScraper):
     url = 'http://melvin.jeaniebottle.com/'
 
 
-class TheNoob(_BasicScraper):
+class TheNoob(_WordPressScraper):
-    url = 'http://www.thenoobcomic.com/index.php'
+    url = 'http://thenoobcomic.com/'
-    stripUrl = url + '?pos=%s'
+    stripUrl = url + 'comic/%s/'
     firstStripUrl = stripUrl % '1'
-    imageSearch = compile(tagre("img", "src", r'(/headquarters/comics/[^"]+)'))
+    help = 'Index format: n (unpadded)'
-    prevSearch = compile(tagre("a", "href", r'(\?pos=\d+)',
-                               before="comic_nav_previous_button"))
-    help = 'Index format: nnnn'
 
 
 class TheOrderOfTheStick(_BasicScraper):
@@ -100,33 +98,21 @@ class TheParkingLotIsFull(_BasicScraper):
     help = 'Index format: nnn'
 
 
-class TheThinHLine(_BasicScraper):
+class TheThinHLine(_TumblrScraper):
     url = 'http://thinhline.tumblr.com/'
-    rurl = escape(url)
+    firstStripUrl = url + 'post/4177372348/thl-1-a-cats-got-his-tongue-click-on-the'
-    stripUrl = url + 'post/%s'
+    imageSearch = '//img[@id="content-image"]/@data-src'
-    firstStripUrl = stripUrl % '3517345105'
+    prevSearch = '//div[@id="pagination"]/a[text()=">"]'
-    imageSearch = compile(tagre('img', 'data-src',
+    latestSearch = '//a[@class="timestamp"]'
-                                r'([^"]+media.tumblr.com/[^"]+)',
-                                before='content-image'))
-    prevSearch = compile(tagre("a", "href", r'([^"]+)') + '&gt;</a>')
-    latestSearch = compile(tagre("a", "href", r'([^"]+)',
-                                 after='class="timestamp"'))
-    starter = indirectStarter
     adult = True
 
-    indirectImageSearch = compile(tagre('a', 'href', r'(%simage/\d+)' % rurl))
+    indirectImageSearch = '//div[@id="post"]//a[not(@rel) and img]'
 
     def getComicStrip(self, url, data):
         """The comic strip image is in a separate page."""
-        pageUrl = self.fetchUrl(url, data, self.indirectImageSearch)
+        subPage = self.fetchUrl(url, data, self.indirectImageSearch)
-        pageData = self.getPage(pageUrl)
+        pageData = self.getPage(subPage)
-        return super(TheThinHLine, self).getComicStrip(pageUrl, pageData)
+        return super(TheThinHLine, self).getComicStrip(subPage, pageData)
-
-    def namer(self, image_url, page_url):
-        """Use page URL sequence which is apparently increasing."""
-        num = page_url.split('/')[-1]
-        ext = image_url.rsplit('.', 1)[1]
-        return "thethinhline-%s.%s" % (num, ext)
 
 
 class TheWhiteboard(_BasicScraper):
@@ -137,13 +123,9 @@ class TheWhiteboard(_BasicScraper):
     help = 'Index format: twb or wb + n wg. twb1000'
 
 
-class TheWotch(_BasicScraper):
+class TheWotch(_WordPressScraper):
     url = 'http://www.thewotch.com/'
-    stripUrl = url + '?date=%s'
+    firstStripUrl = url + '?comic=enter-the-wotch'
-    firstStripUrl = stripUrl % '2002-11-21'
-    imageSearch = compile(r"<img.+?src='(comics/.+?)'")
-    prevSearch = compile(r"<link rel='Previous' href='(/\?date=\d+-\d+-\d+)'")
-    help = 'Index format: yyyy-mm-dd'
 
 
 class ThisIsIndexed(_BasicScraper):
@@ -157,23 +139,16 @@ class ThisIsIndexed(_BasicScraper):
     help = 'Index format: number'
 
 
-class ThreePanelSoul(_BasicScraper):
+class ThreePanelSoul(_ComicControlScraper):
     url = 'http://threepanelsoul.com/'
-    rurl = escape(url)
+    firstStripUrl = url + 'comic/a-test-comic'
-    stripUrl = url + '%s/'
-    firstStripUrl = stripUrl % '2006/05/11/a-test-comic'
-    imageSearch = compile(tagre("img", "src", r'(%scomics/[^"]+)' % rurl))
-    prevSearch = compile(tagre("a", "href", r'(%s\d+/\d+/\d+/[^"]+)' % rurl, after="prev"))
-    help = 'Index format: yyyy/mm/dd/stripname'
 
 
-class ToonHole(_BasicScraper):
+class ToonHole(_WordPressScraper):
     url = 'http://www.toonhole.com/'
-    rurl = escape(url)
     stripUrl = url + '%s/'
     firstStripUrl = stripUrl % '2009/12/toon-hole-coming-soon-2010'
-    imageSearch = compile(tagre("img", "src", r'(%scomics/\d+-\d+-\d+[^"]+)' % rurl))
+    prevSearch = '//a[@rel="prev"]'
-    prevSearch = compile(tagre("a", "href", r'(%s\d+/\d+/[^"]+)' % rurl, after="prev"))
     help = 'Index format: yyyy/mm/stripname'
 
     def shouldSkipUrl(self, url, data):