Merge branch 'master' of https://github.com/webcomics/dosage

dosage

@@ -75,6 +75,7 @@ def setupOptions():
     parser.add_argument('-m', '--modulehelp', action='store_true', help='display help for comic modules')
     parser.add_argument('-t', '--timestamps', action='store_true', help='print timestamps for all output at any info level')
     parser.add_argument('-o', '--output', action='append', dest='handler', choices=events.getHandlerNames(), help='sets output handlers for downloaded comics')
+    parser.add_argument('-p', '--parallel', action='store', type=int, default=1, help='fetch comics in parallel. Specify the number of connections')
     parser.add_argument('--adult', action='store_true', help='confirms that you are old enough to view adult content')
     # used for development testing prev/next matching
     parser.add_argument('--dry-run', action='store_true', help=argparse.SUPPRESS)

dosagelib/director.py

@@ -161,7 +161,7 @@ def getComics(options):
         for scraperobj in getScrapers(options.comic, options.basepath, options.adult, options.multimatch):
             jobs.put(scraperobj)
         # start threads
-        num_threads = 1# XXX max(1, min(10, jobs.qsize()))
+        num_threads = min(options.parallel, jobs.qsize())
         for i in range(num_threads):
             t = ComicGetter(options)
             threads.append(t)

dosagelib/plugins/d.py

@@ -4,7 +4,7 @@
 
 from re import compile, escape
 
-from ..scraper import _BasicScraper
+from ..scraper import _BasicScraper, _ParserScraper
 from ..helpers import indirectStarter, bounceStarter
 from ..util import tagre
 
@@ -199,6 +199,16 @@ class DMFA(_BasicScraper):
     help = 'Index format: nnn (normally, some specials)'
 
 
+class DoctorCat(_ParserScraper):
+    url = "http://doctorcatmd.com/"
+    stripUrl = url + "comic/%s"
+    firstStripUrl = stripUrl % "doctor-cat"
+    css = True
+    imageSearch = '#comic img'
+    prevSearch = '.navi-prev'
+    help = 'Index format: stripname'
+
+
 class DoemainOfOurOwn(_BasicScraper):
     url = 'http://www.doemain.com/'
     stripUrl = url + 'index.cgi/%s'

dosagelib/plugins/m.py

@@ -42,13 +42,13 @@ class MagickChicks(_BasicScraper):
     help = 'Index format: name'
 
 
-class ManlyGuysDoingManlyThings(_BasicScraper):
+class ManlyGuysDoingManlyThings(_ParserScraper):
     url = 'http://thepunchlineismachismo.com/'
-    rurl = escape(url)
     stripUrl = url + 'archives/comic/%s'
     firstStripUrl = stripUrl % '02222010'
-    imageSearch = compile(tagre("img", "src", r'(%swp-content/uploads/\d+/\d+/\d+-\d+-\d+[^"]+)' % rurl))
-    prevSearch = compile(tagre("a", "href", r'(%sarchives/comic/[^"]+)' % rurl, after="previous"))
+    css = True
+    imageSearch = "#comic img"
+    prevSearch = ".comic-nav-previous"
     help = 'Index format: ddmmyyyy'
 
 

dosagelib/plugins/o.py

@@ -3,7 +3,7 @@
 # Copyright (C) 2012-2014 Bastian Kleineidam
 
 from re import compile, escape
-from ..scraper import _BasicScraper
+from ..scraper import _BasicScraper, _ParserScraper
 from ..helpers import indirectStarter
 from ..util import tagre
 
@@ -68,15 +68,13 @@ class OkCancel(_BasicScraper):
     help = 'Index format: yyyymmdd'
 
 
-class OmakeTheater(_BasicScraper):
-    url = 'http://omaketheater.com/'
-    rurl = escape(url)
-    stripUrl = url + 'comic/%s/'
+class OmakeTheater(_ParserScraper):
+    url = 'http://omaketheater.com/comics/'
+    stripUrl = url + '%s/'
     firstStripUrl = stripUrl % '1'
-    imageSearch = compile(tagre("img", "src", r'(http://media\.omaketheater\.com/4koma/[^"]+)'))
-    prevSearch = compile(tagre("a", "href", r'(%scomic/\d+/)' % rurl, after="prev"))
-    starter = indirectStarter(url,
-        compile(tagre("a", "href", r'(%scomic/\d+/)' % rurl)))
+    css = True
+    imageSearch = ".comicImage img"
+    prevSearch = ".previous a"
     help = 'Index format: number (unpadded)'
 
 

dosagelib/plugins/t.py

@@ -3,7 +3,7 @@
 # Copyright (C) 2012-2014 Bastian Kleineidam
 
 from re import compile, escape, IGNORECASE
-from ..scraper import _BasicScraper
+from ..scraper import _BasicScraper, _ParserScraper
 from ..helpers import indirectStarter
 from ..util import tagre
 
@@ -46,14 +46,14 @@ class TheDreamlandChronicles(_BasicScraper):
             bp[2] = bp[2][:-1]
         return "%s-%s-%s.%s" % (bp[0], bp[1], bp[2], ext)
 
-class TheGamerCat(_BasicScraper):
-    url = 'http://www.thegamercat.com/'
-    rurl = escape(url)
-    stripUrl = url + '%s/'
-    firstStripUrl = stripUrl % '2011/06/06102011'
-    imageSearch = compile(tagre("img", "src", r'(%swordpress/comics/[^"/]+)' % rurl))
-    prevSearch = compile(tagre("a", "href", r'(%s\d+/\d+/[^"/]+/)' % rurl , after="navi navi-prev"))
-    help = 'Index format: yyyy/mm/mmddyyyy'
+class TheGamerCat(_ParserScraper):
+    url = "http://www.thegamercat.com/"
+    stripUrl = url + "comic/%s/"
+    firstStripUrl = stripUrl % "06102011"
+    css = True
+    imageSearch = '#comic img'
+    prevSearch = '.comic-nav-previous'
+    help = 'Index format: stripname'
 
 
 class TheGentlemansArmchair(_BasicScraper):