diff --git a/dosage b/dosage index 5a0cd5612..a18466db8 100755 --- a/dosage +++ b/dosage @@ -75,6 +75,7 @@ def setupOptions(): parser.add_argument('-m', '--modulehelp', action='store_true', help='display help for comic modules') parser.add_argument('-t', '--timestamps', action='store_true', help='print timestamps for all output at any info level') parser.add_argument('-o', '--output', action='append', dest='handler', choices=events.getHandlerNames(), help='sets output handlers for downloaded comics') + parser.add_argument('-p', '--parallel', action='store', type=int, default=1, help='fetch comics in parallel. Specify the number of connections') parser.add_argument('--adult', action='store_true', help='confirms that you are old enough to view adult content') # used for development testing prev/next matching parser.add_argument('--dry-run', action='store_true', help=argparse.SUPPRESS) diff --git a/dosagelib/director.py b/dosagelib/director.py index f18593536..a621c5e2d 100644 --- a/dosagelib/director.py +++ b/dosagelib/director.py @@ -161,7 +161,7 @@ def getComics(options): for scraperobj in getScrapers(options.comic, options.basepath, options.adult, options.multimatch): jobs.put(scraperobj) # start threads - num_threads = 1# XXX max(1, min(10, jobs.qsize())) + num_threads = min(options.parallel, jobs.qsize()) for i in range(num_threads): t = ComicGetter(options) threads.append(t) diff --git a/dosagelib/plugins/d.py b/dosagelib/plugins/d.py index bd34d9922..8224444aa 100644 --- a/dosagelib/plugins/d.py +++ b/dosagelib/plugins/d.py @@ -4,7 +4,7 @@ from re import compile, escape -from ..scraper import _BasicScraper +from ..scraper import _BasicScraper, _ParserScraper from ..helpers import indirectStarter, bounceStarter from ..util import tagre @@ -199,6 +199,16 @@ class DMFA(_BasicScraper): help = 'Index format: nnn (normally, some specials)' +class DoctorCat(_ParserScraper): + url = "http://doctorcatmd.com/" + stripUrl = url + "comic/%s" + firstStripUrl = stripUrl % "doctor-cat" + css = True + imageSearch = '#comic img' + prevSearch = '.navi-prev' + help = 'Index format: stripname' + + class DoemainOfOurOwn(_BasicScraper): url = 'http://www.doemain.com/' stripUrl = url + 'index.cgi/%s' diff --git a/dosagelib/plugins/m.py b/dosagelib/plugins/m.py index d49652a5d..3be621fb7 100755 --- a/dosagelib/plugins/m.py +++ b/dosagelib/plugins/m.py @@ -42,13 +42,13 @@ class MagickChicks(_BasicScraper): help = 'Index format: name' -class ManlyGuysDoingManlyThings(_BasicScraper): +class ManlyGuysDoingManlyThings(_ParserScraper): url = 'http://thepunchlineismachismo.com/' - rurl = escape(url) stripUrl = url + 'archives/comic/%s' firstStripUrl = stripUrl % '02222010' - imageSearch = compile(tagre("img", "src", r'(%swp-content/uploads/\d+/\d+/\d+-\d+-\d+[^"]+)' % rurl)) - prevSearch = compile(tagre("a", "href", r'(%sarchives/comic/[^"]+)' % rurl, after="previous")) + css = True + imageSearch = "#comic img" + prevSearch = ".comic-nav-previous" help = 'Index format: ddmmyyyy' diff --git a/dosagelib/plugins/o.py b/dosagelib/plugins/o.py index a606717be..1c9c2c2f4 100644 --- a/dosagelib/plugins/o.py +++ b/dosagelib/plugins/o.py @@ -3,7 +3,7 @@ # Copyright (C) 2012-2014 Bastian Kleineidam from re import compile, escape -from ..scraper import _BasicScraper +from ..scraper import _BasicScraper, _ParserScraper from ..helpers import indirectStarter from ..util import tagre @@ -68,15 +68,13 @@ class OkCancel(_BasicScraper): help = 'Index format: yyyymmdd' -class OmakeTheater(_BasicScraper): - url = 'http://omaketheater.com/' - rurl = escape(url) - stripUrl = url + 'comic/%s/' +class OmakeTheater(_ParserScraper): + url = 'http://omaketheater.com/comics/' + stripUrl = url + '%s/' firstStripUrl = stripUrl % '1' - imageSearch = compile(tagre("img", "src", r'(http://media\.omaketheater\.com/4koma/[^"]+)')) - prevSearch = compile(tagre("a", "href", r'(%scomic/\d+/)' % rurl, after="prev")) - starter = indirectStarter(url, - compile(tagre("a", "href", r'(%scomic/\d+/)' % rurl))) + css = True + imageSearch = ".comicImage img" + prevSearch = ".previous a" help = 'Index format: number (unpadded)' diff --git a/dosagelib/plugins/t.py b/dosagelib/plugins/t.py index 0967a8a8b..6911a51c7 100755 --- a/dosagelib/plugins/t.py +++ b/dosagelib/plugins/t.py @@ -3,7 +3,7 @@ # Copyright (C) 2012-2014 Bastian Kleineidam from re import compile, escape, IGNORECASE -from ..scraper import _BasicScraper +from ..scraper import _BasicScraper, _ParserScraper from ..helpers import indirectStarter from ..util import tagre @@ -46,14 +46,14 @@ class TheDreamlandChronicles(_BasicScraper): bp[2] = bp[2][:-1] return "%s-%s-%s.%s" % (bp[0], bp[1], bp[2], ext) -class TheGamerCat(_BasicScraper): - url = 'http://www.thegamercat.com/' - rurl = escape(url) - stripUrl = url + '%s/' - firstStripUrl = stripUrl % '2011/06/06102011' - imageSearch = compile(tagre("img", "src", r'(%swordpress/comics/[^"/]+)' % rurl)) - prevSearch = compile(tagre("a", "href", r'(%s\d+/\d+/[^"/]+/)' % rurl , after="navi navi-prev")) - help = 'Index format: yyyy/mm/mmddyyyy' +class TheGamerCat(_ParserScraper): + url = "http://www.thegamercat.com/" + stripUrl = url + "comic/%s/" + firstStripUrl = stripUrl % "06102011" + css = True + imageSearch = '#comic img' + prevSearch = '.comic-nav-previous' + help = 'Index format: stripname' class TheGentlemansArmchair(_BasicScraper):