From f095f6309ee57a73668f403e2aec0622a40eac66 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Damjan=20Ko=C5=A1ir?= Date: Tue, 10 Dec 2013 16:31:03 +1300 Subject: [PATCH 01/14] added comic Edmund Finney's Quest to Find the Meaning of Life --- dosagelib/plugins/e.py | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/dosagelib/plugins/e.py b/dosagelib/plugins/e.py index 884c18cf0..0de9a2fde 100644 --- a/dosagelib/plugins/e.py +++ b/dosagelib/plugins/e.py @@ -56,6 +56,15 @@ class EdibleDirt(_BasicScraper): prevSearch = compile(tagre("a", "href", r"(index\.php\?id=\d+)")+"Previous") help = 'Index format: number' +class EdmundFinney(_BasicScraper): + description = u"Edmund Finney's Quest to Find the Meaning of Life" + url = 'http://eqcomics.com/' + rurl = escape(url) + stripUrl = url + '%s/' + firstStripUrl = stripUrl % '2009/03/08/sunday-aliens' + imageSearch = compile(tagre("img", "src", r'(%scomics/\d+-\d+-\d+-[^"/]+)' % rurl)) + prevSearch = compile(tagre("a", "href", r'(%s\d+/\d+/\d+/[^"/]+/)' % rurl, after="navi navi-prev")) + help = 'Index format: yyyy/mm/dd/stripname' class EerieCuties(_BasicScraper): url = 'http://www.eeriecuties.com/' From b53ca04ee7bdee825566ad4ed508ac4d81f965c7 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Damjan=20Ko=C5=A1ir?= Date: Tue, 10 Dec 2013 16:32:16 +1300 Subject: [PATCH 02/14] added comic Internet Webcomic --- dosagelib/plugins/i.py | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/dosagelib/plugins/i.py b/dosagelib/plugins/i.py index b1064ab8d..4bda675ab 100644 --- a/dosagelib/plugins/i.py +++ b/dosagelib/plugins/i.py @@ -38,6 +38,17 @@ class IDreamOfAJeanieBottle(_BasicScraper): help = 'Index format: n (unpadded)' +class InternetWebcomic(_BasicScraper): + description = u"Internet Webcomic" + url = 'http://www.internet-webcomic.com/' + rurl = escape(url) + stripUrl = url + '?p=%s' + firstStripUrl = stripUrl % '30' + imageSearch = compile(tagre("img", "src", r'(%scomics/[^"/]+)' % rurl)) + prevSearch = compile(tagre("a", "href", r'(%s\?p=\d+)' % rurl, after="navi navi-prev")) + help = 'Index format: n' + + class IrregularWebcomic(_BasicScraper): url = 'http://www.irregularwebcomic.net/' stripUrl = url + '%s.html' From 4f47792dee239967fa9457ef188780e16024fe76 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Damjan=20Ko=C5=A1ir?= Date: Tue, 10 Dec 2013 16:33:07 +1300 Subject: [PATCH 03/14] added comic The Gamer Cat --- dosagelib/plugins/t.py | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/dosagelib/plugins/t.py b/dosagelib/plugins/t.py index 59563a6f5..52e78a1a2 100644 --- a/dosagelib/plugins/t.py +++ b/dosagelib/plugins/t.py @@ -49,6 +49,15 @@ class TheDreamlandChronicles(_BasicScraper): bp[2] = bp[2][:-1] return "%s-%s-%s.%s" % (bp[0], bp[1], bp[2], ext) +class TheGamerCat(_BasicScraper): + description = u"The Gamer Cat" + url = 'http://www.thegamercat.com/' + rurl = escape(url) + stripUrl = url + '%s/' + firstStripUrl = stripUrl % '2011/06/06102011' + imageSearch = compile(tagre("img", "src", r'(%swordpress/comics/[^"/]+)' % rurl)) + prevSearch = compile(tagre("a", "href", r'(%s\d+/\d+/[^"/]+/)' % rurl , after="navi navi-prev")) + help = 'Index format: yyyy/mm/mmddyyyy' class TheNoob(_BasicScraper): url = 'http://www.thenoobcomic.com/index.php' From 5982e27c7ba7597958d32ee8e4d10a7276baba5c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Damjan=20Ko=C5=A1ir?= Date: Tue, 10 Dec 2013 16:34:47 +1300 Subject: [PATCH 04/14] added comic Red's Planet --- dosagelib/plugins/r.py | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/dosagelib/plugins/r.py b/dosagelib/plugins/r.py index 6f104e24b..9f8115b80 100644 --- a/dosagelib/plugins/r.py +++ b/dosagelib/plugins/r.py @@ -49,6 +49,15 @@ class RedMeat(_BasicScraper): def namer(cls, imageUrl, pageUrl): return imageUrl.split('/')[-2] +class RedsPlanet(_BasicScraper): + description = u"Red's Planet" + url = 'http://www.redsplanet.com/comic/' + rurl = escape(url) + stripUrl = url + 'rp/%s/' + firstStripUrl = stripUrl % 'pro/prologue-01' + imageSearch = compile(tagre("img", "src", r'(%scomics/\d+-\d+-\d+_[^"/]+)' % rurl)) + prevSearch = compile(tagre("a", "href", r'(%srp/[^"/]+/[^"/]+/)' % rurl)) + help = 'Index format: chapter/stripname' class RedString(_BasicScraper): description = u'A web comics about love and growing up. Art by Gina Biggs.' From 58b62dbad3f21395437613beaef470800e0fd109 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Damjan=20Ko=C5=A1ir?= Date: Tue, 10 Dec 2013 16:37:35 +1300 Subject: [PATCH 05/14] added comic Scandinavia and the World --- dosagelib/plugins/s.py | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/dosagelib/plugins/s.py b/dosagelib/plugins/s.py index 05f03c5ff..159ff016e 100644 --- a/dosagelib/plugins/s.py +++ b/dosagelib/plugins/s.py @@ -79,6 +79,16 @@ class SandraOnTheRocks(_BasicScraper): prevSearch = compile(tagre("a", "href", r'([^"]*/strips-sotr/[^"]+)', before="cn[id]prev")) help = 'Index format: name' +class ScandinaviaAndTheWorld(_BasicScraper): + description = u'Scandinavia and the World' + url = 'http://satwcomic.com/' + rurl = escape(url) + stripUrl = url + '%s/' + firstStripUrl = stripUrl % 'sweden-denmark-and-norway' + imageSearch = compile(tagre("img", "src", r'(%sart/[^"/]+)' % rurl)) + prevSearch = compile(tagre("a", "href", r'(%s[^"/]+)' % rurl)+"\s*"+tagre('span', 'class', 'spritePrevious')) + help = 'Index format: stripname' + class ScaryGoRound(_BasicScraper): url = 'http://www.scarygoround.com/' stripUrl = url + '?date=%s' From e181b287c937b5ea44b70974efc307d949f412c0 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Damjan=20Ko=C5=A1ir?= Date: Tue, 10 Dec 2013 16:39:30 +1300 Subject: [PATCH 06/14] added comic Romantically Apocalyptic --- dosagelib/plugins/r.py | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/dosagelib/plugins/r.py b/dosagelib/plugins/r.py index 9f8115b80..a39b411b3 100644 --- a/dosagelib/plugins/r.py +++ b/dosagelib/plugins/r.py @@ -68,6 +68,15 @@ class RedString(_BasicScraper): prevSearch = compile(tagre("a", "href", r'(/index\.php\?id=\d+)', after="prev")) help = 'Index format: nnn' +class RomanticallyApocalyptic(_BasicScraper): + description = u'Romantically Apocalyptic' + url = 'http://romanticallyapocalyptic.com/' + rurl = escape(url) + stripUrl = url + '%s/' + firstStripUrl = stripUrl % '1' + imageSearch = compile(tagre("img", "src", r'(%sart/\d+[^"]+)' % rurl)) + prevSearch = compile(tagre("a", "href", r'(%s\d+[^"]+)' % rurl)+"\s*"+tagre('span', 'class', 'spritePrevious')) + help = 'Index format: n' class Roza(_BasicScraper): url = 'http://www.junglestudio.com/roza/index.php' From f48b22b5128238f61fe697ae44b0252702e980cf Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Damjan=20Ko=C5=A1ir?= Date: Tue, 10 Dec 2013 16:40:44 +1300 Subject: [PATCH 07/14] added comic Not Invented Here --- dosagelib/plugins/n.py | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/dosagelib/plugins/n.py b/dosagelib/plugins/n.py index dcae95739..1781250b9 100644 --- a/dosagelib/plugins/n.py +++ b/dosagelib/plugins/n.py @@ -163,6 +163,15 @@ class NoNeedForBushido(_BasicScraper): starter = indirectStarter(url, compile(tagre("a", "href", r'(%s\?webcomic1=[^"]+)' % rurl, after="last-webcomic"))) +class NotInventedHere(_BasicScraper): + description = u'Not Invented Here' + url = 'http://notinventedhe.re/' + rurl = escape(url) + stripUrl = url + '%s/' + firstStripUrl = stripUrl % 'on/2009-9-21' + imageSearch = compile(tagre("img", "src", r'(http://thiswas.notinventedhe.re/on/\d+-\d+-\d+)')) + prevSearch = compile(tagre("a", "href", r'(/on/\d+-\d+-\d+)')+'\s*Previous') + help = 'Index format: yyyy-mm-dd' class Nukees(_BasicScraper): description = u'Nukees, The Atomic Comic Strip' From 4e5717be57f15017ddb27347b3f88df44dc43340 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Damjan=20Ko=C5=A1ir?= Date: Tue, 10 Dec 2013 17:08:15 +1300 Subject: [PATCH 08/14] added comic Gaia --- dosagelib/plugins/g.py | 9 +++++++++ dosagelib/plugins/w.py | 11 +++++++++++ 2 files changed, 20 insertions(+) diff --git a/dosagelib/plugins/g.py b/dosagelib/plugins/g.py index c931d33d2..326db56b8 100644 --- a/dosagelib/plugins/g.py +++ b/dosagelib/plugins/g.py @@ -8,6 +8,15 @@ from ..scraper import _BasicScraper from ..helpers import indirectStarter from ..util import tagre +class Gaia(_BasicScraper): + description = u'Gaia, a story about the nature of reality, and the answer to Lilith?s simple, meek, world-shattering question: ?Will you come along??' + url = 'http://www.sandraandwoo.com/gaia/' + rurl = escape(url) + stripUrl = url + '%s/' + firstStripUrl = stripUrl % '2000/01/01/welcome-to-gaia/' + imageSearch = compile(tagre("img", "src", r'(%scomics/\d+-\d+-\d+-[^"]+)' % rurl)) + prevSearch = compile(tagre("a", "href", r'(%s\d+/\d+/\d+/[^"]+/)' % rurl, after="prev")) + help = 'Index format: yyyy/mm/dd/number-stripname' class Galaxion(_BasicScraper): description = u'Galaxion - Life. Love. Hyperspace.' diff --git a/dosagelib/plugins/w.py b/dosagelib/plugins/w.py index 20ec1f9b5..93c34fc4c 100644 --- a/dosagelib/plugins/w.py +++ b/dosagelib/plugins/w.py @@ -75,6 +75,17 @@ class WeCanSleepTomorrow(_BasicScraper): help = 'Index format: yyyy/mm/dd/stripname' +class Weregeek(_BasicScraper): + description = u'Weregeek' + url = 'http://www.weregeek.com/' + rurl = escape(url) + stripUrl = url + '%s/' + firstStripUrl = stripUrl % '2006/11/27/' + imageSearch = compile(tagre("img", "src", r'(%scomics/\d+-\d+-\d+[^"]+)' % rurl)) + prevSearch = compile(tagre("a", "href", r'((%s)?(/)?\d+/\d+/\d+/)'% rurl)+'\s*'+ tagre('img', 'src', '[^"]*previous_day.gif')) + help = 'Index format: yyyy/mm/dd' + + class WhiteNinja(_BasicScraper): baseUrl = 'http://www.whiteninjacomics.com/' url = baseUrl + 'comics.shtml' From 4e40f026426ce66c7d085279166e6e3a7cb6b05c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Damjan=20Ko=C5=A1ir?= Date: Tue, 10 Dec 2013 18:02:20 +1300 Subject: [PATCH 09/14] added comic Gaia in German --- dosagelib/plugins/g.py | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/dosagelib/plugins/g.py b/dosagelib/plugins/g.py index 326db56b8..061614a85 100644 --- a/dosagelib/plugins/g.py +++ b/dosagelib/plugins/g.py @@ -18,6 +18,17 @@ class Gaia(_BasicScraper): prevSearch = compile(tagre("a", "href", r'(%s\d+/\d+/\d+/[^"]+/)' % rurl, after="prev")) help = 'Index format: yyyy/mm/dd/number-stripname' +class GaiaGerman(_BasicScraper): + description = u'Gaia, eine Geschichte über das Wesen der Wirklichkeit und die Antwort auf Liliths einfache, bescheidene, welterschütternde Frage: ?Kommt ihr mit??' + url = 'http://www.sandraandwoo.com/gaiade/' + rurl = escape(url) + stripUrl = url + '%s/' + firstStripUrl = stripUrl % '2000/01/01/welcome-to-gaia/' + imageSearch = compile(tagre("img", "src", r'(%scomics/\d+-\d+-\d+-[^"]+)' % rurl)) + prevSearch = compile(tagre("a", "href", r'(%s\d+/\d+/\d+/[^"]+/)' % rurl, after="prev")) + help = 'Index format: yyyy/mm/dd/number-stripname' + lang = 'de' + class Galaxion(_BasicScraper): description = u'Galaxion - Life. Love. Hyperspace.' url = 'http://galaxioncomics.com/' From dae269810262e79929bd3298852678d7ed98d7b7 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Damjan=20Ko=C5=A1ir?= Date: Sat, 25 Apr 2015 20:40:28 +1200 Subject: [PATCH 10/14] removing mismerge --- dosagelib/plugins/r.py | 18 ------------------ 1 file changed, 18 deletions(-) diff --git a/dosagelib/plugins/r.py b/dosagelib/plugins/r.py index ea64e3941..f95bbab15 100644 --- a/dosagelib/plugins/r.py +++ b/dosagelib/plugins/r.py @@ -65,15 +65,6 @@ class RedsPlanet(_BasicScraper): prevSearch = compile(tagre("a", "href", r'(%srp/[^"/]+/[^"/]+/)' % rurl)) help = 'Index format: chapter/stripname' -class RedsPlanet(_BasicScraper): - description = u"Red's Planet" - url = 'http://www.redsplanet.com/comic/' - rurl = escape(url) - stripUrl = url + 'rp/%s/' - firstStripUrl = stripUrl % 'pro/prologue-01' - imageSearch = compile(tagre("img", "src", r'(%scomics/\d+-\d+-\d+_[^"/]+)' % rurl)) - prevSearch = compile(tagre("a", "href", r'(%srp/[^"/]+/[^"/]+/)' % rurl)) - help = 'Index format: chapter/stripname' class RedString(_BasicScraper): url = 'http://www.redstring.strawberrycomics.com/' @@ -83,15 +74,6 @@ class RedString(_BasicScraper): prevSearch = compile(tagre("a", "href", r'(/index\.php\?id=\d+)', after="prev")) help = 'Index format: nnn' -class RomanticallyApocalyptic(_BasicScraper): - description = u'Romantically Apocalyptic' - url = 'http://romanticallyapocalyptic.com/' - rurl = escape(url) - stripUrl = url + '%s/' - firstStripUrl = stripUrl % '1' - imageSearch = compile(tagre("img", "src", r'(%sart/\d+[^"]+)' % rurl)) - prevSearch = compile(tagre("a", "href", r'(%s\d+[^"]+)' % rurl)+"\s*"+tagre('span', 'class', 'spritePrevious')) - help = 'Index format: n' class RomanticallyApocalyptic(_BasicScraper): url = 'http://romanticallyapocalyptic.com/' From 1e94a3c7c51a9e2a65c7e6982e4c59fac26ce6cb Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Damjan=20Ko=C5=A1ir?= Date: Sat, 25 Apr 2015 20:52:03 +1200 Subject: [PATCH 11/14] now the same as offical version --- dosagelib/plugins/g.py | 20 -------------------- 1 file changed, 20 deletions(-) diff --git a/dosagelib/plugins/g.py b/dosagelib/plugins/g.py index dfdffca00..b9dfe3869 100644 --- a/dosagelib/plugins/g.py +++ b/dosagelib/plugins/g.py @@ -8,26 +8,6 @@ from ..scraper import _BasicScraper from ..helpers import indirectStarter from ..util import tagre -class Gaia(_BasicScraper): - description = u'Gaia, a story about the nature of reality, and the answer to Lilith?s simple, meek, world-shattering question: ?Will you come along??' - url = 'http://www.sandraandwoo.com/gaia/' - rurl = escape(url) - stripUrl = url + '%s/' - firstStripUrl = stripUrl % '2000/01/01/welcome-to-gaia/' - imageSearch = compile(tagre("img", "src", r'(%scomics/\d+-\d+-\d+-[^"]+)' % rurl)) - prevSearch = compile(tagre("a", "href", r'(%s\d+/\d+/\d+/[^"]+/)' % rurl, after="prev")) - help = 'Index format: yyyy/mm/dd/number-stripname' - -class GaiaGerman(_BasicScraper): - description = u'Gaia, eine Geschichte über das Wesen der Wirklichkeit und die Antwort auf Liliths einfache, bescheidene, welterschütternde Frage: ?Kommt ihr mit??' - url = 'http://www.sandraandwoo.com/gaiade/' - rurl = escape(url) - stripUrl = url + '%s/' - firstStripUrl = stripUrl % '2000/01/01/welcome-to-gaia/' - imageSearch = compile(tagre("img", "src", r'(%scomics/\d+-\d+-\d+-[^"]+)' % rurl)) - prevSearch = compile(tagre("a", "href", r'(%s\d+/\d+/\d+/[^"]+/)' % rurl, after="prev")) - help = 'Index format: yyyy/mm/dd/number-stripname' - lang = 'de' class Galaxion(_BasicScraper): url = 'http://galaxioncomics.com/' From ac7b0d7e0e07e602c3fa21cbfb31bd4c6a61801e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Damjan=20Ko=C5=A1ir?= Date: Sun, 26 Apr 2015 00:19:08 +1200 Subject: [PATCH 12/14] adding parallel run option --- dosage | 1 + dosagelib/director.py | 2 +- 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/dosage b/dosage index 5a0cd5612..a18466db8 100755 --- a/dosage +++ b/dosage @@ -75,6 +75,7 @@ def setupOptions(): parser.add_argument('-m', '--modulehelp', action='store_true', help='display help for comic modules') parser.add_argument('-t', '--timestamps', action='store_true', help='print timestamps for all output at any info level') parser.add_argument('-o', '--output', action='append', dest='handler', choices=events.getHandlerNames(), help='sets output handlers for downloaded comics') + parser.add_argument('-p', '--parallel', action='store', type=int, default=1, help='fetch comics in parallel. Specify the number of connections') parser.add_argument('--adult', action='store_true', help='confirms that you are old enough to view adult content') # used for development testing prev/next matching parser.add_argument('--dry-run', action='store_true', help=argparse.SUPPRESS) diff --git a/dosagelib/director.py b/dosagelib/director.py index f18593536..474c497fe 100644 --- a/dosagelib/director.py +++ b/dosagelib/director.py @@ -161,7 +161,7 @@ def getComics(options): for scraperobj in getScrapers(options.comic, options.basepath, options.adult, options.multimatch): jobs.put(scraperobj) # start threads - num_threads = 1# XXX max(1, min(10, jobs.qsize())) + num_threads = max(1, min(options.parallel, jobs.qsize())) for i in range(num_threads): t = ComicGetter(options) threads.append(t) From 561005887a87a4e7fdb7c10224b55ebf44166d42 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Damjan=20Ko=C5=A1ir?= Date: Sun, 26 Apr 2015 00:23:45 +1200 Subject: [PATCH 13/14] unneeded max --- dosagelib/director.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/dosagelib/director.py b/dosagelib/director.py index 474c497fe..a621c5e2d 100644 --- a/dosagelib/director.py +++ b/dosagelib/director.py @@ -161,7 +161,7 @@ def getComics(options): for scraperobj in getScrapers(options.comic, options.basepath, options.adult, options.multimatch): jobs.put(scraperobj) # start threads - num_threads = max(1, min(options.parallel, jobs.qsize())) + num_threads = min(options.parallel, jobs.qsize()) for i in range(num_threads): t = ComicGetter(options) threads.append(t) From dc427d6066019af55257ae4a4d8b8f3c37aea7fa Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Damjan=20Ko=C5=A1ir?= Date: Sun, 26 Apr 2015 21:52:31 +1200 Subject: [PATCH 14/14] fixed the gamercat --- dosagelib/plugins/t.py | 17 +++++++++-------- 1 file changed, 9 insertions(+), 8 deletions(-) diff --git a/dosagelib/plugins/t.py b/dosagelib/plugins/t.py index 0967a8a8b..42b0d41a1 100755 --- a/dosagelib/plugins/t.py +++ b/dosagelib/plugins/t.py @@ -3,7 +3,7 @@ # Copyright (C) 2012-2014 Bastian Kleineidam from re import compile, escape, IGNORECASE -from ..scraper import _BasicScraper +from ..scraper import _BasicScraper, _ParserScraper from ..helpers import indirectStarter from ..util import tagre @@ -46,14 +46,15 @@ class TheDreamlandChronicles(_BasicScraper): bp[2] = bp[2][:-1] return "%s-%s-%s.%s" % (bp[0], bp[1], bp[2], ext) -class TheGamerCat(_BasicScraper): - url = 'http://www.thegamercat.com/' +class TheGamerCat(_ParserScraper): + url = "http://www.thegamercat.com/" rurl = escape(url) - stripUrl = url + '%s/' - firstStripUrl = stripUrl % '2011/06/06102011' - imageSearch = compile(tagre("img", "src", r'(%swordpress/comics/[^"/]+)' % rurl)) - prevSearch = compile(tagre("a", "href", r'(%s\d+/\d+/[^"/]+/)' % rurl , after="navi navi-prev")) - help = 'Index format: yyyy/mm/mmddyyyy' + stripUrl = url + "comic/%s/" + firstStripUrl = stripUrl % "06102011" + css = True + imageSearch = '#comic img' + prevSearch = '.comic-nav-previous' + help = 'Index format: stripname' class TheGentlemansArmchair(_BasicScraper):