From 7440dba84ea156e4ef01c8e68bda8a9b0490510d Mon Sep 17 00:00:00 2001 From: Techwolf Date: Sun, 11 Apr 2021 18:48:54 -0700 Subject: [PATCH 01/71] Mark ProjectFuture as endOfLife --- dosagelib/plugins/projectfuture.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/dosagelib/plugins/projectfuture.py b/dosagelib/plugins/projectfuture.py index 6a7853bb9..ec44c7229 100644 --- a/dosagelib/plugins/projectfuture.py +++ b/dosagelib/plugins/projectfuture.py @@ -34,7 +34,7 @@ class ProjectFuture(_ParserScraper): cls('FishingTrip', 'fishing', '01-00'), cls('HeadsYouLose', 'heads', '00-01', last='07-12'), cls('NiallsStory', 'niall', '00'), - cls('ProjectFuture', 'strip', '0'), + cls('ProjectFuture', 'strip', '0', last='664'), cls('RedValentine', 'redvalentine', '1', last='6'), cls('ShortStories', 'shorts', '01-00'), cls('StrangeBedfellows', 'bedfellows', '1', last='6'), From 9834ca4caf95a5c17e7584b531230738edaea327 Mon Sep 17 00:00:00 2001 From: Techwolf Date: Sun, 11 Apr 2021 18:49:02 -0700 Subject: [PATCH 02/71] Add ProjectFuture/Xerian --- dosagelib/plugins/projectfuture.py | 1 + 1 file changed, 1 insertion(+) diff --git a/dosagelib/plugins/projectfuture.py b/dosagelib/plugins/projectfuture.py index ec44c7229..bf5a2db81 100644 --- a/dosagelib/plugins/projectfuture.py +++ b/dosagelib/plugins/projectfuture.py @@ -46,4 +46,5 @@ class ProjectFuture(_ParserScraper): cls('TheSierraChronicles', 'sierra', '0', last='29'), cls('TheTuppenyMan', 'tuppenny', '00', last='16'), cls('TurningANewPage', 'azrael', '1', last='54'), + cls('Xerian', 'xerian', '01-00'), ) From 5eb6cd40db5c76db88821fe92461191c53aaf7c3 Mon Sep 17 00:00:00 2001 From: Techwolf Date: Thu, 22 Apr 2021 01:45:51 -0700 Subject: [PATCH 03/71] Fix DeerMe --- dosagelib/plugins/comicfury.py | 1 + dosagelib/plugins/d.py | 13 ------------- dosagelib/plugins/old.py | 1 + 3 files changed, 2 insertions(+), 13 deletions(-) diff --git a/dosagelib/plugins/comicfury.py b/dosagelib/plugins/comicfury.py index ff98e2ef4..3bba2afa0 100644 --- a/dosagelib/plugins/comicfury.py +++ b/dosagelib/plugins/comicfury.py @@ -336,6 +336,7 @@ class ComicFury(_ParserScraper): cls('DeadFingers', 'deadfingers'), cls('DeadRabbitCa', 'afairtrade'), cls('DeepBlue', 'deepblue'), + cls('DeerMe', 'deerme'), cls('DefineHero', 'definehero'), cls('DELIA', 'delia'), cls('DemasPokmonAdventure', 'nuzlocke-dema'), diff --git a/dosagelib/plugins/d.py b/dosagelib/plugins/d.py index bf49824be..d0ca80b09 100644 --- a/dosagelib/plugins/d.py +++ b/dosagelib/plugins/d.py @@ -84,19 +84,6 @@ class DeepFried(_BasicScraper): help = 'Index format: none' -class DeerMe(_ParserScraper): - url = 'http://deerme.net/comics/' - stripUrl = url + '%s' - firstStripUrl = stripUrl % '1' - imageSearch = ('//img[@id="comicimage"]', '//img[@id="latestcomicimage"]') - prevSearch = '//a[@rel="prev"]' - nextSearch = '//a[@rel="next"]' - starter = bounceStarter - - def namer(self, imageUrl, pageUrl): - return pageUrl.rsplit('/', 1)[-1] + '.' + imageUrl.rsplit('.', 1)[-1] - - class Delve(_WordPressScraper): url = 'https://thisis.delvecomic.com/NewWP/' stripUrl = url + 'comic/%s/' diff --git a/dosagelib/plugins/old.py b/dosagelib/plugins/old.py index 1ba2a7546..adfa8961b 100644 --- a/dosagelib/plugins/old.py +++ b/dosagelib/plugins/old.py @@ -1602,6 +1602,7 @@ class Renamed(Scraper): # Renamed in 3.0 cls('CrapIDrewOnMyLunchBreak', 'WebToons/CrapIDrewOnMyLunchBreak'), + cls('DeerMe', 'ComicFury/DeerMe'), cls('GoComics/BloomCounty2017', 'GoComics/BloomCounty2019'), cls('GoComics/Cathy', 'GoComics/CathyClassics'), cls('GoComics/Owlturd', 'GoComics/ShenComix'), From 4ecb05b6b360ed66eec609e9ad2a6a7de39328d1 Mon Sep 17 00:00:00 2001 From: Techwolf Date: Thu, 22 Apr 2021 01:56:38 -0700 Subject: [PATCH 04/71] Fix StudioKhimera/UberQuest --- dosagelib/plugins/studiokhimera.py | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/dosagelib/plugins/studiokhimera.py b/dosagelib/plugins/studiokhimera.py index 267a578f4..081215f93 100644 --- a/dosagelib/plugins/studiokhimera.py +++ b/dosagelib/plugins/studiokhimera.py @@ -50,9 +50,11 @@ class StudioKhimera(_ParserScraper): class UberQuest(_WordPressScraper): name = 'StudioKhimera/UberQuest' - url = 'https://uberquest.studiokhimera.com/' - stripUrl = url + 'comic/%s/' - firstStripUrl = stripUrl % 'chapter-1-cover' + stripUrl = 'https://uberquest.studiokhimera.com/comic/page/%s/' + url = stripUrl % 'latest' + firstStripUrl = stripUrl % 'cover' + imageSearch = '//div[@class="prj--comic-image"]/img' + prevSearch = '//uq-image-button[d:class("prj--comic-control-prev")]' def namer(self, imageUrl, pageUrl): # Fix inconsistent filenames From 29ad39d003d48a74363b4e326d6254b2a6e8e074 Mon Sep 17 00:00:00 2001 From: Techwolf Date: Sat, 24 Apr 2021 18:11:45 -0700 Subject: [PATCH 05/71] Fix Centralia2050 --- dosagelib/plugins/c.py | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/dosagelib/plugins/c.py b/dosagelib/plugins/c.py index f29c0d1f0..cb77b7a56 100644 --- a/dosagelib/plugins/c.py +++ b/dosagelib/plugins/c.py @@ -188,10 +188,13 @@ class CavesAndCritters(_WPWebcomic): adult = True -class Centralia2050(_WordPressScraper): - url = 'http://centralia2050.com/' +class Centralia2050(_ParserScraper): + url = 'https://centralia2050.com/' stripUrl = url + 'comic/%s/' firstStripUrl = stripUrl % 'ch1cover' + imageSearch = '//div[@id="spliced-comic"]//img' + prevSearch = '//a[@class="previous-comic"]' + nextSearch = '//a[@class="next-comic"]' starter = bounceStarter def namer(self, imageUrl, pageUrl): From 73675fb16270a60c1978a635aff02d9a6d04581a Mon Sep 17 00:00:00 2001 From: Techwolf Date: Sat, 24 Apr 2021 18:35:21 -0700 Subject: [PATCH 06/71] Fix unDivine --- dosagelib/plugins/u.py | 16 +++++++++++++++- 1 file changed, 15 insertions(+), 1 deletion(-) diff --git a/dosagelib/plugins/u.py b/dosagelib/plugins/u.py index 0ef199e12..d719c2cc4 100644 --- a/dosagelib/plugins/u.py +++ b/dosagelib/plugins/u.py @@ -27,7 +27,21 @@ class Undertow(_BasicScraper): class unDivine(_ComicControlScraper): - url = 'http://undivinecomic.com/' + url = 'https://www.undivinecomic.com/' + stripUrl = url + 'comic/%s' + firstStripUrl = stripUrl % 'page-1' + + def namer(self, imageUrl, pageUrl): + # Fix inconsistent filenames + filename = imageUrl.rsplit('/', 1)[-1].replace(' ', '-') + filename = filename.replace('10B311D9-0992-4D74-AEB8-DAB714DA67C6', 'UD-322') + filename = filename.replace('99266624-7EF7-4E99-9EC9-DDB5F59CBDFD', 'UD-311') + filename = filename.replace('33C6A5A1-F703-4A0A-BCD5-DE1A09359D8E', 'UD-310') + filename = filename.replace('6CE01E81-C299-43C7-A221-8DE0670EFA30', 'ch4endbonusq4') + filename = filename.replace('DB66D93B-1FE5-49C7-90E0-FFF981DCD6B3', 'bipolar') + if len(filename) > 15 and filename[0].isdigit() and filename[10] == '-': + filename = filename[11:] + return filename class UnicornJelly(_BasicScraper): From ade72c28c5f2bdaa27a0437c0eead80365da8da0 Mon Sep 17 00:00:00 2001 From: Techwolf Date: Sat, 8 May 2021 03:30:00 -0700 Subject: [PATCH 07/71] Fix all MangaDex comics --- dosagelib/plugins/mangadex.py | 156 +++++++++++++++++++--------------- 1 file changed, 88 insertions(+), 68 deletions(-) diff --git a/dosagelib/plugins/mangadex.py b/dosagelib/plugins/mangadex.py index 7f48d9e0e..ea2cb76e6 100644 --- a/dosagelib/plugins/mangadex.py +++ b/dosagelib/plugins/mangadex.py @@ -7,63 +7,83 @@ from ..scraper import _ParserScraper class MangaDex(_ParserScraper): - imageSearch = '//img[contains(@class, "_images")]/@data-url' - prevSearch = '//a[contains(@class, "_prevEpisode")]' multipleImagesPerStrip = True - def __init__(self, name, mangaid): + def __init__(self, name, mangaId): super(MangaDex, self).__init__('MangaDex/' + name) - baseUrl = 'https://mangadex.org/api/' - self.url = baseUrl + '?id=%s&type=manga' % str(mangaid) - self.stripUrl = baseUrl + '?id=%s&type=chapter' + baseUrl = 'https://api.mangadex.org/' + self.url = baseUrl + 'manga/%s' % mangaId + self.chaptersUrl = baseUrl + 'manga/%s/feed?translatedLanguage[]=en&order[chapter]=desc&limit=500' % mangaId + self.stripUrl = baseUrl + 'chapter/%s' + self.imageUrl = 'https://s2.mangadex.org/data/%s/%%s' def starter(self): # Retrieve manga metadata from API - manga = self.session.get(self.url) - manga.raise_for_status() - mangaData = manga.json() + mangaData = self.session.get(self.url) + mangaData.raise_for_status() + manga = mangaData.json()['data'] + + # Retrieve chapter list from API + chapterList = [] + chapterTotal = 1 + chapterOffset = 0 + while len(chapterList) < chapterTotal: + chapterData = self.session.get(self.chaptersUrl + '&offset=%d' % chapterOffset) + chapterData.raise_for_status() + chapterBlock = chapterData.json() + chapterTotal = chapterBlock['total'] + chapterOffset = chapterBlock['offset'] + chapterBlock['limit'] + chapterList.extend(map(lambda c: c['data'], chapterBlock['results'])) + # Determine if manga is complete and/or adult - if mangaData['manga']['last_chapter'] != '0': - for ch in mangaData['chapter']: - if mangaData['chapter'][ch]['chapter'] == mangaData['manga']['last_chapter']: + if manga['attributes']['lastChapter'] != '0': + for chapter in chapterList: + if chapter['attributes']['chapter'] == manga['attributes']['lastChapter']: self.endOfLife = True - if mangaData['manga']['hentai'] != '0': + break + + if manga['attributes']['contentRating'] != 'safe': self.adult = True + # Prepare chapter list self.chapters = [] - for ch in mangaData['chapter']: - if mangaData['chapter'][ch]['lang_code'] != 'gb': - continue + for chapter in chapterList: if len(self.chapters) < 1: - self.chapters.append(ch) + self.chapters.append(chapter) continue - if mangaData['chapter'][ch]['chapter'] == mangaData['chapter'][self.chapters[-1]]['chapter']: + if chapter['attributes']['chapter'] == self.chapters[-1]['attributes']['chapter']: continue - if mangaData['chapter'][ch]['chapter'] == '': + if chapter['attributes']['chapter'] == '': continue - self.chapters.append(ch) + self.chapters.append(chapter) self.chapters.reverse() + # Find first and last chapter - self.firstStripUrl = self.stripUrl % self.chapters[0] - return self.stripUrl % self.chapters[-1] + self.firstStripUrl = self.stripUrl % self.chapters[0]['id'] + return self.stripUrl % self.chapters[-1]['id'] def getPrevUrl(self, url, data): - chapter = url.replace('&type=chapter', '').rsplit('=', 1)[-1] - return self.stripUrl % self.chapters[self.chapters.index(chapter) - 1] + # Determine previous chapter ID from cached list + chapterId = url.rsplit('/', 1)[-1] + chapter = list(filter(lambda c: c['id'] == chapterId, self.chapters)) + if len(chapter) == 0: + return None + return self.stripUrl % self.chapters[self.chapters.index(chapter[0]) - 1]['id'] def fetchUrls(self, url, data, urlSearch): # Retrieve chapter metadata from API chapterData = json.loads(data.text_content()) - self.chapter = chapterData['chapter'] + self.chapter = chapterData['data'] + # Save link order for position-based filenames - imageUrl = chapterData['server'] + chapterData['hash'] + '/%s' - self.imageUrls = [imageUrl % page for page in chapterData['page_array']] + imageUrl = self.imageUrl % self.chapter['attributes']['hash'] + self.imageUrls = [imageUrl % page for page in self.chapter['attributes']['data']] return self.imageUrls def namer(self, imageUrl, pageUrl): # Construct filename from episode number and page index in array - chapterNum = self.chapter + chapterNum = self.chapter['attributes']['chapter'] pageNum = self.imageUrls.index(imageUrl) pageExt = imageUrl.rsplit('.')[-1] return '%s-%02d.%s' % (chapterNum, pageNum, pageExt) @@ -71,44 +91,44 @@ class MangaDex(_ParserScraper): @classmethod def getmodules(cls): return ( - cls('AttackonTitan', 429), - cls('Beastars', 20523), - cls('BokuNoKokoroNoYabaiYatsu', 23811), - cls('DeliciousinDungeon', 13871), - cls('DragonDrive', 5165), - cls('FuguushokuKajishiDakedoSaikyouDesu', 56319), - cls('GanbareDoukiChan', 46585), - cls('HangingOutWithAGamerGirl', 42490), - cls('HoriMiya', 6770), - cls('HowToOpenATriangularRiceball', 19305), - cls('InterspeciesReviewers', 20796), - cls('JahySamaWaKujikenai', 22369), - cls('JingaiNoYomeToIchaIchaSuru', 22651), - cls('KawaiiJoushiWoKomarasetai', 17910), - cls('KanojoOkarishimasu', 22151), - cls('Lv2KaraCheatDattaMotoYuushaKouhoNoMattariIsekaiLife', 33797), - cls('MaouNoOreGaDoreiElfWoYomeNiShitandaGaDouMederebaIi', 25495), - cls('ModernMoGal', 30308), - cls('MyTinySenpaiFromWork', 43610), - cls('OMaidensinYourSavageSeason', 22030), - cls('OokamiShounenWaKyouMoUsoOKasaneru', 14569), - cls('OokamiToKoshinryou', 1168), - cls('OtomeYoukaiZakuro', 4533), - cls('OversimplifiedSCP', 32834), - cls('PashiriNaBokuToKoisuruBanchouSan', 25862), - cls('PleaseDontBullyMeNagatoro', 22631), - cls('PleaseDontBullyMeNagatoroComicAnthology', 31004), - cls('PleaseTellMeGalkochan', 12702), - cls('SaekiSanWaNemutteru', 28834), - cls('SenpaiGaUzaiKouhaiNoHanashi', 23825), - cls('SewayakiKitsuneNoSenkoSan', 22723), - cls('SousouNoFrieren', 48045), - cls('SwordArtOnline', 1360), - cls('SwordArtOnlineProgressive', 9604), - cls('TamenDeGushi', 13939), - cls('TheWolfAndRedRidingHood', 31079), - cls('TomoChanWaOnnanoko', 15722), - cls('TonikakuKawaii', 23439), - cls('YotsubaAnd', 311), - cls('YuYuHakusho', 1738), + cls('AttackOnTitan', '304ceac3-8cdb-4fe7-acf7-2b6ff7a60613'), + cls('Beastars', 'f5e3baad-3cd4-427c-a2ec-ad7d776b370d'), + cls('BokuNoKokoroNoYabaiYatsu', '3df1a9a3-a1be-47a3-9e90-9b3e55b1d0ac'), + cls('DeliciousinDungeon', 'd90ea6cb-7bc3-4d80-8af0-28557e6c4e17'), + cls('DragonDrive', '5c06ae70-b5cf-431a-bcd5-262a411de527'), + cls('FuguushokuKajishiDakedoSaikyouDesu', '17b3b648-fd89-4a69-9a42-6068ffbfa7a7'), + cls('GanbareDoukiChan', '190616bc-7da6-45fd-abd4-dd2ca656c183'), + cls('HangingOutWithAGamerGirl', 'de9e3b62-eac5-4c0a-917d-ffccad694381'), + cls('HoriMiya', 'a25e46ec-30f7-4db6-89df-cacbc1d9a900'), + cls('HowToOpenATriangularRiceball', '6ebd90ce-d5e8-49c0-a4bc-e02e0f8ecb93'), + cls('InterspeciesReviewers', '1b2fddf9-1385-4f3c-b37a-cf86a9428b1a'), + cls('JahySamaWaKujikenai', '2f4e5f5b-d930-4266-8c8a-c4cf9a81e51f'), + cls('JingaiNoYomeToIchaIchaSuru', '809d2493-df3c-4e72-a57e-3e0026cae9fb'), + cls('KawaiiJoushiWoKomarasetai', '23b7cc7a-df89-4049-af28-1fa78f88713e'), + cls('KanojoOkarishimasu', '32fdfe9b-6e11-4a13-9e36-dcd8ea77b4e4'), + cls('Lv2KaraCheatDattaMotoYuushaKouhoNoMattariIsekaiLife', '58bc83a0-1808-484e-88b9-17e167469e23'), + cls('MaouNoOreGaDoreiElfWoYomeNiShitandaGaDouMederebaIi', '55ace2fb-e157-4d76-9e72-67c6bd762a39'), + cls('ModernMoGal', 'b1953f80-36f7-492c-b0f8-e9dd0ad01752'), + cls('MyTinySenpaiFromWork', '28ed63af-61f8-43af-bac3-762030c72963'), + cls('OMaidensinYourSavageSeason', 'c4613b7d-7a6e-48f9-82f0-bce3dd33383a'), + cls('OokamiShounenWaKyouMoUsoOKasaneru', '5e77d9e2-2e44-431a-a995-5fefd411e55e'), + cls('OokamiToKoshinryou', 'de900fd3-c94c-4148-bbcb-ca56eaeb57a4'), + cls('OtomeYoukaiZakuro', 'c1fa97be-0f1f-4686-84bc-806881c97d53'), + cls('OversimplifiedSCP', 'e911fe33-a9b3-43dc-9eb7-f5ee081a6dc8'), + cls('PashiriNaBokuToKoisuruBanchouSan', '838e5b3a-51c8-44cf-b6e2-68193416f6fe'), + cls('PleaseDontBullyMeNagatoro', 'd86cf65b-5f6c-437d-a0af-19a31f94ec55'), + cls('PleaseDontBullyMeNagatoroComicAnthology', '2a4bc9ec-2d70-428a-8b46-27f6218ed267'), + cls('PleaseTellMeGalkochan', '7a2f2f6b-a6a6-4149-879b-3fc2f6916549'), + cls('SaekiSanWaNemutteru', 'd9aecdab-8aef-4b90-98d5-32e86faffb28'), + cls('SenpaiGaUzaiKouhaiNoHanashi', 'af38f328-8df1-4b4c-a272-e737625c3ddc'), + cls('SewayakiKitsuneNoSenkoSan', 'c26269c7-0f5d-4966-8cd5-b79acb86fb7a'), + cls('SousouNoFrieren', 'b0b721ff-c388-4486-aa0f-c2b0bb321512'), + cls('SwordArtOnline', '3dd0b814-23f4-4342-b75b-f206598534f6'), + cls('SwordArtOnlineProgressive', '22ea3f54-11e4-4932-a527-89d63d3a62d9'), + cls('TamenDeGushi', '3f1453fb-9dac-4aca-a2ea-69613856c952'), + cls('TheWolfAndRedRidingHood', 'a7d1283b-ed38-4659-b8bc-47bfca5ccb8a'), + cls('TomoChanWaOnnanoko', '76ee7069-23b4-493c-bc44-34ccbf3051a8'), + cls('TonikakuKawaii', '30f3ac69-21b6-45ad-a110-d011b7aaadaa'), + cls('YotsubaAnd', '58be6aa6-06cb-4ca5-bd20-f1392ce451fb'), + cls('YuYuHakusho', '44a5cbe1-0204-4cc7-a1ff-0fda2ac004b6'), ) From 615dee26e7b45f3685e66c10913c90e143789906 Mon Sep 17 00:00:00 2001 From: Techwolf Date: Sun, 9 May 2021 03:31:32 -0700 Subject: [PATCH 08/71] Add MangaDex/TheNewGate --- dosagelib/plugins/mangadex.py | 1 + 1 file changed, 1 insertion(+) diff --git a/dosagelib/plugins/mangadex.py b/dosagelib/plugins/mangadex.py index ea2cb76e6..75593fa9d 100644 --- a/dosagelib/plugins/mangadex.py +++ b/dosagelib/plugins/mangadex.py @@ -126,6 +126,7 @@ class MangaDex(_ParserScraper): cls('SwordArtOnline', '3dd0b814-23f4-4342-b75b-f206598534f6'), cls('SwordArtOnlineProgressive', '22ea3f54-11e4-4932-a527-89d63d3a62d9'), cls('TamenDeGushi', '3f1453fb-9dac-4aca-a2ea-69613856c952'), + cls('TheNewGate', 'b41bef1e-7df9-4255-bd82-ecf570fec566'), cls('TheWolfAndRedRidingHood', 'a7d1283b-ed38-4659-b8bc-47bfca5ccb8a'), cls('TomoChanWaOnnanoko', '76ee7069-23b4-493c-bc44-34ccbf3051a8'), cls('TonikakuKawaii', '30f3ac69-21b6-45ad-a110-d011b7aaadaa'), From c50b872939177424265bee7593839bc6965c26aa Mon Sep 17 00:00:00 2001 From: Techwolf Date: Wed, 19 May 2021 20:26:34 -0700 Subject: [PATCH 09/71] Add Hackles --- dosagelib/plugins/h.py | 11 ++++++++++- 1 file changed, 10 insertions(+), 1 deletion(-) diff --git a/dosagelib/plugins/h.py b/dosagelib/plugins/h.py index 0ae21bce7..ec44f606a 100644 --- a/dosagelib/plugins/h.py +++ b/dosagelib/plugins/h.py @@ -5,12 +5,21 @@ # Copyright (C) 2019-2020 Daniel Ring from re import compile, escape -from ..scraper import _BasicScraper +from ..scraper import _BasicScraper, _ParserScraper from ..util import tagre from ..helpers import bounceStarter, indirectStarter from .common import _ComicControlScraper, _WordPressScraper, _WPNaviIn +class Hackles(_ParserScraper): + url = 'http://hackles.org/' + stripUrl = url + 'cgi-bin/archives.pl?request=%s' + firstStripUrl = stripUrl % '1' + imageSearch = '//img[contains(@src, "strips/")]' + prevSearch = '//a[text()="< previous"]' + endOfLife = True + + class HagarTheHorrible(_BasicScraper): url = 'http://www.hagarthehorrible.net/' stripUrl = 'http://www.hagardunor.net/comicstrips_us.php?serietype=9&colortype=1&serieno=%s' From 346941b66bc14da93c02b6bd12fc644feb7bb03e Mon Sep 17 00:00:00 2001 From: Techwolf Date: Tue, 25 May 2021 22:42:10 -0700 Subject: [PATCH 10/71] Add StupidFox --- dosagelib/plugins/s.py | 13 +++++++++++++ 1 file changed, 13 insertions(+) diff --git a/dosagelib/plugins/s.py b/dosagelib/plugins/s.py index e6e07e6b5..28ee7675b 100644 --- a/dosagelib/plugins/s.py +++ b/dosagelib/plugins/s.py @@ -660,6 +660,19 @@ class StrongFemaleProtagonist(_ParserScraper): ) +class StupidFox(_ParserScraper): + url = 'http://stupidfox.net/' + stripUrl = url + '%s' + firstStripUrl = stripUrl % 'hello' + imageSearch = '//div[@class="comicmid"]//img' + prevSearch = '//a[@accesskey="p"]' + + def namer(self, imageUrl, pageUrl): + page = self.getPage(pageUrl) + title = page.xpath(self.imageSearch + '/@title')[0].replace(' - ', '-').replace(' ', '-') + return title + '.' + imageUrl.rsplit('.', 1)[-1] + + class SuburbanJungle(_ParserScraper): url = 'http://suburbanjungleclassic.com/' stripUrl = url + '?p=%s' From a657a126007e36a66a3ee47d414feec97d76167a Mon Sep 17 00:00:00 2001 From: Techwolf Date: Mon, 5 Jul 2021 01:05:45 -0700 Subject: [PATCH 11/71] Add ComicFury/Alderwood --- dosagelib/plugins/comicfury.py | 1 + 1 file changed, 1 insertion(+) diff --git a/dosagelib/plugins/comicfury.py b/dosagelib/plugins/comicfury.py index 3bba2afa0..c893b3909 100644 --- a/dosagelib/plugins/comicfury.py +++ b/dosagelib/plugins/comicfury.py @@ -123,6 +123,7 @@ class ComicFury(_ParserScraper): cls('AgentBishop', 'agentbishop'), cls('AHappierKindOfSad', 'ahappierkindofsad'), cls('AlbinoBrothers', 'albinobros'), + cls('Alderwood', 'alderwood'), cls('AlexanderAndLucasRebooted', 'alexanderandlucas'), cls('AliaTerra', 'alia-terra'), cls('AlienIrony', 'alien-irony'), From dee9fa41b770a0dac472e9ac83b649808b441a65 Mon Sep 17 00:00:00 2001 From: Techwolf Date: Sat, 10 Jul 2021 00:35:49 -0700 Subject: [PATCH 12/71] Fix Cloudscratcher --- dosagelib/plugins/c.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/dosagelib/plugins/c.py b/dosagelib/plugins/c.py index cb77b7a56..efe4ea8ed 100644 --- a/dosagelib/plugins/c.py +++ b/dosagelib/plugins/c.py @@ -270,9 +270,9 @@ class Cloudscratcher(_ParserScraper): url = 'http://www.cloudscratcher.com/' stripUrl = url + 'comic.php?page=%s' firstStripUrl = stripUrl % '1' - imageSearch = '//div[@id="main_content"]//img[contains(@src, "comic")]' - prevSearch = '//a[./img[contains(@src, "previous-page")]]' - latestSearch = '//a[@alt="Newest_Page"]' + imageSearch = '//img[contains(@src, "pages/")]' + prevSearch = '//a[./img[@alt="Previous Page"]]' + latestSearch = '//a[./img[@alt="Comic"]]' starter = indirectStarter From 049bbbef688cb8eb56d211b394741e488fa099d1 Mon Sep 17 00:00:00 2001 From: Techwolf Date: Sat, 24 Jul 2021 23:16:59 -0700 Subject: [PATCH 13/71] Add ProjectFuture/IPanther --- dosagelib/plugins/projectfuture.py | 1 + 1 file changed, 1 insertion(+) diff --git a/dosagelib/plugins/projectfuture.py b/dosagelib/plugins/projectfuture.py index bf5a2db81..1ebad3055 100644 --- a/dosagelib/plugins/projectfuture.py +++ b/dosagelib/plugins/projectfuture.py @@ -33,6 +33,7 @@ class ProjectFuture(_ParserScraper): cls('Emily', 'emily', '01-00'), cls('FishingTrip', 'fishing', '01-00'), cls('HeadsYouLose', 'heads', '00-01', last='07-12'), + cls('IPanther', 'panther', '00'), cls('NiallsStory', 'niall', '00'), cls('ProjectFuture', 'strip', '0', last='664'), cls('RedValentine', 'redvalentine', '1', last='6'), From a8f34077b6312a1de9cf9abb16ba3b9a4d622d80 Mon Sep 17 00:00:00 2001 From: Techwolf Date: Sun, 25 Jul 2021 17:28:50 -0700 Subject: [PATCH 14/71] Add WebToons/KissItGoodbye --- dosagelib/plugins/webtoons.py | 1 + 1 file changed, 1 insertion(+) diff --git a/dosagelib/plugins/webtoons.py b/dosagelib/plugins/webtoons.py index 2a2c6fe91..1bd98a42c 100644 --- a/dosagelib/plugins/webtoons.py +++ b/dosagelib/plugins/webtoons.py @@ -224,6 +224,7 @@ class WebToons(_ParserScraper): cls('JustPancakes', 'comedy/just-pancakes', 1651), cls('KidsAreAllRight', 'drama/kids-are-all-right', 283), cls('Killstagram', 'thriller/killstagram', 1971), + cls('KissItGoodbye', 'challenge/kiss-it-goodbye', 443703), cls('KindOfConfidential', 'romance/kind-of-confidential', 663), cls('KindOfLove', 'slice-of-life/kind-of-love', 1850), cls('KnightRun', 'sf/knight-run', 67), From 1856546d6fcc730ad2966bb468d121e593b31b72 Mon Sep 17 00:00:00 2001 From: Techwolf Date: Sun, 25 Jul 2021 17:29:00 -0700 Subject: [PATCH 15/71] Add WebToons/ClawShot --- dosagelib/plugins/webtoons.py | 1 + 1 file changed, 1 insertion(+) diff --git a/dosagelib/plugins/webtoons.py b/dosagelib/plugins/webtoons.py index 1bd98a42c..31e67898d 100644 --- a/dosagelib/plugins/webtoons.py +++ b/dosagelib/plugins/webtoons.py @@ -115,6 +115,7 @@ class WebToons(_ParserScraper): cls('CityOfBlank', 'sf/city-of-blank', 1895), cls('CityOfWalls', 'drama/city-of-wall', 505), cls('CityVamps', 'challenge/city-vamps-', 119224), + cls('ClawShot', 'challenge/clawshot', 621465), cls('ClusterFudge', 'slice-of-life/cluster-fudge', 355), cls('CodeAdam', 'action/code-adam', 1657), cls('CookingComically', 'tiptoon/cooking-comically', 622), From 94339800ad5332bd0cf3e25f33b7b8849acc169e Mon Sep 17 00:00:00 2001 From: Techwolf Date: Wed, 4 Aug 2021 22:26:50 -0700 Subject: [PATCH 16/71] Add DoesNotPlayWellWithOthers --- dosagelib/plugins/d.py | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/dosagelib/plugins/d.py b/dosagelib/plugins/d.py index d0ca80b09..7cce6578f 100644 --- a/dosagelib/plugins/d.py +++ b/dosagelib/plugins/d.py @@ -226,6 +226,13 @@ class DoemainOfOurOwn(_ParserScraper): return filename +class DoesNotPlayWellWithOthers(_WPNavi): + url = 'http://www.doesnotplaywellwithothers.com/' + stripUrl = url + 'comic/%s' + firstStripUrl = stripUrl % 'pwc-0001' + adult = True + + class DoghouseDiaries(_ParserScraper): url = 'http://thedoghousediaries.com/' stripUrl = url + '%s' From 9b6ea0994b63b19ec1e99903ae4494dd1afa6158 Mon Sep 17 00:00:00 2001 From: Techwolf Date: Wed, 4 Aug 2021 23:30:57 -0700 Subject: [PATCH 17/71] Add RaineDog --- dosagelib/plugins/r.py | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/dosagelib/plugins/r.py b/dosagelib/plugins/r.py index 95725338e..b3b1b6774 100644 --- a/dosagelib/plugins/r.py +++ b/dosagelib/plugins/r.py @@ -12,6 +12,16 @@ from ..util import tagre from .common import _WordPressScraper, _WPNavi, _WPWebcomic +class RaineDog(_ParserScraper): + baseUrl = 'http://stuff.dvd3000.ca/dir/rainedog/mirror/' + stripUrl = baseUrl + 'd/%s.html' + url = stripUrl % '20091128' + firstStripUrl = stripUrl % '20090116' + imageSearch = '//img[contains(@src, "imgs/rd")]' + prevSearch = '//a[./img[@id="previous_day1"]]' + endOfLife = True + + class RalfTheDestroyer(_WordPressScraper): url = 'http://ralfthedestroyer.com/' From ca11599ed70d7e057434521370950bddaaf65f29 Mon Sep 17 00:00:00 2001 From: Techwolf Date: Wed, 4 Aug 2021 23:48:44 -0700 Subject: [PATCH 18/71] Add Shifters --- dosagelib/plugins/s.py | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/dosagelib/plugins/s.py b/dosagelib/plugins/s.py index 28ee7675b..592fa77e0 100644 --- a/dosagelib/plugins/s.py +++ b/dosagelib/plugins/s.py @@ -216,6 +216,12 @@ class Sheldon(_BasicScraper): help = 'Index format: yymmdd' +class Shifters(_WPNavi): + url = 'http://shiftersonline.com/' + stripUrl = url + '%s/' + firstStripUrl = stripUrl % 'shifters-redux-promo' + + class ShipInABottle(_WordPressScraper): url = 'http://shipinbottle.pepsaga.com/' stripUrl = url + '?p=%s' From 526947b07e5dd7829e6cd10aa007369c7f02c1bc Mon Sep 17 00:00:00 2001 From: Techwolf Date: Tue, 10 Aug 2021 23:18:13 -0700 Subject: [PATCH 19/71] Fix MangaDex --- dosagelib/plugins/mangadex.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/dosagelib/plugins/mangadex.py b/dosagelib/plugins/mangadex.py index 75593fa9d..826b5a947 100644 --- a/dosagelib/plugins/mangadex.py +++ b/dosagelib/plugins/mangadex.py @@ -16,7 +16,7 @@ class MangaDex(_ParserScraper): self.url = baseUrl + 'manga/%s' % mangaId self.chaptersUrl = baseUrl + 'manga/%s/feed?translatedLanguage[]=en&order[chapter]=desc&limit=500' % mangaId self.stripUrl = baseUrl + 'chapter/%s' - self.imageUrl = 'https://s2.mangadex.org/data/%s/%%s' + self.imageUrl = 'https://s5.mangadex.org/data/%s/%%s' def starter(self): # Retrieve manga metadata from API From afe36b19b936ccb2e39b1a10e0134d843966a305 Mon Sep 17 00:00:00 2001 From: Techwolf Date: Thu, 26 Aug 2021 21:29:34 -0700 Subject: [PATCH 20/71] Add MangaDex/DoChokkyuuKareshiXKanojo --- dosagelib/plugins/mangadex.py | 1 + 1 file changed, 1 insertion(+) diff --git a/dosagelib/plugins/mangadex.py b/dosagelib/plugins/mangadex.py index 826b5a947..84979b737 100644 --- a/dosagelib/plugins/mangadex.py +++ b/dosagelib/plugins/mangadex.py @@ -94,6 +94,7 @@ class MangaDex(_ParserScraper): cls('AttackOnTitan', '304ceac3-8cdb-4fe7-acf7-2b6ff7a60613'), cls('Beastars', 'f5e3baad-3cd4-427c-a2ec-ad7d776b370d'), cls('BokuNoKokoroNoYabaiYatsu', '3df1a9a3-a1be-47a3-9e90-9b3e55b1d0ac'), + cls('DoChokkyuuKareshiXKanojo', 'efb62763-c940-4495-aba5-69c192a999a4'), cls('DeliciousinDungeon', 'd90ea6cb-7bc3-4d80-8af0-28557e6c4e17'), cls('DragonDrive', '5c06ae70-b5cf-431a-bcd5-262a411de527'), cls('FuguushokuKajishiDakedoSaikyouDesu', '17b3b648-fd89-4a69-9a42-6068ffbfa7a7'), From 71db0d5f320569a667b332e52c58661669fe19fc Mon Sep 17 00:00:00 2001 From: Techwolf Date: Sat, 25 Sep 2021 21:07:49 -0700 Subject: [PATCH 21/71] Add KemonoCafe/Bethellium --- dosagelib/plugins/kemonocafe.py | 1 + 1 file changed, 1 insertion(+) diff --git a/dosagelib/plugins/kemonocafe.py b/dosagelib/plugins/kemonocafe.py index b45668247..b2c1a2205 100644 --- a/dosagelib/plugins/kemonocafe.py +++ b/dosagelib/plugins/kemonocafe.py @@ -47,6 +47,7 @@ class KemonoCafe(_ParserScraper): def getmodules(cls): return ( cls('AddictiveScience', 'addictivescience', 'page0001'), + cls('Bethellium', 'bethellium', 'c01p00'), cls('CaribbeanBlue', 'cb', 'page000', last='page325'), cls('IMew', 'imew', 'imew00', last='imew50'), cls('Knighthood', 'knighthood', 'kh0001'), From 8a7476e81dc560d8c09562684254bac453e2ce85 Mon Sep 17 00:00:00 2001 From: Techwolf Date: Sat, 25 Sep 2021 21:08:02 -0700 Subject: [PATCH 22/71] Fix MangaDex --- dosagelib/plugins/mangadex.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/dosagelib/plugins/mangadex.py b/dosagelib/plugins/mangadex.py index 84979b737..425928f53 100644 --- a/dosagelib/plugins/mangadex.py +++ b/dosagelib/plugins/mangadex.py @@ -34,7 +34,7 @@ class MangaDex(_ParserScraper): chapterBlock = chapterData.json() chapterTotal = chapterBlock['total'] chapterOffset = chapterBlock['offset'] + chapterBlock['limit'] - chapterList.extend(map(lambda c: c['data'], chapterBlock['results'])) + chapterList.extend(chapterBlock['data']) # Determine if manga is complete and/or adult if manga['attributes']['lastChapter'] != '0': From 11dce19af5d917e64e28d718cf0fd4d3a34fd991 Mon Sep 17 00:00:00 2001 From: Techwolf Date: Mon, 11 Oct 2021 21:55:10 -0700 Subject: [PATCH 23/71] Fix Unsounded --- dosagelib/plugins/u.py | 20 +++++++++++++++++++- 1 file changed, 19 insertions(+), 1 deletion(-) diff --git a/dosagelib/plugins/u.py b/dosagelib/plugins/u.py index d719c2cc4..983d565f5 100644 --- a/dosagelib/plugins/u.py +++ b/dosagelib/plugins/u.py @@ -4,6 +4,8 @@ # Copyright (C) 2015-2020 Tobias Gruetzmacher # Copyright (C) 2019-2020 Daniel Ring from re import compile +from urllib.parse import urljoin +from lxml import etree from ..scraper import _BasicScraper, _ParserScraper from ..helpers import indirectStarter @@ -59,13 +61,29 @@ class Unsounded(_ParserScraper): startUrl = url + 'comic+index/' stripUrl = url + 'comic/ch%s/ch%s_%s.html' firstStripUrl = stripUrl % ('01', '01', '01') - imageSearch = '//img[contains(@src, "pageart/")]' + imageSearch = '//div[@id="comic"]//img' prevSearch = '//a[d:class("back")]' latestSearch = '//div[@id="chapter_box"][1]//a[last()]' multipleImagesPerStrip = True starter = indirectStarter help = 'Index format: chapter-page' + def fetchUrls(self, url, data, urlSearch): + imageUrls = super(Unsounded, self).fetchUrls(url, data, urlSearch) + # Include background for multi-image pages + imageRegex = compile(r'background-image: url\((pageart/.*)\)') + for match in imageRegex.finditer(str(etree.tostring(data))): + print(match) + searchUrls.append(normaliseURL(urljoin(data[1], match.group(1)))) + return imageUrls + + def namer(self, imageUrl, pageUrl): + filename = imageUrl.rsplit('/', 1)[-1] + pagename = pageUrl.rsplit('/', 1)[-1] + if pagename.split('.', 1)[0] != filename.split('.', 1)[0]: + filename = pagename.split('_', 1)[0] + '_' + filename + return filename + def getPrevUrl(self, url, data): # Fix missing navigation links between chapters if 'ch13/you_let_me_fall' in url: From f82ee7ca14f7620fb37dc199d1da729372b49e39 Mon Sep 17 00:00:00 2001 From: Techwolf Date: Sat, 16 Oct 2021 23:43:58 -0700 Subject: [PATCH 24/71] Fix Everblue --- dosagelib/plugins/e.py | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/dosagelib/plugins/e.py b/dosagelib/plugins/e.py index c70053a4c..049518024 100644 --- a/dosagelib/plugins/e.py +++ b/dosagelib/plugins/e.py @@ -170,12 +170,13 @@ class Erstwhile(_WPNavi): endOfLife = True -class Everblue(_ParserScraper): +class Everblue(_ComicControlScraper): url = 'http://www.everblue-comic.com/comic/' stripUrl = url + '%s' firstStripUrl = stripUrl % '1' - imageSearch = '//article[@id="comic"]//img' - prevSearch = '//a[contains(@class, "prev")]' + + def namer(self, imageUrl, pageUrl): + return imageUrl.rsplit('/', 1)[-1].split('-', 1)[1] class EverybodyLovesEricRaymond(_ParserScraper): From e75362da5d9b0489b8b3cd0f03912a96dd14b26e Mon Sep 17 00:00:00 2001 From: Techwolf Date: Mon, 8 Nov 2021 01:03:13 -0800 Subject: [PATCH 25/71] Fix MonsterSoup --- dosagelib/plugins/comicfury.py | 1 - dosagelib/plugins/m.py | 6 ++++++ dosagelib/plugins/old.py | 1 + 3 files changed, 7 insertions(+), 1 deletion(-) diff --git a/dosagelib/plugins/comicfury.py b/dosagelib/plugins/comicfury.py index c893b3909..c49e81224 100644 --- a/dosagelib/plugins/comicfury.py +++ b/dosagelib/plugins/comicfury.py @@ -674,7 +674,6 @@ class ComicFury(_ParserScraper): cls('MonochromeRainbow', 'monobow'), cls('MonsterBait', 'deadnight'), cls('MonsterInTheKingdom', 'monster'), - cls('MonsterSoup', 'monstersoup'), cls('MonstersWithBenefits', 'failmonsters'), cls('MonstroniverseAdventures', 'monstroniverse'), cls('MoonlitBrew', 'moonlitbrew'), diff --git a/dosagelib/plugins/m.py b/dosagelib/plugins/m.py index 1fbfb89cd..bef5d6f57 100644 --- a/dosagelib/plugins/m.py +++ b/dosagelib/plugins/m.py @@ -187,6 +187,12 @@ class MonsieurLeChien(_BasicScraper): help = 'Index format: n' +class MonsterSoup(_WordPressScraper): + url = 'https://monstersoupcomic.com/' + stripUrl = url + '?comic=%s' + firstStripUrl = stripUrl % 'chapter-1-cover' + + class Moonlace(_WPWebcomic): url = 'https://moonlace.darkbluecomics.com/' stripUrl = url + 'comic/%s/' diff --git a/dosagelib/plugins/old.py b/dosagelib/plugins/old.py index adfa8961b..35c498384 100644 --- a/dosagelib/plugins/old.py +++ b/dosagelib/plugins/old.py @@ -1560,6 +1560,7 @@ class Renamed(Scraper): cls('ComicFury/Elektroanthology', 'ComicFury/ElektrosComicAnthology'), cls('ComicFury/ICanSeeYourFeels', 'ComicFury/SeeYourFeels'), cls('ComicFury/MAGISAupdatesMonWedFri', 'ComicFury/MAGISAPARASAYOupdatesMonFri'), + cls('ComicFury/MonsterSoup', 'MonsterSoup'), cls('ComicFury/OopsComicAdventure', 'OopsComicAdventure'), cls('ComicFury/ThomasAndZachary', 'ComicFury/ThomasAndZacharyArchives'), cls('ComicGenesis/TheLounge', 'KeenSpot/TheLounge'), From 517de17c178b671f7f49bc4ded6b8cd5cabb2063 Mon Sep 17 00:00:00 2001 From: Techwolf Date: Thu, 25 Nov 2021 01:43:24 -0800 Subject: [PATCH 26/71] Add KemonoCafe/PrincessBunny --- dosagelib/plugins/kemonocafe.py | 1 + 1 file changed, 1 insertion(+) diff --git a/dosagelib/plugins/kemonocafe.py b/dosagelib/plugins/kemonocafe.py index b2c1a2205..e0a079ac9 100644 --- a/dosagelib/plugins/kemonocafe.py +++ b/dosagelib/plugins/kemonocafe.py @@ -55,6 +55,7 @@ class KemonoCafe(_ParserScraper): cls('LasLindas', 'laslindas', 'll0001', adult=True), cls('Paprika', 'paprika', 'page000'), cls('PracticeMakesPerfect', 'pmp', 'title-001'), + cls('PrincessBunny', 'princessbunny', 'pg001'), cls('Rascals', 'rascals', 'rascals-pg-0', adult=True), cls('TheEyeOfRamalach', 'theeye', 'theeye-page01'), cls('TinaOfTheSouth', 'tots', 'tos-01-01'), From 78ae0451c9025c0d185ff3c55c8c5ac55bb71930 Mon Sep 17 00:00:00 2001 From: Techwolf Date: Fri, 24 Dec 2021 15:19:16 -0800 Subject: [PATCH 27/71] Remove comics no longer available on the web --- dosagelib/plugins/comicfury.py | 3 --- dosagelib/plugins/old.py | 9 +++++++++ dosagelib/plugins/s.py | 7 ------- dosagelib/plugins/studiokhimera.py | 1 - dosagelib/plugins/t.py | 8 -------- dosagelib/plugins/v.py | 14 -------------- dosagelib/plugins/w.py | 8 -------- 7 files changed, 9 insertions(+), 41 deletions(-) diff --git a/dosagelib/plugins/comicfury.py b/dosagelib/plugins/comicfury.py index c49e81224..cbe9f79fb 100644 --- a/dosagelib/plugins/comicfury.py +++ b/dosagelib/plugins/comicfury.py @@ -577,7 +577,6 @@ class ComicFury(_ParserScraper): cls('KingdomOfTheDinosaurs', 'dinosaurkingdom'), cls('KingdomPrettyCure', 'kingdomprettycure'), cls('KirbyVsShyGuy', 'kvsg'), - cls('Kitsune', 'kitsune'), cls('KMLsSticks', 'kmlssticks'), cls('KnavesEnd', 'knavesend'), cls('KnightGuy', 'knightguy'), @@ -801,8 +800,6 @@ class ComicFury(_ParserScraper): cls('RequiemsGate', 'requiemsgate'), cls('ReSetArt', 'resetfanarts'), cls('ResidentWeirdo', 'residentweirdo'), - cls('ResNullius', 'resnullius'), - cls('ResNulliusCS', 'resnulliuscs'), cls('ReturnOfWonderland', 'returnofwonderland'), cls('Revive', 'revive'), cls('RexAfterDark', 'rexafterdark'), diff --git a/dosagelib/plugins/old.py b/dosagelib/plugins/old.py index 35c498384..e213f766d 100644 --- a/dosagelib/plugins/old.py +++ b/dosagelib/plugins/old.py @@ -617,12 +617,15 @@ class Removed(Scraper): cls('ComicFury/GreenerGrass'), cls('ComicFury/HelloWanderingStar'), cls('ComicFury/Hodgemosh'), + cls('ComicFury/Kitsune'), cls('ComicFury/LaszloAndEdgar'), cls('ComicFury/MegamanComic'), cls('ComicFury/PatchworkPeople'), cls('ComicFury/PornographyInFiveActs'), cls('ComicFury/PoussireDeFe'), cls('ComicFury/RED'), + cls('ComicFury/ResNullius'), + cls('ComicFury/ResNulliusCS'), cls('ComicFury/Seed'), cls('ComicFury/TheAcryden'), cls('ComicFury/TheHourlyComic'), @@ -955,6 +958,7 @@ class Removed(Scraper): cls('MangaDex/HeavensDesignTeam', 'legal'), cls('MangaDex/SPYxFAMILY', 'legal'), cls('Ryugou'), + cls('SeelPeel'), cls('SmackJeeves/20TimesKirby'), cls('SmackJeeves/2Kingdoms'), cls('SmackJeeves/355Days'), @@ -1521,11 +1525,16 @@ class Removed(Scraper): cls('Shivae/CafeAnime'), cls('Shivae/Extras'), cls('SnafuComics/Titan'), + cls('StudioKhimera/Eorah', 'mov'), cls('StuffNoOneToldMe'), + cls('TaleOfTenThousand'), cls('TheCyantianChronicles/CookieCaper'), cls('TheCyantianChronicles/Pawprints'), + cls('VGCats/Adventure'), + cls('VGCats/Super'), cls('VictimsOfTheSystem'), cls('WebDesignerCOTW'), + cls('WintersLight'), ) diff --git a/dosagelib/plugins/s.py b/dosagelib/plugins/s.py index 592fa77e0..e55fbdd05 100644 --- a/dosagelib/plugins/s.py +++ b/dosagelib/plugins/s.py @@ -156,13 +156,6 @@ class Science(_ParserScraper): endOfLife = True -class SeelPeel(_WPNaviIn): - url = 'https://seelpeel.com/' - stripUrl = url + 'comic/%s/' - firstStripUrl = stripUrl % 'seelpeel-goes-live' - multipleImagesPerStrip = True - - class SequentialArt(_ParserScraper): url = 'https://www.collectedcurios.com/sequentialart.php' stripUrl = url + '?s=%s' diff --git a/dosagelib/plugins/studiokhimera.py b/dosagelib/plugins/studiokhimera.py index 081215f93..66dd0447c 100644 --- a/dosagelib/plugins/studiokhimera.py +++ b/dosagelib/plugins/studiokhimera.py @@ -43,7 +43,6 @@ class StudioKhimera(_ParserScraper): @classmethod def getmodules(cls): return ( - cls('Eorah', 'eorah'), cls('Mousechievous', 'mousechievous'), ) diff --git a/dosagelib/plugins/t.py b/dosagelib/plugins/t.py index 0792f63e1..ee055b689 100644 --- a/dosagelib/plugins/t.py +++ b/dosagelib/plugins/t.py @@ -23,14 +23,6 @@ class TailsAndTactics(_ParserScraper): prevSearch = '//a[text()=" Back"]' -class TaleOfTenThousand(_ParserScraper): - url = 'http://www.t10k.club/' - stripUrl = url + 'comic/%s' - firstStripUrl = stripUrl % '1-01_00' - imageSearch = '//article[@id="comic"]//img' - prevSearch = '//a[d:class("prev")]' - - class TekMage(_WPNavi): url = 'https://tekmagecomic.com/' stripUrl = url + 'comic/%s/' diff --git a/dosagelib/plugins/v.py b/dosagelib/plugins/v.py index d375a7ee0..0ea1e7ff5 100644 --- a/dosagelib/plugins/v.py +++ b/dosagelib/plugins/v.py @@ -40,20 +40,6 @@ class VGCats(_ParserScraper): help = 'Index format: n (unpadded)' -class VGCatsAdventure(VGCats): - name = 'VGCats/Adventure' - url = 'http://www.vgcats.com/ffxi/' - stripUrl = url + '?strip_id=%s' - imageSearch = '//p/img[contains(@src, "images/")]' - - -class VGCatsSuper(VGCats): - name = 'VGCats/Super' - url = 'http://www.vgcats.com/super/' - stripUrl = url + '?strip_id=%s' - imageSearch = '//p/img[contains(@src, "images/")]' - - class VickiFox(_ParserScraper): url = 'http://www.vickifox.com/comic/strip' stripUrl = url + '?id=%s' diff --git a/dosagelib/plugins/w.py b/dosagelib/plugins/w.py index 3f8f80cf1..e061c0a3a 100644 --- a/dosagelib/plugins/w.py +++ b/dosagelib/plugins/w.py @@ -133,14 +133,6 @@ class WildeLife(_ComicControlScraper): firstStripUrl = stripUrl % '1' -class WintersLight(_ParserScraper): - url = 'https://winterslight.anaria.net/' - stripUrl = url + 'comic/%s' - firstStripUrl = stripUrl % 'winterslight00' - imageSearch = '//img[contains(@src, "comic/pages/")]' - prevSearch = '//a[contains(text(), "Previous")]' - - class Wonderella(_BasicScraper): url = 'http://nonadventures.com/' rurl = escape(url) From b879ecfb97e3bf5d9eb4d2dcd4475f7df51ecc69 Mon Sep 17 00:00:00 2001 From: Techwolf Date: Fri, 24 Dec 2021 15:24:53 -0800 Subject: [PATCH 28/71] Fix FoxDad --- dosagelib/plugins/f.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/dosagelib/plugins/f.py b/dosagelib/plugins/f.py index 3bd69e1d1..1274899c4 100644 --- a/dosagelib/plugins/f.py +++ b/dosagelib/plugins/f.py @@ -144,7 +144,7 @@ class FoxDad(_ParserScraper): url = 'https://foxdad.com/' stripUrl = url + 'post/%s' firstStripUrl = stripUrl % '149683014997/some-people-are-just-different-support-the-comic' - imageSearch = '//figure[@class="photo-hires-item"]//img' + imageSearch = ('//figure[@class="photo-hires-item"]//img', '//figure[@class="tmblr-full"]//img') prevSearch = '//a[@class="previous-button"]' def namer(self, imageUrl, pageUrl): From 3fb0229780bcecaccf2ac3cd34e1f806e5f4a28c Mon Sep 17 00:00:00 2001 From: Techwolf Date: Fri, 24 Dec 2021 15:37:22 -0800 Subject: [PATCH 29/71] Fix SoloLeveling --- dosagelib/plugins/s.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/dosagelib/plugins/s.py b/dosagelib/plugins/s.py index e55fbdd05..c80da54b9 100644 --- a/dosagelib/plugins/s.py +++ b/dosagelib/plugins/s.py @@ -361,7 +361,7 @@ class SoloLeveling(_ParserScraper): firstStripUrl = stripUrl % '1' imageSearch = '//div[@class="img_container"]//img' prevSearch = '//a[@rel="prev"]' - latestSearch = '//table[@class="chap_tab"]//a' + latestSearch = '//div[@id="latestChapters"]//a' starter = indirectStarter multipleImagesPerStrip = True imageUrlFixes = { From be2116ec278fbed183076a448d65519665e2a8bc Mon Sep 17 00:00:00 2001 From: Techwolf Date: Fri, 24 Dec 2021 17:00:18 -0800 Subject: [PATCH 30/71] Add ComicFury/VHV --- dosagelib/plugins/comicfury.py | 1 + 1 file changed, 1 insertion(+) diff --git a/dosagelib/plugins/comicfury.py b/dosagelib/plugins/comicfury.py index cbe9f79fb..7d6dcb768 100644 --- a/dosagelib/plugins/comicfury.py +++ b/dosagelib/plugins/comicfury.py @@ -1094,6 +1094,7 @@ class ComicFury(_ParserScraper): cls('VampireCatgirlPart2', 'vampirecatgirl2'), cls('VeldaGirlDetective', 'veldagirldetective'), cls('Verboten', 'verboten'), + cls('VHV', 'vhv'), cls('Victory', 'victoryadventures'), cls('ViewHub', 'viewhub'), cls('ViolentBlue', 'violentblue'), From 741fbd1cb1d79d3c0f08e4a154c03e1370bb073b Mon Sep 17 00:00:00 2001 From: Techwolf Date: Fri, 24 Dec 2021 17:03:16 -0800 Subject: [PATCH 31/71] Add TheChroniclesOfHuxcyn --- dosagelib/plugins/t.py | 20 ++++++++++++++++++++ 1 file changed, 20 insertions(+) diff --git a/dosagelib/plugins/t.py b/dosagelib/plugins/t.py index ee055b689..82f72788e 100644 --- a/dosagelib/plugins/t.py +++ b/dosagelib/plugins/t.py @@ -53,6 +53,26 @@ class TheBrads(_ParserScraper): multipleImagesPerStrip = True endOfLife = True +class TheChroniclesOfHuxcyn(_WordPressScraper): + url = 'https://huxcyn.com/' + stripUrl = url + 'comic/%s' + firstStripUrl = stripUrl % 'opening-001' + + def namer(self, imageUrl, pageUrl): + # Fix inconsistent filenames + filename = imageUrl.rsplit('/', 1)[-1] + filename = filename.replace('IMG_0504', 'TCoH109') + filename = filename.replace('tcoh', 'TCoH') + filename = filename.replace('1599151639.xizana_f3a6458e-8d94-4259-bec3-5a92706fe493_jpeg', 'october.2020.cover') + filename = filename.replace('huxonsword', 'october.2020.huxonsword') + filename = filename.replace('New_Canvas100pageswebimage', 'TCoH100') + if filename[0] == '0': + filename = 'TCoH' + filename + elif filename[0] == '3': + pagenum = int(filename.rsplit('.', 1)[0].split('_', 1)[1].split('_', 1)[0]) + filename = 'TCoH' + str(40 + pagenum) + filename.rsplit('.', 1)[-1] + return filename + class TheClassMenagerie(_ParserScraper): stripUrl = 'http://www.theclassm.com/d/%s.html' From 61df811ca3111ee0d31b91de40cfaeaa643df39b Mon Sep 17 00:00:00 2001 From: Techwolf Date: Fri, 24 Dec 2021 21:28:19 -0800 Subject: [PATCH 32/71] Fix FalseStart and VampireHunterBoyfriends --- dosagelib/plugins/f.py | 19 ++++++++++++++----- dosagelib/plugins/v.py | 19 ++++++++++++++----- 2 files changed, 28 insertions(+), 10 deletions(-) diff --git a/dosagelib/plugins/f.py b/dosagelib/plugins/f.py index 1274899c4..0078d69db 100644 --- a/dosagelib/plugins/f.py +++ b/dosagelib/plugins/f.py @@ -21,13 +21,22 @@ class FalconTwin(_BasicScraper): class FalseStart(_ParserScraper): - url = 'https://boneitiscomics.com/falsestart.php' - stripUrl = url + '?pg=%s' - firstStripUrl = stripUrl % '1' - imageSearch = '//div[@class="page"]//img' - prevSearch = '//a[@id="prev"]' + baseUrl = 'https://boneitisindustries.com/' + url = baseUrl + 'comics/false-start/' + stripUrl = baseUrl + 'comic/%s/' + firstStripUrl = stripUrl % 'false-start-chapter-zero-page-1' + imageSearch = '//div[@id="content"]//img[d:class("size-full")]' + prevSearch = '//a[./span[d:class("ticon-chevron-left")]]' adult = True + def starter(self): + archivePage = self.getPage(self.url) + self.archive = archivePage.xpath('//div[contains(@class, "vcex-portfolio-grid")]//a/@href') + return self.archive[-1] + + def getPrevUrl(self, url, data): + return self.archive[self.archive.index(url) - 1] + class Faneurysm(_WPNaviIn): url = 'http://hijinksensue.com/comic/think-only-tree/' diff --git a/dosagelib/plugins/v.py b/dosagelib/plugins/v.py index 0ea1e7ff5..e807882e8 100644 --- a/dosagelib/plugins/v.py +++ b/dosagelib/plugins/v.py @@ -8,13 +8,22 @@ from ..helpers import bounceStarter, indirectStarter class VampireHunterBoyfriends(_ParserScraper): - url = 'https://boneitiscomics.com/vhb.php' - stripUrl = url + '?pg=%s' - firstStripUrl = stripUrl % '1' - imageSearch = '//div[@class="page"]//img' - prevSearch = '//a[@id="prev"]' + baseUrl = 'https://boneitisindustries.com/' + url = baseUrl + 'comics/vampire-hunter-boyfriends/' + stripUrl = baseUrl + 'comic/%s/' + firstStripUrl = stripUrl % 'vampire-hunter-boyfriends-chapter-1-cover' + imageSearch = '//div[@id="content"]//img[d:class("size-full")]' + prevSearch = '//a[./span[d:class("ticon-chevron-left")]]' adult = True + def starter(self): + archivePage = self.getPage(self.url) + self.archive = archivePage.xpath('//div[contains(@class, "vcex-portfolio-grid")]//a/@href') + return self.archive[-1] + + def getPrevUrl(self, url, data): + return self.archive[self.archive.index(url) - 1] + class Vexxarr(_ParserScraper): baseUrl = 'http://www.vexxarr.com/' From 6e5ae63331a6993d64946ba96023fdfb0e91bbec Mon Sep 17 00:00:00 2001 From: Techwolf Date: Tue, 28 Dec 2021 00:40:45 -0800 Subject: [PATCH 33/71] Fix Wrongside and add side stories --- dosagelib/plugins/w.py | 6 --- dosagelib/plugins/wrongside.py | 75 ++++++++++++++++++++++++++++++++++ 2 files changed, 75 insertions(+), 6 deletions(-) create mode 100644 dosagelib/plugins/wrongside.py diff --git a/dosagelib/plugins/w.py b/dosagelib/plugins/w.py index e061c0a3a..63d5657cc 100644 --- a/dosagelib/plugins/w.py +++ b/dosagelib/plugins/w.py @@ -224,9 +224,3 @@ class WormWorldSagaGerman(WormWorldSaga): class WormWorldSagaSpanish(WormWorldSaga): lang = 'es' - - -class Wrongside(_WPNavi): - url = 'http://www.ayzewi.com/comic/' - stripUrl = url + '?comic=%s' - firstStripUrl = stripUrl % 'intro-2' diff --git a/dosagelib/plugins/wrongside.py b/dosagelib/plugins/wrongside.py new file mode 100644 index 000000000..785ed7ec5 --- /dev/null +++ b/dosagelib/plugins/wrongside.py @@ -0,0 +1,75 @@ +# SPDX-License-Identifier: MIT +# Copyright (C) 2019-2020 Tobias Gruetzmacher +# Copyright (C) 2019-2022 Daniel Ring +from ..scraper import _ParserScraper +from ..helpers import indirectStarter + + +class Wrongside(_ParserScraper): + baseUrl = 'http://ayzewi.com/maingallery3/' + url = baseUrl + 'index.php?/category/5' + stripUrl = baseUrl + 'picture.php?%s' + firstStripUrl = stripUrl % '/175/category/21' + imageSearch = '//img[@id="theMainImage"]/@src' + prevSearch = '//a[contains(@title, "Previous :")]' + + def starter(self): + archivePage = self.getPage(self.url) + chapterUrls = archivePage.xpath('//ul[@class="albThumbs"]//a/@href') + self.archive = [] + for chapterUrl in chapterUrls: + chapterPage = self.getPage(chapterUrl) + self.archive.append(chapterPage.xpath('(//ul[@id="thumbnails"]//a/@href)[last()]')[0]) + return self.archive[0] + + def getPrevUrl(self, url, data): + if data.xpath(self.prevSearch) == [] and len(self.archive) > 0: + return self.archive.pop() + return super(Wrongside, self).getPrevUrl(url, data) + + def namer(self, imageUrl, pageUrl): + page = self.getPage(pageUrl) + title = page.xpath('//div[@class="browsePath"]/h2/text()')[0] + return title.replace('"', '') + '.' + imageUrl.rsplit('.', 1)[-1] + + +class WrongsideBeginnings(Wrongside): + name = 'Wrongside/Beginnings' + baseUrl = 'http://ayzewi.com/maingallery3/' + url = baseUrl + 'index.php?/category/4' + stripUrl = baseUrl + 'picture.php?%s' + firstStripUrl = stripUrl % '/2/category/18' + + +class WrongsideSideStories(_ParserScraper): + baseUrl = 'http://ayzewi.com/maingallery3/' + stripUrl = baseUrl + 'picture.php?%s' + imageSearch = '//img[@id="theMainImage"]/@src' + prevSearch = '//a[contains(@title, "Previous :")]' + latestSearch = '(//ul[@id="thumbnails"]//a/@href)[last()]' + starter = indirectStarter + + def __init__(self, name, category, first, last=None): + super().__init__('Wrongside/' + name) + self.url = self.baseUrl + 'index.php?/category/' + category + self.firstStripUrl = self.stripUrl % ('/' + first + '/category/' + category) + + if last: + self.endOfLife = True + + @classmethod + def getmodules(cls): + return ( + cls('AnarkisRising', '7', '302'), + cls('CommonsDreams', '9', '324'), + cls('Faith', '11', '349'), + cls('Sarah', '10', '337'), + cls('ThereAreNoAviansHere', '8', '313'), + cls('TheScientificProphet', '13', '358'), + cls('TheStrangers', '12', '361'), + ) + + def namer(self, imageUrl, pageUrl): + page = self.getPage(pageUrl) + title = page.xpath('//div[@class="browsePath"]/h2/text()')[0] + return title.replace('"', '') + '.' + imageUrl.rsplit('.', 1)[-1] From c691885035fc37ffcb6f118d9cc7680cdd48db1c Mon Sep 17 00:00:00 2001 From: Techwolf Date: Wed, 29 Dec 2021 19:24:39 -0800 Subject: [PATCH 34/71] Add ComicFury/Skylords --- dosagelib/plugins/comicfury.py | 1 + 1 file changed, 1 insertion(+) diff --git a/dosagelib/plugins/comicfury.py b/dosagelib/plugins/comicfury.py index 7d6dcb768..212dd9663 100644 --- a/dosagelib/plugins/comicfury.py +++ b/dosagelib/plugins/comicfury.py @@ -854,6 +854,7 @@ class ComicFury(_ParserScraper): cls('SixteenCandlesHuntersAgency', 'sixteencandles'), cls('Skeeter', 'herecomesskeeter'), cls('Sketchy', 'sketchy'), + cls('Skylords', 'skylords'), cls('SlugMan', 'slug-man'), cls('SmallTownValues', 'smalltownvalues'), cls('SmitheeZombieHunter', 'smitheezombiehunter'), From 764b8432edcefee0a0fb982827cde4fc6e24f0e3 Mon Sep 17 00:00:00 2001 From: Techwolf Date: Fri, 31 Dec 2021 22:44:26 -0800 Subject: [PATCH 35/71] Add MangaDex/CheerfulAmnesia --- dosagelib/plugins/mangadex.py | 1 + 1 file changed, 1 insertion(+) diff --git a/dosagelib/plugins/mangadex.py b/dosagelib/plugins/mangadex.py index 425928f53..de8478e1b 100644 --- a/dosagelib/plugins/mangadex.py +++ b/dosagelib/plugins/mangadex.py @@ -94,6 +94,7 @@ class MangaDex(_ParserScraper): cls('AttackOnTitan', '304ceac3-8cdb-4fe7-acf7-2b6ff7a60613'), cls('Beastars', 'f5e3baad-3cd4-427c-a2ec-ad7d776b370d'), cls('BokuNoKokoroNoYabaiYatsu', '3df1a9a3-a1be-47a3-9e90-9b3e55b1d0ac'), + cls('CheerfulAmnesia', 'f9448f90-c068-4b6a-8c85-03d739aef255'), cls('DoChokkyuuKareshiXKanojo', 'efb62763-c940-4495-aba5-69c192a999a4'), cls('DeliciousinDungeon', 'd90ea6cb-7bc3-4d80-8af0-28557e6c4e17'), cls('DragonDrive', '5c06ae70-b5cf-431a-bcd5-262a411de527'), From 63360b481deccee2b8360ad85d889d2d247c5a17 Mon Sep 17 00:00:00 2001 From: Techwolf Date: Fri, 31 Dec 2021 22:44:45 -0800 Subject: [PATCH 36/71] Add MangaDex/KoiToUtatane --- dosagelib/plugins/mangadex.py | 1 + 1 file changed, 1 insertion(+) diff --git a/dosagelib/plugins/mangadex.py b/dosagelib/plugins/mangadex.py index de8478e1b..be959ea91 100644 --- a/dosagelib/plugins/mangadex.py +++ b/dosagelib/plugins/mangadex.py @@ -108,6 +108,7 @@ class MangaDex(_ParserScraper): cls('JingaiNoYomeToIchaIchaSuru', '809d2493-df3c-4e72-a57e-3e0026cae9fb'), cls('KawaiiJoushiWoKomarasetai', '23b7cc7a-df89-4049-af28-1fa78f88713e'), cls('KanojoOkarishimasu', '32fdfe9b-6e11-4a13-9e36-dcd8ea77b4e4'), + cls('KoiToUtatane', 'f7d40a27-e289-45b3-9c68-d1cb251897e6'), cls('Lv2KaraCheatDattaMotoYuushaKouhoNoMattariIsekaiLife', '58bc83a0-1808-484e-88b9-17e167469e23'), cls('MaouNoOreGaDoreiElfWoYomeNiShitandaGaDouMederebaIi', '55ace2fb-e157-4d76-9e72-67c6bd762a39'), cls('ModernMoGal', 'b1953f80-36f7-492c-b0f8-e9dd0ad01752'), From 6b5ad351b09c00d59815444f360d5eb714e9034a Mon Sep 17 00:00:00 2001 From: Techwolf Date: Fri, 31 Dec 2021 22:44:55 -0800 Subject: [PATCH 37/71] Add MangaDex/RebuildWorld --- dosagelib/plugins/mangadex.py | 1 + 1 file changed, 1 insertion(+) diff --git a/dosagelib/plugins/mangadex.py b/dosagelib/plugins/mangadex.py index be959ea91..9c80d3c5d 100644 --- a/dosagelib/plugins/mangadex.py +++ b/dosagelib/plugins/mangadex.py @@ -122,6 +122,7 @@ class MangaDex(_ParserScraper): cls('PleaseDontBullyMeNagatoro', 'd86cf65b-5f6c-437d-a0af-19a31f94ec55'), cls('PleaseDontBullyMeNagatoroComicAnthology', '2a4bc9ec-2d70-428a-8b46-27f6218ed267'), cls('PleaseTellMeGalkochan', '7a2f2f6b-a6a6-4149-879b-3fc2f6916549'), + cls('RebuildWorld', '99182618-ae92-4aec-a5df-518659b7b613'), cls('SaekiSanWaNemutteru', 'd9aecdab-8aef-4b90-98d5-32e86faffb28'), cls('SenpaiGaUzaiKouhaiNoHanashi', 'af38f328-8df1-4b4c-a272-e737625c3ddc'), cls('SewayakiKitsuneNoSenkoSan', 'c26269c7-0f5d-4966-8cd5-b79acb86fb7a'), From fa14f3f1bc5b8bdf49dbd38d49b92820ae40985c Mon Sep 17 00:00:00 2001 From: Techwolf Date: Fri, 31 Dec 2021 22:47:27 -0800 Subject: [PATCH 38/71] Add MangaDex/IchaichasuruToOkaneGaWaichauFutariNoHanashi --- dosagelib/plugins/mangadex.py | 1 + 1 file changed, 1 insertion(+) diff --git a/dosagelib/plugins/mangadex.py b/dosagelib/plugins/mangadex.py index 9c80d3c5d..8ba02a717 100644 --- a/dosagelib/plugins/mangadex.py +++ b/dosagelib/plugins/mangadex.py @@ -103,6 +103,7 @@ class MangaDex(_ParserScraper): cls('HangingOutWithAGamerGirl', 'de9e3b62-eac5-4c0a-917d-ffccad694381'), cls('HoriMiya', 'a25e46ec-30f7-4db6-89df-cacbc1d9a900'), cls('HowToOpenATriangularRiceball', '6ebd90ce-d5e8-49c0-a4bc-e02e0f8ecb93'), + cls('IchaichasuruToOkaneGaWaichauFutariNoHanashi', '8eaaec7d-7aa7-490e-8d52-5a3d0a28e78b'), cls('InterspeciesReviewers', '1b2fddf9-1385-4f3c-b37a-cf86a9428b1a'), cls('JahySamaWaKujikenai', '2f4e5f5b-d930-4266-8c8a-c4cf9a81e51f'), cls('JingaiNoYomeToIchaIchaSuru', '809d2493-df3c-4e72-a57e-3e0026cae9fb'), From 88d65597c887cabcdf46da502dcdf39b48fb2d19 Mon Sep 17 00:00:00 2001 From: Techwolf Date: Sat, 1 Jan 2022 01:38:55 -0800 Subject: [PATCH 39/71] Add MangaDex/UramikoiKoiUramikoi --- dosagelib/plugins/mangadex.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/dosagelib/plugins/mangadex.py b/dosagelib/plugins/mangadex.py index 8ba02a717..7486434f9 100644 --- a/dosagelib/plugins/mangadex.py +++ b/dosagelib/plugins/mangadex.py @@ -83,7 +83,7 @@ class MangaDex(_ParserScraper): def namer(self, imageUrl, pageUrl): # Construct filename from episode number and page index in array - chapterNum = self.chapter['attributes']['chapter'] + chapterNum = self.chapter['attributes']['chapter'] if self.chapter['attributes']['chapter'] != None else 0 pageNum = self.imageUrls.index(imageUrl) pageExt = imageUrl.rsplit('.')[-1] return '%s-%02d.%s' % (chapterNum, pageNum, pageExt) @@ -135,6 +135,7 @@ class MangaDex(_ParserScraper): cls('TheWolfAndRedRidingHood', 'a7d1283b-ed38-4659-b8bc-47bfca5ccb8a'), cls('TomoChanWaOnnanoko', '76ee7069-23b4-493c-bc44-34ccbf3051a8'), cls('TonikakuKawaii', '30f3ac69-21b6-45ad-a110-d011b7aaadaa'), + cls('UramikoiKoiUramikoi', '009b6788-48f3-4e78-975c-097f54def7ab'), cls('YotsubaAnd', '58be6aa6-06cb-4ca5-bd20-f1392ce451fb'), cls('YuYuHakusho', '44a5cbe1-0204-4cc7-a1ff-0fda2ac004b6'), ) From 19a644cc7c39a9b1d7f108162f456629b85ec065 Mon Sep 17 00:00:00 2001 From: Techwolf Date: Sun, 2 Jan 2022 22:32:07 -0800 Subject: [PATCH 40/71] Fix Supercell --- dosagelib/plugins/s.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/dosagelib/plugins/s.py b/dosagelib/plugins/s.py index c80da54b9..1eb61e218 100644 --- a/dosagelib/plugins/s.py +++ b/dosagelib/plugins/s.py @@ -687,8 +687,9 @@ class SuburbanJungleRoughHousing(_WordPressScraper): class Supercell(_ParserScraper): - url = 'https://www.supercellcomic.com/' - stripUrl = url + 'pages/%s.html' + baseUrl = 'https://www.supercellcomic.com/' + url = baseUrl + 'latest.html' + stripUrl = baseUrl + 'pages/%s.html' firstStripUrl = stripUrl % '0001' imageSearch = '//img[@class="comicStretch"]' prevSearch = '//div[@class="comicnav"]/a[./img[contains(@src, "comnav_02")]]' From cbdf2bc0021145e5127ab23c925af34cf6f5d11f Mon Sep 17 00:00:00 2001 From: Techwolf Date: Sun, 2 Jan 2022 22:32:36 -0800 Subject: [PATCH 41/71] Fix CollegeCatastrophe, NineToNine, and SwordsAndSausages --- dosagelib/plugins/c.py | 2 +- dosagelib/plugins/n.py | 2 +- dosagelib/plugins/s.py | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/dosagelib/plugins/c.py b/dosagelib/plugins/c.py index efe4ea8ed..7f26ce0bf 100644 --- a/dosagelib/plugins/c.py +++ b/dosagelib/plugins/c.py @@ -280,7 +280,7 @@ class CollegeCatastrophe(_ParserScraper): url = 'https://www.tigerknight.com/cc' stripUrl = url + '/%s' firstStripUrl = stripUrl % '2000-11-10' - imageSearch = '//img[@class="comic-image"]' + imageSearch = '//img[d:class("comic-image")]' prevSearch = '//a[./span[contains(text(), "Previous")]]' endOfLife = True multipleImagesPerStrip = True diff --git a/dosagelib/plugins/n.py b/dosagelib/plugins/n.py index 2fb7f4009..2c45ddc53 100644 --- a/dosagelib/plugins/n.py +++ b/dosagelib/plugins/n.py @@ -152,7 +152,7 @@ class NineToNine(_ParserScraper): url = 'https://www.tigerknight.com/99' stripUrl = url + '/%s' firstStripUrl = stripUrl % '2014-01-01' - imageSearch = '//img[@class="comic-image"]' + imageSearch = '//img[d:class("comic-image")]' prevSearch = '//a[./span[contains(text(), "Previous")]]' multipleImagesPerStrip = True diff --git a/dosagelib/plugins/s.py b/dosagelib/plugins/s.py index 1eb61e218..af9051f52 100644 --- a/dosagelib/plugins/s.py +++ b/dosagelib/plugins/s.py @@ -720,6 +720,6 @@ class SwordsAndSausages(_ParserScraper): url = 'https://www.tigerknight.com/ss' stripUrl = url + '/%s' firstStripUrl = stripUrl % '1-1' - imageSearch = '//img[@class="comic-image"]' + imageSearch = '//img[d:class("comic-image")]' prevSearch = '//a[./span[contains(text(), "Previous")]]' multipleImagesPerStrip = True From 4e7b4670697d4431bb4cac1836ef9756779b851b Mon Sep 17 00:00:00 2001 From: Techwolf Date: Sat, 8 Jan 2022 21:48:59 -0800 Subject: [PATCH 42/71] Add MangaDex/SeijoSamaIieToorisugariNoMamonotsukaiDesu --- dosagelib/plugins/mangadex.py | 1 + 1 file changed, 1 insertion(+) diff --git a/dosagelib/plugins/mangadex.py b/dosagelib/plugins/mangadex.py index 7486434f9..4d674087f 100644 --- a/dosagelib/plugins/mangadex.py +++ b/dosagelib/plugins/mangadex.py @@ -125,6 +125,7 @@ class MangaDex(_ParserScraper): cls('PleaseTellMeGalkochan', '7a2f2f6b-a6a6-4149-879b-3fc2f6916549'), cls('RebuildWorld', '99182618-ae92-4aec-a5df-518659b7b613'), cls('SaekiSanWaNemutteru', 'd9aecdab-8aef-4b90-98d5-32e86faffb28'), + cls('SeijoSamaIieToorisugariNoMamonotsukaiDesu', 'd4c40e73-251a-4bcb-a5a6-1edeec1e00e7'), cls('SenpaiGaUzaiKouhaiNoHanashi', 'af38f328-8df1-4b4c-a272-e737625c3ddc'), cls('SewayakiKitsuneNoSenkoSan', 'c26269c7-0f5d-4966-8cd5-b79acb86fb7a'), cls('SousouNoFrieren', 'b0b721ff-c388-4486-aa0f-c2b0bb321512'), From 71cd52b7ca2c867f379743f1e8873b343a12df3d Mon Sep 17 00:00:00 2001 From: Techwolf Date: Sat, 8 Jan 2022 21:49:08 -0800 Subject: [PATCH 43/71] Fix MangaDex site engine --- dosagelib/plugins/mangadex.py | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/dosagelib/plugins/mangadex.py b/dosagelib/plugins/mangadex.py index 4d674087f..d244aa885 100644 --- a/dosagelib/plugins/mangadex.py +++ b/dosagelib/plugins/mangadex.py @@ -16,6 +16,7 @@ class MangaDex(_ParserScraper): self.url = baseUrl + 'manga/%s' % mangaId self.chaptersUrl = baseUrl + 'manga/%s/feed?translatedLanguage[]=en&order[chapter]=desc&limit=500' % mangaId self.stripUrl = baseUrl + 'chapter/%s' + self.cdnUrl = baseUrl + 'at-home/server/%s' self.imageUrl = 'https://s5.mangadex.org/data/%s/%%s' def starter(self): @@ -75,10 +76,13 @@ class MangaDex(_ParserScraper): # Retrieve chapter metadata from API chapterData = json.loads(data.text_content()) self.chapter = chapterData['data'] + cdnData = self.session.get(self.cdnUrl % self.chapter['id']) + cdnData.raise_for_status() + cdnBlock = cdnData.json() # Save link order for position-based filenames - imageUrl = self.imageUrl % self.chapter['attributes']['hash'] - self.imageUrls = [imageUrl % page for page in self.chapter['attributes']['data']] + imageUrl = self.imageUrl % cdnBlock['chapter']['hash'] + self.imageUrls = [imageUrl % page for page in cdnBlock['chapter']['data']] return self.imageUrls def namer(self, imageUrl, pageUrl): From 38c9fc269d6847dd6d57ff46a13875017c892486 Mon Sep 17 00:00:00 2001 From: Techwolf Date: Sun, 16 Jan 2022 02:20:44 -0800 Subject: [PATCH 44/71] Add WebToons/TheGreenhouse --- dosagelib/plugins/webtoons.py | 1 + 1 file changed, 1 insertion(+) diff --git a/dosagelib/plugins/webtoons.py b/dosagelib/plugins/webtoons.py index 31e67898d..4b69a9648 100644 --- a/dosagelib/plugins/webtoons.py +++ b/dosagelib/plugins/webtoons.py @@ -382,6 +382,7 @@ class WebToons(_ParserScraper): cls('TheGirlDownstairs', 'romance/the-girl-downstairs', 1809), cls('THEGIRLFROMCLASS', 'drama/the-girl-from-class', 73), cls('TheGodOfHighSchool', 'action/the-god-of-high-school', 66), + cls('TheGreenhouse', 'challenge/the-greenhouse-gl', 278312), cls('TheKissBet', 'romance/the-kiss-bet', 1617), cls('TheLifeOfTheThreeBears', 'slice-of-life/the-life-of-the-three-bears', 390), cls('ThePurpleHeart', 'super-hero/the-purple-heart', 723), From 9ca43be8b60f44c1cdf8702da26ed26259e5d89a Mon Sep 17 00:00:00 2001 From: Techwolf Date: Thu, 20 Jan 2022 18:04:02 -0800 Subject: [PATCH 45/71] Add MangaDex/KaetteKudasaiAkutsuSan --- dosagelib/plugins/mangadex.py | 1 + 1 file changed, 1 insertion(+) diff --git a/dosagelib/plugins/mangadex.py b/dosagelib/plugins/mangadex.py index d244aa885..20a55c427 100644 --- a/dosagelib/plugins/mangadex.py +++ b/dosagelib/plugins/mangadex.py @@ -111,6 +111,7 @@ class MangaDex(_ParserScraper): cls('InterspeciesReviewers', '1b2fddf9-1385-4f3c-b37a-cf86a9428b1a'), cls('JahySamaWaKujikenai', '2f4e5f5b-d930-4266-8c8a-c4cf9a81e51f'), cls('JingaiNoYomeToIchaIchaSuru', '809d2493-df3c-4e72-a57e-3e0026cae9fb'), + cls('KaetteKudasaiAkutsuSan', '737a846b-2e67-4d63-9f7e-f54b3beebac4'), cls('KawaiiJoushiWoKomarasetai', '23b7cc7a-df89-4049-af28-1fa78f88713e'), cls('KanojoOkarishimasu', '32fdfe9b-6e11-4a13-9e36-dcd8ea77b4e4'), cls('KoiToUtatane', 'f7d40a27-e289-45b3-9c68-d1cb251897e6'), From a8fa0d20649f800a8f4b7055c7f9ab61a7896246 Mon Sep 17 00:00:00 2001 From: Techwolf Date: Fri, 21 Jan 2022 21:28:47 -0800 Subject: [PATCH 46/71] Add KemonoCafe/UltraRosa --- dosagelib/plugins/kemonocafe.py | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/dosagelib/plugins/kemonocafe.py b/dosagelib/plugins/kemonocafe.py index e0a079ac9..38fd68279 100644 --- a/dosagelib/plugins/kemonocafe.py +++ b/dosagelib/plugins/kemonocafe.py @@ -25,10 +25,11 @@ class KemonoCafe(_ParserScraper): def namer(self, imageUrl, pageUrl): # Strip date from filenames filename = imageUrl.rsplit('/', 1)[-1] - if filename[4] == '-' and filename[7] == '-': - filename = filename[10:] - if filename[0] == '-' or filename[0] == '_': - filename = filename[1:] + if not 'ultrarosa' in pageUrl: + if filename[4] == '-' and filename[7] == '-': + filename = filename[10:] + if filename[0] == '-' or filename[0] == '_': + filename = filename[1:] # Fix duplicate filenames if 'paprika' in pageUrl and '69-2' in pageUrl: filename = filename.replace('69', '69-2') @@ -59,4 +60,5 @@ class KemonoCafe(_ParserScraper): cls('Rascals', 'rascals', 'rascals-pg-0', adult=True), cls('TheEyeOfRamalach', 'theeye', 'theeye-page01'), cls('TinaOfTheSouth', 'tots', 'tos-01-01'), + cls('UltraRosa', 'ultrarosa', 'pg001'), ) From 586b5eba3ff5156e03660db93749af074e0a60d6 Mon Sep 17 00:00:00 2001 From: Techwolf Date: Tue, 25 Jan 2022 22:44:25 -0800 Subject: [PATCH 47/71] WebToons/UnintentionalGame --- dosagelib/plugins/webtoons.py | 1 + 1 file changed, 1 insertion(+) diff --git a/dosagelib/plugins/webtoons.py b/dosagelib/plugins/webtoons.py index 4b69a9648..e7aa243b3 100644 --- a/dosagelib/plugins/webtoons.py +++ b/dosagelib/plugins/webtoons.py @@ -415,6 +415,7 @@ class WebToons(_ParserScraper): cls('UnderPrin', 'supernatural/underprin', 78), cls('UnderTheAegis', 'fantasy/under-the-aegis', 436), cls('UnholyBlood', 'supernatural/unholy-blood', 1262), + cls('UnintentionalGame', 'challenge/unintentional-game', 162674), cls('UnknownCaller', 'thriller/ar-toon', 775), cls('UnlovableReplacement', 'romance/unlovable-replacement', 1762), cls('UnluckyIsAsLuckyDoes', 'comedy/unlucky-is-as-lucky-does', 1554), From e7126507653f9896a931ae3d54e120eeb7e30a31 Mon Sep 17 00:00:00 2001 From: Techwolf Date: Wed, 26 Jan 2022 00:38:58 -0800 Subject: [PATCH 48/71] Add WebToons/ReturnToPlayer --- dosagelib/plugins/webtoons.py | 1 + 1 file changed, 1 insertion(+) diff --git a/dosagelib/plugins/webtoons.py b/dosagelib/plugins/webtoons.py index e7aa243b3..c7b2a6195 100644 --- a/dosagelib/plugins/webtoons.py +++ b/dosagelib/plugins/webtoons.py @@ -319,6 +319,7 @@ class WebToons(_ParserScraper): cls('RANDOMPHILIA', 'comedy/randomphilia', 386), cls('Rebirth', 'sf/rebirth', 1412), cls('RefundHighSchool', 'fantasy/refundhighschool', 1360), + cls('ReturnToPlayer', 'action/return-to-player', 2574), cls('RiseFromAshes', 'supernatural/rise-from-ashes', 959), cls('RoarStreetJournal', 'slice-of-life/roar-street-journal', 397), cls('RoomOfSwords', 'sf/room-of-swords', 1261), From b2be007de8449566b6b27121d10dd6ea82d3ee4e Mon Sep 17 00:00:00 2001 From: Techwolf Date: Sat, 12 Feb 2022 03:43:01 -0800 Subject: [PATCH 49/71] Add MangaDex/TadokoroSan --- dosagelib/plugins/mangadex.py | 1 + 1 file changed, 1 insertion(+) diff --git a/dosagelib/plugins/mangadex.py b/dosagelib/plugins/mangadex.py index 20a55c427..2c2fec63d 100644 --- a/dosagelib/plugins/mangadex.py +++ b/dosagelib/plugins/mangadex.py @@ -136,6 +136,7 @@ class MangaDex(_ParserScraper): cls('SousouNoFrieren', 'b0b721ff-c388-4486-aa0f-c2b0bb321512'), cls('SwordArtOnline', '3dd0b814-23f4-4342-b75b-f206598534f6'), cls('SwordArtOnlineProgressive', '22ea3f54-11e4-4932-a527-89d63d3a62d9'), + cls('TadokoroSan', '8ffbfa2f-23fa-4490-848e-942581a4d873'), cls('TamenDeGushi', '3f1453fb-9dac-4aca-a2ea-69613856c952'), cls('TheNewGate', 'b41bef1e-7df9-4255-bd82-ecf570fec566'), cls('TheWolfAndRedRidingHood', 'a7d1283b-ed38-4659-b8bc-47bfca5ccb8a'), From e6571adbd8af795c463041048f6b3b77df506bad Mon Sep 17 00:00:00 2001 From: Techwolf Date: Fri, 11 Feb 2022 20:59:58 -0800 Subject: [PATCH 50/71] Add MangaDex/TenseiShitaraSlimeDattaKen --- dosagelib/plugins/mangadex.py | 1 + 1 file changed, 1 insertion(+) diff --git a/dosagelib/plugins/mangadex.py b/dosagelib/plugins/mangadex.py index 2c2fec63d..4fe169782 100644 --- a/dosagelib/plugins/mangadex.py +++ b/dosagelib/plugins/mangadex.py @@ -138,6 +138,7 @@ class MangaDex(_ParserScraper): cls('SwordArtOnlineProgressive', '22ea3f54-11e4-4932-a527-89d63d3a62d9'), cls('TadokoroSan', '8ffbfa2f-23fa-4490-848e-942581a4d873'), cls('TamenDeGushi', '3f1453fb-9dac-4aca-a2ea-69613856c952'), + cls('TenseiShitaraSlimeDattaKen', 'e78a489b-6632-4d61-b00b-5206f5b8b22b'), cls('TheNewGate', 'b41bef1e-7df9-4255-bd82-ecf570fec566'), cls('TheWolfAndRedRidingHood', 'a7d1283b-ed38-4659-b8bc-47bfca5ccb8a'), cls('TomoChanWaOnnanoko', '76ee7069-23b4-493c-bc44-34ccbf3051a8'), From 3152b0ba28c3c6f6a124ac488822c153d017cbac Mon Sep 17 00:00:00 2001 From: Techwolf Date: Fri, 11 Feb 2022 21:00:07 -0800 Subject: [PATCH 51/71] Add MangaDex/HunterXHunter --- dosagelib/plugins/mangadex.py | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/dosagelib/plugins/mangadex.py b/dosagelib/plugins/mangadex.py index 4fe169782..52bc56d03 100644 --- a/dosagelib/plugins/mangadex.py +++ b/dosagelib/plugins/mangadex.py @@ -50,13 +50,12 @@ class MangaDex(_ParserScraper): # Prepare chapter list self.chapters = [] for chapter in chapterList: - if len(self.chapters) < 1: - self.chapters.append(chapter) - continue - if chapter['attributes']['chapter'] == self.chapters[-1]['attributes']['chapter']: - continue if chapter['attributes']['chapter'] == '': continue + if chapter['attributes']['pages'] == 0: + continue + if len(self.chapters) >= 1 and chapter['attributes']['chapter'] == self.chapters[-1]['attributes']['chapter']: + continue self.chapters.append(chapter) self.chapters.reverse() @@ -107,6 +106,7 @@ class MangaDex(_ParserScraper): cls('HangingOutWithAGamerGirl', 'de9e3b62-eac5-4c0a-917d-ffccad694381'), cls('HoriMiya', 'a25e46ec-30f7-4db6-89df-cacbc1d9a900'), cls('HowToOpenATriangularRiceball', '6ebd90ce-d5e8-49c0-a4bc-e02e0f8ecb93'), + cls('HunterXHunter', 'db692d58-4b13-4174-ae8c-30c515c0689c'), cls('IchaichasuruToOkaneGaWaichauFutariNoHanashi', '8eaaec7d-7aa7-490e-8d52-5a3d0a28e78b'), cls('InterspeciesReviewers', '1b2fddf9-1385-4f3c-b37a-cf86a9428b1a'), cls('JahySamaWaKujikenai', '2f4e5f5b-d930-4266-8c8a-c4cf9a81e51f'), From e30f63ec4d8c4964a4c81f473d86067c8576090f Mon Sep 17 00:00:00 2001 From: Techwolf Date: Wed, 21 Aug 2019 22:36:09 -0700 Subject: [PATCH 52/71] Add Tapastic site engine --- dosagelib/plugins/tapastic.py | 69 +++++++++++++++++++++++++++++++++++ scripts/generate_json.sh | 2 +- scripts/tapastic.py | 36 ++++++++++++++++++ scripts/update_plugins.sh | 2 +- 4 files changed, 107 insertions(+), 2 deletions(-) create mode 100644 dosagelib/plugins/tapastic.py create mode 100644 scripts/tapastic.py diff --git a/dosagelib/plugins/tapastic.py b/dosagelib/plugins/tapastic.py new file mode 100644 index 000000000..3a80fe4a5 --- /dev/null +++ b/dosagelib/plugins/tapastic.py @@ -0,0 +1,69 @@ +# SPDX-License-Identifier: MIT +# Copyright (C) 2019-2020 Tobias Gruetzmacher +# Copyright (C) 2019-2020 Daniel Ring +import json +import re + +from ..scraper import _ParserScraper + + +class Tapastic(_ParserScraper): + baseUrl = 'https://tapas.io/' + imageSearch = '//article[@class="ep-contents"]//img' + episodeIdSearch = re.compile(r'episodeId : (\d+),') + episodeListSearch = re.compile(r'episodeList : (.*),') + multipleImagesPerStrip = True + + def __init__(self, name, url): + super(Tapastic, self).__init__('Tapastic/' + name) + self.url = self.baseUrl + 'series/' + url + self.stripUrl = self.baseUrl + 'episode/%s' + + def starter(self): + # Retrieve series data object + seriesPage = self.getPage(self.url) + dataScript = seriesPage.xpath('//script[contains(text(), "var _data")]')[0].text + # Extract episode list + currentEpisode = self.episodeIdSearch.findall(dataScript)[0] + self.episodeList = json.loads(self.episodeListSearch.findall(dataScript)[0]) + return self.stripUrl % currentEpisode + + def fetchUrls(self, url, data, urlSearch): + # Save link order for position-based filenames + self.imageUrls = super().fetchUrls(url, data, urlSearch) + # Update firstStripUrl with the correct episode title + if int(url.rsplit('/', 1)[-1]) == self.episodeList[0]['id']: + self.firstStripUrl = url + return self.imageUrls + + def getPrevUrl(self, url, data): + episodeId = int(url.rsplit('/', 1)[-1]) + index = [i for i, ep in enumerate(self.episodeList) if ep['id'] == episodeId][0] + if index == 0: + return None + return self.stripUrl % str(self.episodeList[index - 1]['id']) + + def namer(self, imageUrl, pageUrl): + # Construct filename from episode number and image position on page + episodeNum = pageUrl.rsplit('/', 1)[-1] + imageNum = self.imageUrls.index(imageUrl) + imageExt = pageUrl.rsplit('.', 1)[-1] + if len(self.imageUrls) > 1: + filename = "%s-%d.%s" % (episodeNum, imageNum, imageExt) + else: + filename = "%s.%s" % (episodeNum, imageExt) + return filename + + @classmethod + def getmodules(cls): + return ( + # Manually-added comics + cls('NoFuture', 'NoFuture'), + cls('OrensForge', 'OrensForge'), + cls('RavenWolf', 'RavenWolf'), + cls('TheCatTheVineAndTheVictory', 'The-Cat-The-Vine-and-The-Victory'), + cls('TheGodsPack', 'The-Gods-Pack'), + + # START AUTOUPDATE + # END AUTOUPDATE + ) diff --git a/scripts/generate_json.sh b/scripts/generate_json.sh index c28c21ddc..6ff7617c5 100755 --- a/scripts/generate_json.sh +++ b/scripts/generate_json.sh @@ -9,7 +9,7 @@ d=$(dirname $0) if [ $# -ge 1 ]; then list="$*" else - list="arcamax comicfury comicgenesis comicskingdom creators gocomics keenspot webcomicfactory webtoons" + list="arcamax comicfury comicgenesis comicskingdom creators gocomics keenspot tapastic webcomicfactory webtoons" fi for script in $list; do echo "Executing ${script}.py" diff --git a/scripts/tapastic.py b/scripts/tapastic.py new file mode 100644 index 000000000..d285c024e --- /dev/null +++ b/scripts/tapastic.py @@ -0,0 +1,36 @@ +#!/usr/bin/env python3 +# SPDX-License-Identifier: MIT +# Copyright (C) 2019-2020 Tobias Gruetzmacher +# Copyright (C) 2019-2020 Daniel Ring +""" +Script to get a list of Tapastic comics and save the info in a +JSON file for further processing. +""" +from urllib.parse import urlsplit, parse_qs + +from scriptutil import ComicListUpdater +from dosagelib.util import check_robotstxt + + +class TapasticUpdater(ComicListUpdater): + def collect_results(self): + # Retrieve the first 10 top comics list pages + url = 'https://tapas.io/comics?browse=ALL&sort_type=LIKE&pageNumber=' + count = 10 + + data = [self.get_url(url + str(i), robot=False) for i in range(0, count)] + for page in data: + for comiclink in page.xpath('//a[@class="preferred title"]'): + comicurl = comiclink.attrib['href'] + name = comiclink.text + self.add_comic(name, comicurl) + + def get_entry(self, name, url): + shortName = name.replace(' ', '').replace('\'', '') + titleNum = int(parse_qs(urlsplit(url).query)['title_no'][0]) + url = url.rsplit('/', 1)[0].replace('/series/', '') + return u"cls('%s', '%s', %d)," % (shortName, url, titleNum) + + +if __name__ == '__main__': + TapasticUpdater(__file__).run() diff --git a/scripts/update_plugins.sh b/scripts/update_plugins.sh index 97bc7d3e6..8ce0a9126 100755 --- a/scripts/update_plugins.sh +++ b/scripts/update_plugins.sh @@ -11,7 +11,7 @@ d=$(dirname $0) if [ $# -ge 1 ]; then list="$*" else - list="arcamax comicfury comicgenesis comicskingdom creators gocomics keenspot webcomicfactory webtoons" + list="arcamax comicfury comicgenesis comicskingdom creators gocomics keenspot tapastic webcomicfactory webtoons" fi for script in $list; do target="${d}/../dosagelib/plugins/${script}.py" From cb8f67911d96ac6dcf40796803b65a3d0957cad6 Mon Sep 17 00:00:00 2001 From: Techwolf Date: Tue, 7 Apr 2020 01:18:20 -0700 Subject: [PATCH 53/71] Fix Tapastic scraper --- dosagelib/plugins/tapastic.py | 27 +++++---------------------- 1 file changed, 5 insertions(+), 22 deletions(-) diff --git a/dosagelib/plugins/tapastic.py b/dosagelib/plugins/tapastic.py index 3a80fe4a5..4d3bc45a0 100644 --- a/dosagelib/plugins/tapastic.py +++ b/dosagelib/plugins/tapastic.py @@ -5,13 +5,15 @@ import json import re from ..scraper import _ParserScraper +from ..helpers import indirectStarter class Tapastic(_ParserScraper): baseUrl = 'https://tapas.io/' - imageSearch = '//article[@class="ep-contents"]//img' - episodeIdSearch = re.compile(r'episodeId : (\d+),') - episodeListSearch = re.compile(r'episodeList : (.*),') + imageSearch = '//article[contains(@class, "js-episode-article")]//img/@data-src' + prevSearch = '//a[contains(@class, "js-prev-ep-btn")]' + latestSearch = '//ul[contains(@class, "js-episode-list")]//a' + starter = indirectStarter multipleImagesPerStrip = True def __init__(self, name, url): @@ -19,30 +21,11 @@ class Tapastic(_ParserScraper): self.url = self.baseUrl + 'series/' + url self.stripUrl = self.baseUrl + 'episode/%s' - def starter(self): - # Retrieve series data object - seriesPage = self.getPage(self.url) - dataScript = seriesPage.xpath('//script[contains(text(), "var _data")]')[0].text - # Extract episode list - currentEpisode = self.episodeIdSearch.findall(dataScript)[0] - self.episodeList = json.loads(self.episodeListSearch.findall(dataScript)[0]) - return self.stripUrl % currentEpisode - def fetchUrls(self, url, data, urlSearch): # Save link order for position-based filenames self.imageUrls = super().fetchUrls(url, data, urlSearch) - # Update firstStripUrl with the correct episode title - if int(url.rsplit('/', 1)[-1]) == self.episodeList[0]['id']: - self.firstStripUrl = url return self.imageUrls - def getPrevUrl(self, url, data): - episodeId = int(url.rsplit('/', 1)[-1]) - index = [i for i, ep in enumerate(self.episodeList) if ep['id'] == episodeId][0] - if index == 0: - return None - return self.stripUrl % str(self.episodeList[index - 1]['id']) - def namer(self, imageUrl, pageUrl): # Construct filename from episode number and image position on page episodeNum = pageUrl.rsplit('/', 1)[-1] From 3846907435da720c075ab89579b970da5019b49f Mon Sep 17 00:00:00 2001 From: Techwolf Date: Fri, 24 Apr 2020 19:48:54 -0700 Subject: [PATCH 54/71] Add Tapastic/AmpleTime --- dosagelib/plugins/tapastic.py | 1 + 1 file changed, 1 insertion(+) diff --git a/dosagelib/plugins/tapastic.py b/dosagelib/plugins/tapastic.py index 4d3bc45a0..1be31c225 100644 --- a/dosagelib/plugins/tapastic.py +++ b/dosagelib/plugins/tapastic.py @@ -41,6 +41,7 @@ class Tapastic(_ParserScraper): def getmodules(cls): return ( # Manually-added comics + cls('AmpleTime', 'Ample-Time'), cls('NoFuture', 'NoFuture'), cls('OrensForge', 'OrensForge'), cls('RavenWolf', 'RavenWolf'), From 2d56f9417030410ecb606ee6aa5dc8ea58633f83 Mon Sep 17 00:00:00 2001 From: Techwolf Date: Fri, 19 Jun 2020 23:12:53 -0700 Subject: [PATCH 55/71] Fix Tapastic scraper --- dosagelib/plugins/tapastic.py | 14 ++++++++++---- 1 file changed, 10 insertions(+), 4 deletions(-) diff --git a/dosagelib/plugins/tapastic.py b/dosagelib/plugins/tapastic.py index 1be31c225..4cf83777b 100644 --- a/dosagelib/plugins/tapastic.py +++ b/dosagelib/plugins/tapastic.py @@ -1,9 +1,6 @@ # SPDX-License-Identifier: MIT # Copyright (C) 2019-2020 Tobias Gruetzmacher # Copyright (C) 2019-2020 Daniel Ring -import json -import re - from ..scraper import _ParserScraper from ..helpers import indirectStarter @@ -18,9 +15,18 @@ class Tapastic(_ParserScraper): def __init__(self, name, url): super(Tapastic, self).__init__('Tapastic/' + name) - self.url = self.baseUrl + 'series/' + url + self.url = self.baseUrl + 'series/' + url + '/info' self.stripUrl = self.baseUrl + 'episode/%s' + def getPrevUrl(self, url, data): + # Retrieve comic metadata from API + data = self.session.get(url + '/info') + data.raise_for_status() + apiData = data.json()['data'] + if apiData['scene'] == 2: + self.firstStripUrl = self.stripUrl % apiData['prev_ep_id'] + return self.stripUrl % apiData['prev_ep_id'] + def fetchUrls(self, url, data, urlSearch): # Save link order for position-based filenames self.imageUrls = super().fetchUrls(url, data, urlSearch) From 67f787c5895cfd8176bc890e1ceeaa4883fdfea4 Mon Sep 17 00:00:00 2001 From: Techwolf Date: Thu, 21 Jan 2021 01:41:26 -0800 Subject: [PATCH 56/71] Fix Tapastic scraper for comics with episode list in ascending order --- dosagelib/plugins/tapastic.py | 11 ++++++++++- 1 file changed, 10 insertions(+), 1 deletion(-) diff --git a/dosagelib/plugins/tapastic.py b/dosagelib/plugins/tapastic.py index 4cf83777b..57fe4cfe4 100644 --- a/dosagelib/plugins/tapastic.py +++ b/dosagelib/plugins/tapastic.py @@ -10,7 +10,6 @@ class Tapastic(_ParserScraper): imageSearch = '//article[contains(@class, "js-episode-article")]//img/@data-src' prevSearch = '//a[contains(@class, "js-prev-ep-btn")]' latestSearch = '//ul[contains(@class, "js-episode-list")]//a' - starter = indirectStarter multipleImagesPerStrip = True def __init__(self, name, url): @@ -18,6 +17,16 @@ class Tapastic(_ParserScraper): self.url = self.baseUrl + 'series/' + url + '/info' self.stripUrl = self.baseUrl + 'episode/%s' + def starter(self): + # Retrieve comic metadata from info page + info = self.getPage(self.url) + series = info.xpath('//@data-series-id')[0] + # Retrieve comic metadata from API + data = self.session.get(self.baseUrl + 'series/' + series + '/episodes?sort=NEWEST') + data.raise_for_status() + episodes = data.json()['data']['body'] + return self.stripUrl % episodes.split('data-id="')[1].split('"')[0] + def getPrevUrl(self, url, data): # Retrieve comic metadata from API data = self.session.get(url + '/info') From c7d5bbcba5d64f8c6d449c8d8ab7989df22b3359 Mon Sep 17 00:00:00 2001 From: Techwolf Date: Thu, 21 Jan 2021 01:43:43 -0800 Subject: [PATCH 57/71] Fix InsignificantOtters --- dosagelib/plugins/i.py | 8 -------- dosagelib/plugins/tapastic.py | 1 + 2 files changed, 1 insertion(+), 8 deletions(-) diff --git a/dosagelib/plugins/i.py b/dosagelib/plugins/i.py index c5493a1cf..979536bb8 100644 --- a/dosagelib/plugins/i.py +++ b/dosagelib/plugins/i.py @@ -35,14 +35,6 @@ class IDreamOfAJeanieBottle(_WordPressScraper): url = 'http://jeaniebottle.com/' -class InsignificantOtters(_WPWebcomic): - stripUrl = 'https://www.thedepthscomic.com/i-otters/%s/' - firstStripUrl = stripUrl % 'page-1' - url = firstStripUrl - imageSearch = '//div[contains(@class, "webcomic-media")]//img' - starter = indirectStarter - - class InternetWebcomic(_WPNavi): url = 'http://www.internet-webcomic.com/' stripUrl = url + '?p=%s' diff --git a/dosagelib/plugins/tapastic.py b/dosagelib/plugins/tapastic.py index 57fe4cfe4..d373a1572 100644 --- a/dosagelib/plugins/tapastic.py +++ b/dosagelib/plugins/tapastic.py @@ -57,6 +57,7 @@ class Tapastic(_ParserScraper): return ( # Manually-added comics cls('AmpleTime', 'Ample-Time'), + cls('InsignificantOtters', 'IOtters'), cls('NoFuture', 'NoFuture'), cls('OrensForge', 'OrensForge'), cls('RavenWolf', 'RavenWolf'), From 941540e04ee9b7205967f65bd9be04ff5c2abda3 Mon Sep 17 00:00:00 2001 From: Techwolf Date: Tue, 16 Mar 2021 00:55:44 -0700 Subject: [PATCH 58/71] Add Tapastic/SyntheticInstinct --- dosagelib/plugins/tapastic.py | 1 + 1 file changed, 1 insertion(+) diff --git a/dosagelib/plugins/tapastic.py b/dosagelib/plugins/tapastic.py index d373a1572..09965bd54 100644 --- a/dosagelib/plugins/tapastic.py +++ b/dosagelib/plugins/tapastic.py @@ -61,6 +61,7 @@ class Tapastic(_ParserScraper): cls('NoFuture', 'NoFuture'), cls('OrensForge', 'OrensForge'), cls('RavenWolf', 'RavenWolf'), + cls('SyntheticInstinct', 'Synthetic-Instinct'), cls('TheCatTheVineAndTheVictory', 'The-Cat-The-Vine-and-The-Victory'), cls('TheGodsPack', 'The-Gods-Pack'), From fb2e1799700cb678521f9fa44a657b0d2947ed12 Mon Sep 17 00:00:00 2001 From: Techwolf Date: Sat, 24 Apr 2021 02:11:15 -0700 Subject: [PATCH 59/71] Remove Tapastic/TheGodsPack (deleted from host) --- dosagelib/plugins/tapastic.py | 1 - 1 file changed, 1 deletion(-) diff --git a/dosagelib/plugins/tapastic.py b/dosagelib/plugins/tapastic.py index 09965bd54..2dd48b292 100644 --- a/dosagelib/plugins/tapastic.py +++ b/dosagelib/plugins/tapastic.py @@ -63,7 +63,6 @@ class Tapastic(_ParserScraper): cls('RavenWolf', 'RavenWolf'), cls('SyntheticInstinct', 'Synthetic-Instinct'), cls('TheCatTheVineAndTheVictory', 'The-Cat-The-Vine-and-The-Victory'), - cls('TheGodsPack', 'The-Gods-Pack'), # START AUTOUPDATE # END AUTOUPDATE From f394f6093135e56f93394458d4fec531cc0b62d7 Mon Sep 17 00:00:00 2001 From: Techwolf Date: Sat, 27 Nov 2021 22:01:50 -0800 Subject: [PATCH 60/71] Add Tapastic/VenturaCityDrifters --- dosagelib/plugins/tapastic.py | 1 + 1 file changed, 1 insertion(+) diff --git a/dosagelib/plugins/tapastic.py b/dosagelib/plugins/tapastic.py index 2dd48b292..ae840de56 100644 --- a/dosagelib/plugins/tapastic.py +++ b/dosagelib/plugins/tapastic.py @@ -63,6 +63,7 @@ class Tapastic(_ParserScraper): cls('RavenWolf', 'RavenWolf'), cls('SyntheticInstinct', 'Synthetic-Instinct'), cls('TheCatTheVineAndTheVictory', 'The-Cat-The-Vine-and-The-Victory'), + cls('VenturaCityDrifters', 'Ventura-City-Drifters'), # START AUTOUPDATE # END AUTOUPDATE From 5c8156f2e79a76b8589bf2159edaba513427e264 Mon Sep 17 00:00:00 2001 From: Techwolf Date: Thu, 17 Feb 2022 18:12:58 -0800 Subject: [PATCH 61/71] Remove RaineDog (removed from the web) --- dosagelib/plugins/r.py | 10 ---------- 1 file changed, 10 deletions(-) diff --git a/dosagelib/plugins/r.py b/dosagelib/plugins/r.py index b3b1b6774..95725338e 100644 --- a/dosagelib/plugins/r.py +++ b/dosagelib/plugins/r.py @@ -12,16 +12,6 @@ from ..util import tagre from .common import _WordPressScraper, _WPNavi, _WPWebcomic -class RaineDog(_ParserScraper): - baseUrl = 'http://stuff.dvd3000.ca/dir/rainedog/mirror/' - stripUrl = baseUrl + 'd/%s.html' - url = stripUrl % '20091128' - firstStripUrl = stripUrl % '20090116' - imageSearch = '//img[contains(@src, "imgs/rd")]' - prevSearch = '//a[./img[@id="previous_day1"]]' - endOfLife = True - - class RalfTheDestroyer(_WordPressScraper): url = 'http://ralfthedestroyer.com/' From a9421c458f2ecc908dbeefce359876eb54d78cae Mon Sep 17 00:00:00 2001 From: Techwolf Date: Sat, 19 Feb 2022 01:51:15 -0800 Subject: [PATCH 62/71] Fix Guardia --- dosagelib/plugins/comicfury.py | 1 + dosagelib/plugins/g.py | 13 ------------- dosagelib/plugins/old.py | 1 + 3 files changed, 2 insertions(+), 13 deletions(-) diff --git a/dosagelib/plugins/comicfury.py b/dosagelib/plugins/comicfury.py index 212dd9663..17e85792b 100644 --- a/dosagelib/plugins/comicfury.py +++ b/dosagelib/plugins/comicfury.py @@ -464,6 +464,7 @@ class ComicFury(_ParserScraper): cls('GrippsBrain', 'grippsbrain'), cls('GrokBoop', 'grokboop'), cls('GrowingTroubles', 'growingtroubles'), + cls('Guardia', 'guardia-tales-of-halgeis'), cls('GUS', 'gus'), cls('HalloweenCameoCaper2012', 'halloween2012'), cls('HalloweenCameoCaper2013', 'halloween2013'), diff --git a/dosagelib/plugins/g.py b/dosagelib/plugins/g.py index da53e9615..d4b72acf3 100644 --- a/dosagelib/plugins/g.py +++ b/dosagelib/plugins/g.py @@ -145,19 +145,6 @@ class GrrlPower(_WordPressScraper): self.session.add_throttle('grrlpowercomic.com', 1.0, 1.5) -class Guardia(_ParserScraper): - url = 'https://ssp-comics.com/comics/Guardia/' - stripUrl = url + '?page=%s' - firstStripUrl = stripUrl % '1' - imageSearch = '//img[contains(@src, "comics/Guardia/")]' - prevSearch = '//a[./button[@id="prevButton"]]' - nextSearch = '//a[./button[@id="nextButton"]]' - starter = bounceStarter - - def namer(self, imageUrl, pageUrl): - return pageUrl.rsplit('=', 1)[-1] + '.' + imageUrl.rsplit('.', 1)[-1] - - class GuildedAge(_WordPressScraper): url = 'http://guildedage.net/' firstStripUrl = url + 'comic/chapter-1-cover/' diff --git a/dosagelib/plugins/old.py b/dosagelib/plugins/old.py index e213f766d..9bf28c03e 100644 --- a/dosagelib/plugins/old.py +++ b/dosagelib/plugins/old.py @@ -1619,6 +1619,7 @@ class Renamed(Scraper): cls('GoComics/PeanutsEnEspanol', 'GoComics/SnoopyEnEspanol'), cls('GoComics/RipleysBelieveItOrNotSpanish', 'GoComics/RipleysAunqueUstedNoLoCrea'), cls('GoComics/WebcomicName', 'WebcomicName'), + cls('Guardia', 'ComicFury/Guardia'), cls('Shivae/BlackRose', 'BlackRose'), cls('SmackJeeves/BlackTapestries', 'ComicFury/BlackTapestries'), cls('SmackJeeves/ByTheBook', 'ByTheBook'), From 84a1aa8fd92b3eb51fb0297d400aadc6599fc558 Mon Sep 17 00:00:00 2001 From: Techwolf Date: Mon, 21 Feb 2022 23:45:33 -0800 Subject: [PATCH 63/71] Fix MangaDex site engine --- dosagelib/plugins/mangadex.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/dosagelib/plugins/mangadex.py b/dosagelib/plugins/mangadex.py index 52bc56d03..cdd8602af 100644 --- a/dosagelib/plugins/mangadex.py +++ b/dosagelib/plugins/mangadex.py @@ -17,7 +17,7 @@ class MangaDex(_ParserScraper): self.chaptersUrl = baseUrl + 'manga/%s/feed?translatedLanguage[]=en&order[chapter]=desc&limit=500' % mangaId self.stripUrl = baseUrl + 'chapter/%s' self.cdnUrl = baseUrl + 'at-home/server/%s' - self.imageUrl = 'https://s5.mangadex.org/data/%s/%%s' + self.imageUrl = 'https://uploads.mangadex.org/data/%s/%%s' def starter(self): # Retrieve manga metadata from API From d061852cc820d2efd61fd88c765d3018401d1ccb Mon Sep 17 00:00:00 2001 From: Techwolf Date: Fri, 25 Feb 2022 01:08:30 -0800 Subject: [PATCH 64/71] Unify RickGriffinStudios comics --- dosagelib/plugins/a.py | 21 --------- dosagelib/plugins/h.py | 6 --- dosagelib/plugins/old.py | 3 ++ dosagelib/plugins/rickgriffinstudios.py | 59 +++++++++++++++++++++++++ dosagelib/plugins/t.py | 20 --------- 5 files changed, 62 insertions(+), 47 deletions(-) create mode 100644 dosagelib/plugins/rickgriffinstudios.py diff --git a/dosagelib/plugins/a.py b/dosagelib/plugins/a.py index e082134d5..8734adb19 100644 --- a/dosagelib/plugins/a.py +++ b/dosagelib/plugins/a.py @@ -106,27 +106,6 @@ class AGirlAndHerFed(_ParserScraper): help = 'Index format: nnn' -class AHClub(_WPNaviIn): - baseUrl = 'http://rickgriffinstudios.com/' - url = baseUrl + 'ah-club/' - stripUrl = baseUrl + 'comic-post/%s/' - firstStripUrl = stripUrl % 'cover' - latestSearch = '//a[contains(@title, "Permanent Link")]' - starter = indirectStarter - nav = { - 'ah-club-2-cover': 'ah-club-1-page-24', - 'ah-club-3-cover': 'ah-club-2-page-28', - 'ah-club-4-cover': 'ah-club-3-page-22', - } - - def getPrevUrl(self, url, data): - # Links between chapters - url = url.rstrip('/').rsplit('/', 1)[-1] - if self.nav and url in self.nav: - return self.stripUrl % self.nav[url] - return super(AHClub, self).getPrevUrl(url, data) - - class AhoiPolloi(_ParserScraper): url = 'https://ahoipolloi.blogger.de/' stripUrl = url + '?day=%s' diff --git a/dosagelib/plugins/h.py b/dosagelib/plugins/h.py index ec44f606a..21b0fbcc3 100644 --- a/dosagelib/plugins/h.py +++ b/dosagelib/plugins/h.py @@ -116,12 +116,6 @@ class HijinksEnsuePhoto(_WPNaviIn): endOfLife = True -class Housepets(_WordPressScraper): - url = 'http://www.housepetscomic.com/' - stripUrl = url + 'comic/%s/' - firstStripUrl = '2008/06/02/when-boredom-strikes' - - class HowToBeAWerewolf(_ComicControlScraper): url = 'http://howtobeawerewolf.com/' stripUrl = url + 'comic/%s' diff --git a/dosagelib/plugins/old.py b/dosagelib/plugins/old.py index 9bf28c03e..91e958509 100644 --- a/dosagelib/plugins/old.py +++ b/dosagelib/plugins/old.py @@ -1611,6 +1611,7 @@ class Renamed(Scraper): cls('ZebraGirl', 'ComicFury/ZebraGirl'), # Renamed in 3.0 + cls('AHClub', 'RickGriffinStudios/AHClub'), cls('CrapIDrewOnMyLunchBreak', 'WebToons/CrapIDrewOnMyLunchBreak'), cls('DeerMe', 'ComicFury/DeerMe'), cls('GoComics/BloomCounty2017', 'GoComics/BloomCounty2019'), @@ -1627,4 +1628,6 @@ class Renamed(Scraper): cls('SmackJeeves/GrowingTroubles', 'ComicFury/GrowingTroubles'), cls('SmackJeeves/TheRealmOfKaerwyn', 'ComicFury/TheRealmOfKaerwyn'), cls('StudioKhimera/Draconia', 'Draconia'), + cls('TracesOfThePast', 'RickGriffinStudios/TracesOfThePast'), + cls('TracesOfThePast/NSFW', 'RickGriffinStudios/TracesOfThePastNSFW'), ) diff --git a/dosagelib/plugins/rickgriffinstudios.py b/dosagelib/plugins/rickgriffinstudios.py new file mode 100644 index 000000000..c57ae6dc9 --- /dev/null +++ b/dosagelib/plugins/rickgriffinstudios.py @@ -0,0 +1,59 @@ +# SPDX-License-Identifier: MIT +# Copyright (C) 2019-2020 Tobias Gruetzmacher +# Copyright (C) 2019-2022 Daniel Ring +from ..helpers import indirectStarter +from .common import _WordPressScraper, _WPNaviIn + + +class Housepets(_WordPressScraper): + url = 'http://www.housepetscomic.com/' + stripUrl = url + 'comic/%s/' + firstStripUrl = '2008/06/02/when-boredom-strikes' + + +class RickGriffinStudios(_WPNaviIn): + baseUrl = 'http://rickgriffinstudios.com/' + stripUrl = baseUrl + 'comic-post/%s/' + latestSearch = '//a[contains(@title, "Permanent Link")]' + starter = indirectStarter + nav = None + + def __init__(self, name, sub, first, last=None, adult=False, nav=None): + super().__init__('RickGriffinStudios/' + name) + self.url = self.baseUrl + sub + '/' + self.firstStripUrl = self.stripUrl % first + + if last: + self.url = self.stripUrl % last + self.starter = super(RickGriffinStudios, self).starter + self.endOfLife = True + + if adult: + self.latestSearch = '//a[contains(@title, "NSFW")]' + self.adult = True + + if nav: + self.nav = nav + + def getPrevUrl(self, url, data): + # Links between chapters + url = url.rstrip('/').rsplit('/', 1)[-1] + if self.nav and url in self.nav: + return self.stripUrl % self.nav[url] + return super(RickGriffinStudios, self).getPrevUrl(url, data) + + @classmethod + def getmodules(cls): + return ( + cls('AHClub', 'ah-club', 'cover', nav={ + 'ah-club-2-cover': 'ah-club-1-page-24', + 'ah-club-3-cover': 'ah-club-2-page-28', + 'ah-club-4-cover': 'ah-club-3-page-22', + 'ah-club-5-cover': 'ah-club-4-page-24' + }), + cls('HayvenCelestia', 'hayven-celestia', 'skinchange-p1'), + cls('TheStoryboard', 'the-storyboard', 'the-storyboard-001'), + cls('TracesOfThePast', 'in-the-new-age', 'totp-page-1'), + cls('TracesOfThePastNSFW', 'in-the-new-age', 'totp-page-1-nsfw', adult=True), + cls('ZootopiaNightTerrors', 'zootopia-night-terrors', 'zootopia-night-terrors-p1', 'zootopia-night-terrors-p7'), + ) diff --git a/dosagelib/plugins/t.py b/dosagelib/plugins/t.py index 82f72788e..7e8c1b369 100644 --- a/dosagelib/plugins/t.py +++ b/dosagelib/plugins/t.py @@ -290,26 +290,6 @@ class ToonHole(_WordPressScraper): return url in (self.url + "comic/if-game-of-thrones-was-animated/",) -class TracesOfThePast(_WPNaviIn): - baseUrl = 'http://rickgriffinstudios.com/' - url = baseUrl + 'in-the-new-age/' - stripUrl = baseUrl + 'comic-post/%s/' - firstStripUrl = stripUrl % 'totp-page-1' - latestSearch = '//a[contains(@title, "Permanent Link")]' - starter = indirectStarter - - -class TracesOfThePastNSFW(_WPNaviIn): - name = 'TracesOfThePast/NSFW' - baseUrl = 'http://rickgriffinstudios.com/' - url = baseUrl + 'in-the-new-age/' - stripUrl = baseUrl + 'comic-post/%s/' - firstStripUrl = stripUrl % 'totp-page-1-nsfw' - latestSearch = '//a[contains(@title, "NSFW")]' - starter = indirectStarter - adult = True - - class TrippingOverYou(_BasicScraper): url = 'http://www.trippingoveryou.com/' stripUrl = url + 'comic/%s' From ec6b9ce71ad72a6b64e502d0f38252c126fd680a Mon Sep 17 00:00:00 2001 From: Techwolf Date: Tue, 8 Mar 2022 01:03:03 -0800 Subject: [PATCH 65/71] Add WebToons/TheRoomies --- dosagelib/plugins/webtoons.py | 1 + 1 file changed, 1 insertion(+) diff --git a/dosagelib/plugins/webtoons.py b/dosagelib/plugins/webtoons.py index c7b2a6195..6468b18b5 100644 --- a/dosagelib/plugins/webtoons.py +++ b/dosagelib/plugins/webtoons.py @@ -390,6 +390,7 @@ class WebToons(_ParserScraper): cls('TheRedBook', 'horror/the-red-book', 467), cls('TheRedHook', 'super-hero/the-red-hook', 643), cls('TheRedKing', 'supernatural/the-red-king', 1687), + cls('TheRoomies', 'challenge/the-roomies-archive', 513669), cls('TheShadowProphet', 'drama/the-shadow-prophet', 1881), cls('TheSoundOfYourHeart', 'comedy/the-sound-of-your-heart', 269), cls('TheSteamDragonExpress', 'fantasy/steam-dragon-express', 1270), From ebf12c732401da1023d9b6eb538b5c12baddc279 Mon Sep 17 00:00:00 2001 From: Techwolf Date: Wed, 30 Mar 2022 23:06:03 -0700 Subject: [PATCH 66/71] Add ComicFury/Sharak --- dosagelib/plugins/comicfury.py | 1 + 1 file changed, 1 insertion(+) diff --git a/dosagelib/plugins/comicfury.py b/dosagelib/plugins/comicfury.py index 17e85792b..eb293fd5c 100644 --- a/dosagelib/plugins/comicfury.py +++ b/dosagelib/plugins/comicfury.py @@ -844,6 +844,7 @@ class ComicFury(_ParserScraper): cls('ShakingOffSorceryPL', 'shakingoffsorcery-pl'), cls('ShamanQuest', 'shamanquest'), cls('ShatteredSkies', 'shatteredskies'), + cls('Sharak', 'sharak'), cls('Shenanigans', 's'), cls('ShenaniganSquares', 'ss-comic'), cls('ShikuTheFirstAndFinal', 'shiku'), From fed7b0825dc2a6dc5c27d38e5b5f2bf19d17b910 Mon Sep 17 00:00:00 2001 From: Techwolf Date: Tue, 5 Apr 2022 01:04:28 -0700 Subject: [PATCH 67/71] Fix WebToons/Defects --- dosagelib/plugins/webtoons.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/dosagelib/plugins/webtoons.py b/dosagelib/plugins/webtoons.py index 6468b18b5..e9c9ccb9d 100644 --- a/dosagelib/plugins/webtoons.py +++ b/dosagelib/plugins/webtoons.py @@ -135,7 +135,7 @@ class WebToons(_ParserScraper): cls('DEADDAYS', 'horror/dead-days', 293), cls('Debunkers', 'challenge/debunkers', 148475), cls('DEEP', 'thriller/deep', 364), - cls('Defects', 'challenge/defects', 221106), + cls('Defects', 'fantasy/defects', 2731), cls('Denma', 'sf/denma', 921), cls('Dents', 'sf/dents', 671), cls('Deor', 'fantasy/deor', 1663), From 01c4ddcd2c5de2b3cd20e9b2922527dbad99e04a Mon Sep 17 00:00:00 2001 From: Techwolf Date: Tue, 5 Apr 2022 01:06:08 -0700 Subject: [PATCH 68/71] Add MangaDex/ShinNoJitsuryokuWaGirigiriMadeKakushiteIyouToOmou --- dosagelib/plugins/mangadex.py | 1 + 1 file changed, 1 insertion(+) diff --git a/dosagelib/plugins/mangadex.py b/dosagelib/plugins/mangadex.py index cdd8602af..3ed0e35e0 100644 --- a/dosagelib/plugins/mangadex.py +++ b/dosagelib/plugins/mangadex.py @@ -133,6 +133,7 @@ class MangaDex(_ParserScraper): cls('SeijoSamaIieToorisugariNoMamonotsukaiDesu', 'd4c40e73-251a-4bcb-a5a6-1edeec1e00e7'), cls('SenpaiGaUzaiKouhaiNoHanashi', 'af38f328-8df1-4b4c-a272-e737625c3ddc'), cls('SewayakiKitsuneNoSenkoSan', 'c26269c7-0f5d-4966-8cd5-b79acb86fb7a'), + cls('ShinNoJitsuryokuWaGirigiriMadeKakushiteIyouToOmou', '22fda941-e603-4601-a536-c3ad6d004ba8'), cls('SousouNoFrieren', 'b0b721ff-c388-4486-aa0f-c2b0bb321512'), cls('SwordArtOnline', '3dd0b814-23f4-4342-b75b-f206598534f6'), cls('SwordArtOnlineProgressive', '22ea3f54-11e4-4932-a527-89d63d3a62d9'), From a1bcdfb98750454b2e9158dd639a8c53c5496f5d Mon Sep 17 00:00:00 2001 From: Techwolf Date: Tue, 5 Apr 2022 01:06:24 -0700 Subject: [PATCH 69/71] Fix SoloLeveling --- dosagelib/plugins/mangadex.py | 1 + dosagelib/plugins/old.py | 1 + dosagelib/plugins/s.py | 60 ----------------------------------- 3 files changed, 2 insertions(+), 60 deletions(-) diff --git a/dosagelib/plugins/mangadex.py b/dosagelib/plugins/mangadex.py index 3ed0e35e0..ea943edd3 100644 --- a/dosagelib/plugins/mangadex.py +++ b/dosagelib/plugins/mangadex.py @@ -134,6 +134,7 @@ class MangaDex(_ParserScraper): cls('SenpaiGaUzaiKouhaiNoHanashi', 'af38f328-8df1-4b4c-a272-e737625c3ddc'), cls('SewayakiKitsuneNoSenkoSan', 'c26269c7-0f5d-4966-8cd5-b79acb86fb7a'), cls('ShinNoJitsuryokuWaGirigiriMadeKakushiteIyouToOmou', '22fda941-e603-4601-a536-c3ad6d004ba8'), + cls('SoloLeveling', '32d76d19-8a05-4db0-9fc2-e0b0648fe9d0'), cls('SousouNoFrieren', 'b0b721ff-c388-4486-aa0f-c2b0bb321512'), cls('SwordArtOnline', '3dd0b814-23f4-4342-b75b-f206598534f6'), cls('SwordArtOnlineProgressive', '22ea3f54-11e4-4932-a527-89d63d3a62d9'), diff --git a/dosagelib/plugins/old.py b/dosagelib/plugins/old.py index 91e958509..da12ea04d 100644 --- a/dosagelib/plugins/old.py +++ b/dosagelib/plugins/old.py @@ -1627,6 +1627,7 @@ class Renamed(Scraper): cls('SmackJeeves/FurryExperience', 'ComicFury/FurryExperience'), cls('SmackJeeves/GrowingTroubles', 'ComicFury/GrowingTroubles'), cls('SmackJeeves/TheRealmOfKaerwyn', 'ComicFury/TheRealmOfKaerwyn'), + cls('SoloLeveling', 'MangaDex/SoloLeveling'), cls('StudioKhimera/Draconia', 'Draconia'), cls('TracesOfThePast', 'RickGriffinStudios/TracesOfThePast'), cls('TracesOfThePast/NSFW', 'RickGriffinStudios/TracesOfThePastNSFW'), diff --git a/dosagelib/plugins/s.py b/dosagelib/plugins/s.py index af9051f52..a543977dd 100644 --- a/dosagelib/plugins/s.py +++ b/dosagelib/plugins/s.py @@ -355,66 +355,6 @@ class SodiumEyes(_WordPressScraper): endOfLife = True -class SoloLeveling(_ParserScraper): - url = 'https://w3.sololeveling.net/' - stripUrl = url + 'manga/solo-leveling-chapter-%s/' - firstStripUrl = stripUrl % '1' - imageSearch = '//div[@class="img_container"]//img' - prevSearch = '//a[@rel="prev"]' - latestSearch = '//div[@id="latestChapters"]//a' - starter = indirectStarter - multipleImagesPerStrip = True - imageUrlFixes = { - '94-0_5dd574efda419/28.': '94-0_5dd574efda419/28a.', - '92-0_5dc2fcb9ed562/22.': '92-0_5dc2fcb9ed562/22s.', - '91-0_5db9b881ac2f0/20k.': '91-0_5db9b881ac2f0/20l.', - '91-0_5db9b881ac2f0/23.': '91-0_5db9b881ac2f0/23a.', - '90-0_5db08467ca2b1/07.': '90-0_5db08467ca2b1/07a.', - '90-0_5db08467ca2b1/09.': '90-0_5db08467ca2b1/09a.', - '90-0_5db08467ca2b1/13.': '90-0_5db08467ca2b1/13a.', - '90-0_5db08467ca2b1/14.': '90-0_5db08467ca2b1/14a.', - '90-0_5db08467ca2b1/21.': '90-0_5db08467ca2b1/21a.', - '90-0_5db08467ca2b1/22.': '90-0_5db08467ca2b1/22a.', - '88-0_5d9e0dedb942e/03.': '88-0_5d9e0dedb942e/03b.', - '88-0_5d9e0dedb942e/05.': '88-0_5d9e0dedb942e/05a.', - '88-0_5d9e0dedb942e/30.': '88-0_5d9e0dedb942e/30a.', - '87-0_5d94cdebd9df7/01a.': '87-0_5d94cdebd9df7/01c.', - } - - def imageUrlModifier(self, imageUrl, data): - if 'url=' in imageUrl: - imageUrl = imageUrl.split('url=')[1].split('&')[0] - for fix in self.imageUrlFixes: - imageUrl = imageUrl.replace(fix, self.imageUrlFixes[fix]) - return imageUrl - - def fetchUrls(self, url, data, urlSearch): - # Save link order for position-based filenames - self.imageUrls = super(SoloLeveling, self).fetchUrls(url, data, urlSearch) - self.imageUrls = [self.imageUrlModifier(x, data) for x in self.imageUrls] - return self.imageUrls - - def getPage(self, url): - try: - return super().getPage(url) - except HTTPError as e: - # CloudFlare WAF - if e.response.status_code == 403 and '1020' in e.response.text: - self.geoblocked() - else: - raise e - - def getPrevUrl(self, url, data): - return self.stripUrl % str(int(url.strip('/').rsplit('-', 1)[-1]) - 1) - - def namer(self, imageUrl, pageUrl): - # Construct filename from episode number and image position on page - episodeNum = pageUrl.strip('/').rsplit('-', 1)[-1] - imageNum = self.imageUrls.index(imageUrl) - imageExt = imageUrl.rsplit('.', 1)[-1] - return "%s-%03d.%s" % (episodeNum, imageNum, imageExt) - - class SomethingPositive(_ParserScraper): url = 'https://www.somethingpositive.net/' stripUrl = url + 'sp%s.shtml' From 18b8a79ab7aa449d5664e586af6994ee4f882bd3 Mon Sep 17 00:00:00 2001 From: Techwolf Date: Wed, 6 Apr 2022 02:43:47 -0700 Subject: [PATCH 70/71] Fix Replay --- dosagelib/plugins/r.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/dosagelib/plugins/r.py b/dosagelib/plugins/r.py index 95725338e..69947e968 100644 --- a/dosagelib/plugins/r.py +++ b/dosagelib/plugins/r.py @@ -116,9 +116,8 @@ class Requiem(_WordPressScraper): class Replay(_ParserScraper): url = 'http://replaycomic.com/' stripUrl = url + 'comic/%s/' - url = stripUrl % 'trying-it-out' firstStripUrl = stripUrl % 'red-desert' - imageSearch = '//div[@id="comic"]//img' + imageSearch = '//div[@id="comic"]//img[@alt]' prevSearch = '//a[contains(@class, "comic-nav-previous")]' nextSearch = '//a[contains(@class, "comic-nav-next")]' From a6d2e11503374ba3e772e202e46b8888164780c4 Mon Sep 17 00:00:00 2001 From: Tobias Gruetzmacher Date: Sat, 16 Apr 2022 00:11:01 +0200 Subject: [PATCH 71/71] Disable geoblocking test Since we removed the standalone SoloLeveling module, we have no geoblocking comic modules at the moment, so there is no good way to test geoblocking anymore... --- tests/test_modules.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/tests/test_modules.py b/tests/test_modules.py index 84f11ab58..c6ec1e021 100644 --- a/tests/test_modules.py +++ b/tests/test_modules.py @@ -7,8 +7,6 @@ import responses import dosagelib.cmd import httpmocks -from dosagelib.plugins.s import SoloLeveling -from dosagelib.scraper import GeoblockedException def cmd(*options): @@ -43,7 +41,11 @@ class TestModules(object): cmd('--basepath', str(tmpdir), 'CalvinAndHobbesEnEspanol:2012/07/22') @responses.activate + @pytest.mark.skip(reason="SoloeLeveling was removed, so we have no way to test this...") def test_sololeveling_geoblock(self, tmpdir): + from dosagelib.plugins.s import SoloLeveling + from dosagelib.scraper import GeoblockedException + responses.add(responses.GET, 'https://w3.sololeveling.net/', '1020', status=403)