More cleanup after SmackJeeves redesign

This commit is contained in:
Tobias Gruetzmacher 2019-12-29 23:19:17 +01:00
parent c0ebf9bbe2
commit addf78fcbc
3 changed files with 40 additions and 37 deletions

View file

@ -29,6 +29,8 @@ class ComicFury(_ParserScraper):
u'//nav//a[contains(text(), "\u2039")]',
# RedSpot
'//a[contains(text(), "Back")]',
# KATRAN
'//a[contains(text(), "Previous")]',
)
nextSearch = (
'//link[@rel="next"]',
@ -39,7 +41,7 @@ class ComicFury(_ParserScraper):
XPATH_IMG % ('Next'),
# Art, ConsolersDLC, etc.
u'//nav//a[contains(text(), "\u203A")]',
# RedSpot
# RedSpot, KATRAN
'//a[contains(text(), "Next")]',
)
help = 'Index format: n'
@ -569,6 +571,7 @@ class ComicFury(_ParserScraper):
cls('JustPeachy', 'justpeachy'),
cls('KarensEdge', 'karensedge'),
cls('Katastrophe', 'katastrophe'),
cls('KATRAN', 'katran'),
cls('KayAndP', 'kayandp'),
cls('KazasMateGwenna', 'kaza-and-gwenna'),
cls('KAZE', 'kaze'),

View file

@ -440,7 +440,10 @@ class Removed(Scraper):
cls('SmackJeeves/Anathemacomics'),
cls('SmackJeeves/AngelBeast'),
cls('SmackJeeves/ArchportCityChronicles'),
# moved to www.mgcomics.com, which has a robots.txt for everything
cls('SmackJeeves/Aware', 'block'),
cls('SmackJeeves/AwesomeSauce'),
cls('SmackJeeves/Babywhatsyoursign'),
cls('SmackJeeves/BetweenLightandDark'),
cls('SmackJeeves/BetweenWorlds'),
cls('SmackJeeves/BeyondTemptation'),
@ -454,6 +457,7 @@ class Removed(Scraper):
cls('SmackJeeves/ChaosTheory2005'),
cls('SmackJeeves/CleanCure'),
cls('SmackJeeves/DaddysGirl'),
cls('SmackJeeves/DeadtoDay'),
cls('SmackJeeves/Debtsettlement'),
cls('SmackJeeves/DebtSettlement2OperationExtinction'),
cls('SmackJeeves/DefyingGravityTheFourGreatGuardians'),
@ -461,11 +465,14 @@ class Removed(Scraper):
cls('SmackJeeves/DevilTrainee'),
cls('SmackJeeves/DevilTraineeSpanish'),
cls('SmackJeeves/Diexemor'),
cls('SmackJeeves/DontDie'),
cls('SmackJeeves/ElfenLiedDifferences'),
cls('SmackJeeves/Entuthrie'),
cls('SmackJeeves/EozinKadonnutKuningas'),
cls('SmackJeeves/EpicChaos'),
cls('SmackJeeves/EternalKnights'),
cls('SmackJeeves/EvD'),
cls('SmackJeeves/FatetheAnthologyofKaienandhisfuckingmagicfriends'),
cls('SmackJeeves/FeathersPI'),
cls('SmackJeeves/FemmeSchism'),
cls('SmackJeeves/FireWire'),
@ -475,10 +482,14 @@ class Removed(Scraper):
cls('SmackJeeves/FurtherDowntheRabbitHole'),
cls('SmackJeeves/GATEKEEPER'),
cls('SmackJeeves/GearTheTakedown'),
cls('SmackJeeves/GoldenSunGenerationsAftermathVolume1'),
cls('SmackJeeves/GoldenSunGenerationsColossoVolume6'),
cls('SmackJeeves/GraveImpressions'),
cls('SmackJeeves/GreenKirbyandabunchofotherpeopledoinstuff'),
cls('SmackJeeves/GuardianGhost'),
cls('SmackJeeves/Harfang'),
cls('SmackJeeves/HIPS'),
cls('SmackJeeves/HitandMiss'),
cls('SmackJeeves/HotChocolate'),
cls('SmackJeeves/Hybristorific'),
cls('SmackJeeves/Ianua'),
@ -492,6 +503,7 @@ class Removed(Scraper):
cls('SmackJeeves/LastLivingSouls'),
cls('SmackJeeves/LatchkeyKingdom'),
cls('SmackJeeves/LegendsofMobiusBookOne'),
cls('SmackJeeves/LetsBreakitforReals'),
cls('SmackJeeves/LiliBleu'),
cls('SmackJeeves/LoveTwister'),
cls('SmackJeeves/MagicalGirlAlice'),
@ -503,31 +515,46 @@ class Removed(Scraper):
cls('SmackJeeves/MixupofallMixups'),
cls('SmackJeeves/MomthegamestorerippedusoffAGAIN'),
cls('SmackJeeves/MoonlitDawnAMythicalTale'),
cls('SmackJeeves/MUTE', 'mov'),
cls('SmackJeeves/MyBoyfriendisaMobBoss'),
cls('SmackJeeves/MyTrollLife'),
cls('SmackJeeves/MyTwoCentsPlusTax'),
cls('SmackJeeves/NeoCrystalAdventures'),
cls('SmackJeeves/NihilWandasJourney'),
cls('SmackJeeves/OddContact'),
cls('SmackJeeves/Ohman', 'brk'),
cls('SmackJeeves/OneFrameGags'),
cls('SmackJeeves/Pahantekija'),
cls('SmackJeeves/Paradox'),
cls('SmackJeeves/Paripety'),
cls('SmackJeeves/Perinto'),
cls('SmackJeeves/PerplexingMagnoliaDisruption', 'mov'),
cls('SmackJeeves/PlatonicBoyfriends'),
cls('SmackJeeves/Plotlessnesses'),
cls('SmackJeeves/PokemonGleamingCrystal', 'mis'),
cls('SmackJeeves/PokemonMysteryDungeonTeamCrystal'),
cls('SmackJeeves/PRAGUERACE'),
cls('SmackJeeves/PumpkinFlower'),
cls('SmackJeeves/Razor'),
cls('SmackJeeves/RedVelvetRequiem'),
cls('SmackJeeves/RuderiQuest'),
cls('SmackJeeves/SAKANA'),
cls('SmackJeeves/SenoireDelirium'),
cls('SmackJeeves/SerendipityAnEquestrianTale'),
cls('SmackJeeves/ShacklesInstallment02'),
cls('SmackJeeves/SimonSues'),
cls('SmackJeeves/SimplySarah'),
cls('SmackJeeves/SomebodyShootMe'),
cls('SmackJeeves/SonicUniverseAsk'),
cls('SmackJeeves/SoulGuardian'),
cls('SmackJeeves/SpaghettiAndMeatballs'),
cls('SmackJeeves/SparkStory', 'mis'),
cls('SmackJeeves/Spidersilk', 'mov'),
cls('SmackJeeves/Symbios'),
cls('SmackJeeves/TechnicolorLondon'),
cls('SmackJeeves/TeKscloset'),
cls('SmackJeeves/TheAttackoftheRecoloursSeason1'),
cls('SmackJeeves/TheAvianStories'),
cls('SmackJeeves/TheCurtandTonyShow'),
cls('SmackJeeves/TheDarkAgeofMobius'),
cls('SmackJeeves/TheHobbitbic'),
@ -535,6 +562,7 @@ class Removed(Scraper):
cls('SmackJeeves/TheKeyToReality'),
cls('SmackJeeves/TheLostland'),
cls('SmackJeeves/TheMewExperiment'),
cls('SmackJeeves/TheMoistTouch'),
cls('SmackJeeves/TheRandomObscureFairyTaleNoOnesEverReallyHeardOf'),
cls('SmackJeeves/TheSomewhereOther'),
cls('SmackJeeves/TheWastelands', 'mis'),
@ -687,6 +715,7 @@ class Renamed(Scraper):
cls('SmackJeeves/CityFolk', 'ComicFury/CityFolk'),
cls('SmackJeeves/DoomsdayMyDear', 'DoomsdayMyDear'),
cls('SmackJeeves/ForestHill', 'ForestHill'),
cls('SmackJeeves/Katran', 'ComicFury/KATRAN'),
cls('SmackJeeves/Magience', 'ComicFury/Magience'),
cls('SmackJeeves/RiversideExtras', 'RiversideExtras'),
cls('SmackJeeves/StarTrip', 'StarTrip'),

View file

@ -16,20 +16,21 @@ class SmackJeeves(_ParserScraper):
imageSearch = re.compile("comicData:[^']*'([^']*)'", re.DOTALL)
help = 'Index format: n'
def __init__(self, name, id, adult=False, endOfLife=False):
def __init__(self, name, id, adult=False, endOfLife=False, last=None):
super(SmackJeeves, self).__init__('SmackJeeves/' + name)
self._comicid = id
self.url = self.baseUrl + 'articleList?titleNo={}'.format(id)
self.stripUrl = self.baseUrl + 'detail?titleNo={}&articleNo=%s'.format(id)
self.firstStripUrl = self.stripUrl % 1
self.adult = adult
self.endOfLife = endOfLife
self.endOfLife = endOfLife or last
self.lastid = (last or 0) - 1
def starter(self):
response = self.session.post(self.apiBase + 'articleList',
params={'titleNo': self._comicid})
response.raise_for_status()
return response.json()['result']['list'][-1]['articleUrl']
return response.json()['result']['list'][self.lastid]['articleUrl']
def fetchUrls(self, url, data, urlsearch):
if urlsearch != self.imageSearch:
@ -94,11 +95,9 @@ class SmackJeeves(_ParserScraper):
cls('AtArmsLength', 39242),
cls('Atlaswebcomic', 140920),
cls('Autophobia', 82522),
cls('Aware', 132405),
cls('AyaTakeo', 60325),
cls('AYuriCollab', 126727, adult=True),
cls('BabysittingFourDemons', 5992),
cls('Babywhatsyoursign', 123587),
cls('BadassRiz', 130263),
cls('BallandChain', 99503),
cls('Bard', 51147),
@ -170,7 +169,6 @@ class SmackJeeves(_ParserScraper):
cls('Dasien', 1779, adult=True),
cls('DavidDoesntGetIt', 115176),
cls('DBON', 76205),
cls('DeadtoDay', 137063),
cls('DeathNoteIridescent', 87149),
cls('DEGAF', 120316),
cls('DEMENTED', 104334, adult=True),
@ -183,10 +181,8 @@ class SmackJeeves(_ParserScraper):
cls('Diaz', 129935),
cls('DigimonSaviors', 43521),
cls('DigimonTamersMiraiProject', 119257),
cls('DigisRandomSpriteshack', 47936),
cls('DigitalInsanity', 144866),
cls('DoItYourself', 103195),
cls('DontDie', 140541),
cls('DoodleBeans', 17473, adult=True),
cls('DoodlingAround', 117225),
cls('Dragonet', 119508),
@ -201,7 +197,6 @@ class SmackJeeves(_ParserScraper):
cls('EnkeltenKentta', 78834, adult=True),
cls('Enthrall', 113047, adult=True),
cls('EntreEuxDeux', 97894, endOfLife=True),
cls('Entuthrie', 114564, adult=True),
cls('Eorah', 135499, adult=True),
cls('Equsopia', 127680),
cls('ERAConvergence', 63750, endOfLife=True),
@ -218,7 +213,6 @@ class SmackJeeves(_ParserScraper):
cls('FallenAngelslove', 113208),
cls('FarOutMantic', 48658),
cls('FarOutThere', 89199),
cls('FatetheAnthologyofKaienandhisfuckingmagicfriends', 140771),
cls('FeralGentry', 119912),
cls('FinalArcanum', 51191),
cls('FireredLisasReise', 109151),
@ -241,12 +235,9 @@ class SmackJeeves(_ParserScraper):
cls('GigisNuzlockeRuns', 157589),
cls('Gloomverse', 75498),
cls('Gnoph', 138286, endOfLife=True),
cls('GoldenSunGenerationsAftermathVolume1', 17885),
cls('GoldenSunGenerationsColossoVolume6', 17945),
cls('GoodGame', 48048, endOfLife=True),
cls('GoodnightMrsGoose', 73655),
cls('Grayscale', 100295, adult=True),
cls('GuardianGhost', 118429),
cls('GuardiansoftheGalaxialSpaceways', 70286),
cls('Habibahssong', 137453),
cls('HarvestMoonParadiseFound', 105422, endOfLife=True),
@ -254,12 +245,10 @@ class SmackJeeves(_ParserScraper):
cls('HatShop', 71816),
cls('Helix', 121066),
cls('HeltonShelton', 124804),
cls('HendrixAndJulian', 155252, adult=True),
cls('Hephaestus', 79909),
cls('HereBeVoodoo', 133691, adult=True),
cls('HiddenStrengthAWhiteNuzlocke', 116170),
cls('Hinata', 134703),
cls('HitandMiss', 82248),
cls('Holocrash', 64113, endOfLife=True),
cls('HolyBlasphemy', 91250),
cls('HolyCrap', 13235),
@ -287,7 +276,6 @@ class SmackJeeves(_ParserScraper):
cls('Justmyluck', 169106),
cls('KaitoShuno', 100498, adult=True),
cls('KasaKeira', 78059),
cls('Katran', 79778),
cls('KazanatoFuneralPlanningService', 24026),
cls('KCNO', 103010, endOfLife=True),
cls('KezroChroniclesPhantomOps', 104447),
@ -319,7 +307,6 @@ class SmackJeeves(_ParserScraper):
cls('LesCendresdelHiver', 131767),
cls('LethalDose', 115067, adult=True),
cls('LetLoveRule', 177216),
cls('LetsBreakitforReals', 127909),
cls('LicensedHeroes', 123974),
cls('LifeAsACutOut', 86222),
cls('LifeAsItWas', 117747),
@ -338,7 +325,7 @@ class SmackJeeves(_ParserScraper):
cls('LoveroftheSunandMoon', 121615),
cls('LsEmpire', 33751),
cls('LuffinpuffandEric', 83450),
cls('LumasParadise', 46770),
cls('LumasParadise', 46770, last=174),
cls('MagicalMisfits', 138166),
cls('Magipunk', 131204),
cls('Manifestedpart1', 140208),
@ -368,7 +355,6 @@ class SmackJeeves(_ParserScraper):
cls('MorphE', 122406),
cls('Mortifer', 17271),
cls('MrFactory', 119931),
cls('MUTE', 32960, endOfLife=True),
cls('MyFakeHeart', 19105),
cls('MySisterTheDragon', 9612, endOfLife=True),
cls('MySparklingPrincesama', 105626),
@ -376,10 +362,8 @@ class SmackJeeves(_ParserScraper):
cls('MysticanDreams', 130154),
cls('MythsOfUnovaAWhiteNuzlockeRunHardMode', 101385),
cls('MYth', 36430),
cls('MyTwoCentsPlusTax', 38595),
cls('Nah', 88350),
cls('Negligence', 66718),
cls('NeoCrystalAdventures', 2060),
cls('NeonGlow', 63747),
cls('NeverTheHero', 132200),
cls('Nexus', 115357),
@ -397,7 +381,6 @@ class SmackJeeves(_ParserScraper):
cls('ObsidianHeart', 130610),
cls('October20th', 122344),
cls('OddPlaceOddTime', 60037),
cls('Ohman', 143967),
cls('OldElastikid', 97110),
cls('OneRainyDay', 111352, adult=True),
cls('Onlyonelovesong', 86555),
@ -407,17 +390,14 @@ class SmackJeeves(_ParserScraper):
cls('Outbreak', 134869),
cls('OutofKey', 123152),
cls('OverSync', 114781),
cls('Pahantekija', 139286),
cls('Panacea', 79601, adult=True),
cls('PantsParty', 70630),
cls('PanzerDragonandEnigmaCompleteEdition', 20039),
cls('Paradox', 29667, adult=True),
cls('Pause', 135615),
cls('PencilviewUpdatesMondayscough', 33041),
cls('PerplexingMagnoliaDisruption', 104698),
cls('PeterPan', 119991),
cls('Phantomland', 106943),
cls('PhotoShootNarusasuDoujinshi', 110984, adult=True),
cls('PhotoShootNarusasuDoujinshi', 110984, adult=True, last=187),
cls('PlasticKings', 132377),
cls('PlayTime', 68440),
cls('PleaseBeMyBoytoy', 26922),
@ -427,7 +407,6 @@ class SmackJeeves(_ParserScraper):
cls('PokemonBeta', 76962, endOfLife=True),
cls('PokemonCrystalDoubleNuzlockeChallenge', 102420, endOfLife=True),
cls('PokemonLANDSKY', 120046),
cls('PokemonMysteryDungeonTeamCrystal', 122892),
cls('PokemonNoRakuen', 22822),
cls('PokemonParallel', 86143),
cls('PokemonSAKOHJU', 120395),
@ -449,7 +428,6 @@ class SmackJeeves(_ParserScraper):
cls('RANDOM', 99296),
cls('RareCandyTreatment', 83853),
cls('RavenWolf', 97826),
cls('RedVelvetRequiem', 127363),
cls('Regina', 129902),
cls('ReidyAndFriendsShowcase', 45097),
cls('RemoteAngel', 46191),
@ -460,7 +438,6 @@ class SmackJeeves(_ParserScraper):
cls('RottenApple', 133069),
cls('RoyalIcing', 81321),
cls('RubyNation', 107039),
cls('RuderiQuest', 111648),
cls('RuneSpark', 140886),
cls('RUScrewed', 144524),
cls('RyuManwebcomicVersion', 135497, endOfLife=True),
@ -488,7 +465,6 @@ class SmackJeeves(_ParserScraper):
cls('ShroudofLight', 121119),
cls('Signifikat', 12217, adult=True),
cls('SimpleBear', 120596),
cls('SimplySarah', 33343),
cls('Sire', 58167),
cls('Skeptical', 108904),
cls('Slackmatic', 144286),
@ -496,7 +472,6 @@ class SmackJeeves(_ParserScraper):
cls('SlipstreamSingularity', 66132),
cls('SmallPressAdventures', 70273),
cls('SocksMittensandScarfs', 132813),
cls('SomebodyShootMe', 121562),
cls('SomethingLikeaPhenomenon', 176182, adult=True),
cls('SonicAuthorAdventII', 5867),
cls('SonicBoom', 111375),
@ -507,9 +482,7 @@ class SmackJeeves(_ParserScraper):
cls('SOSRadio', 107498),
cls('SouthernCross', 94541),
cls('SovereignTheMostAmazingComicEver', 129062),
cls('SpaghettiAndMeatballs', 130401, adult=True),
cls('SparElricsExtras', 125431),
cls('SparkStory', 131006),
cls('Spellcross', 125055),
cls('SpiderWings', 36314),
cls('SplitScreen', 125293, adult=True, endOfLife=True),
@ -545,7 +518,6 @@ class SmackJeeves(_ParserScraper):
cls('TheAntihero', 64167),
cls('TheArchipelago', 142266),
cls('Theatrics', 93578),
cls('TheAvianStories', 145492),
cls('TheBattleInTheSky', 143006),
cls('TheBookOfNosferatu', 97421),
cls('TheBrideoftheShark', 61393, adult=True),
@ -579,7 +551,6 @@ class SmackJeeves(_ParserScraper):
cls('TheLoneSwordsman', 18218),
cls('TheMadMan', 83928),
cls('TheMegaManandSonicSpriteShowcase', 75689),
cls('TheMoistTouch', 135862),
cls('TheNightSurfers', 120705),
cls('TheNocheComicSeries', 133439, adult=True),
cls('TheNOMEDSEGA', 127642),
@ -640,7 +611,7 @@ class SmackJeeves(_ParserScraper):
cls('WildWingBoysKoathArc', 50240),
cls('WildWingBoys', 203574),
cls('WingsOverEthereal', 37417),
cls('WingsTurnedtoDust', 113033),
cls('WingsTurnedtoDust', 113033, last=417),
cls('Wootlabs', 101636),
cls('XXMoralityXx', 132016),
cls('YadotCakeShop', 106995, adult=True),