More cleanup after SmackJeeves redesign
This commit is contained in:
parent
c0ebf9bbe2
commit
addf78fcbc
3 changed files with 40 additions and 37 deletions
|
@ -29,6 +29,8 @@ class ComicFury(_ParserScraper):
|
|||
u'//nav//a[contains(text(), "\u2039")]',
|
||||
# RedSpot
|
||||
'//a[contains(text(), "Back")]',
|
||||
# KATRAN
|
||||
'//a[contains(text(), "Previous")]',
|
||||
)
|
||||
nextSearch = (
|
||||
'//link[@rel="next"]',
|
||||
|
@ -39,7 +41,7 @@ class ComicFury(_ParserScraper):
|
|||
XPATH_IMG % ('Next'),
|
||||
# Art, ConsolersDLC, etc.
|
||||
u'//nav//a[contains(text(), "\u203A")]',
|
||||
# RedSpot
|
||||
# RedSpot, KATRAN
|
||||
'//a[contains(text(), "Next")]',
|
||||
)
|
||||
help = 'Index format: n'
|
||||
|
@ -569,6 +571,7 @@ class ComicFury(_ParserScraper):
|
|||
cls('JustPeachy', 'justpeachy'),
|
||||
cls('KarensEdge', 'karensedge'),
|
||||
cls('Katastrophe', 'katastrophe'),
|
||||
cls('KATRAN', 'katran'),
|
||||
cls('KayAndP', 'kayandp'),
|
||||
cls('KazasMateGwenna', 'kaza-and-gwenna'),
|
||||
cls('KAZE', 'kaze'),
|
||||
|
|
|
@ -440,7 +440,10 @@ class Removed(Scraper):
|
|||
cls('SmackJeeves/Anathemacomics'),
|
||||
cls('SmackJeeves/AngelBeast'),
|
||||
cls('SmackJeeves/ArchportCityChronicles'),
|
||||
# moved to www.mgcomics.com, which has a robots.txt for everything
|
||||
cls('SmackJeeves/Aware', 'block'),
|
||||
cls('SmackJeeves/AwesomeSauce'),
|
||||
cls('SmackJeeves/Babywhatsyoursign'),
|
||||
cls('SmackJeeves/BetweenLightandDark'),
|
||||
cls('SmackJeeves/BetweenWorlds'),
|
||||
cls('SmackJeeves/BeyondTemptation'),
|
||||
|
@ -454,6 +457,7 @@ class Removed(Scraper):
|
|||
cls('SmackJeeves/ChaosTheory2005'),
|
||||
cls('SmackJeeves/CleanCure'),
|
||||
cls('SmackJeeves/DaddysGirl'),
|
||||
cls('SmackJeeves/DeadtoDay'),
|
||||
cls('SmackJeeves/Debtsettlement'),
|
||||
cls('SmackJeeves/DebtSettlement2OperationExtinction'),
|
||||
cls('SmackJeeves/DefyingGravityTheFourGreatGuardians'),
|
||||
|
@ -461,11 +465,14 @@ class Removed(Scraper):
|
|||
cls('SmackJeeves/DevilTrainee'),
|
||||
cls('SmackJeeves/DevilTraineeSpanish'),
|
||||
cls('SmackJeeves/Diexemor'),
|
||||
cls('SmackJeeves/DontDie'),
|
||||
cls('SmackJeeves/ElfenLiedDifferences'),
|
||||
cls('SmackJeeves/Entuthrie'),
|
||||
cls('SmackJeeves/EozinKadonnutKuningas'),
|
||||
cls('SmackJeeves/EpicChaos'),
|
||||
cls('SmackJeeves/EternalKnights'),
|
||||
cls('SmackJeeves/EvD'),
|
||||
cls('SmackJeeves/FatetheAnthologyofKaienandhisfuckingmagicfriends'),
|
||||
cls('SmackJeeves/FeathersPI'),
|
||||
cls('SmackJeeves/FemmeSchism'),
|
||||
cls('SmackJeeves/FireWire'),
|
||||
|
@ -475,10 +482,14 @@ class Removed(Scraper):
|
|||
cls('SmackJeeves/FurtherDowntheRabbitHole'),
|
||||
cls('SmackJeeves/GATEKEEPER'),
|
||||
cls('SmackJeeves/GearTheTakedown'),
|
||||
cls('SmackJeeves/GoldenSunGenerationsAftermathVolume1'),
|
||||
cls('SmackJeeves/GoldenSunGenerationsColossoVolume6'),
|
||||
cls('SmackJeeves/GraveImpressions'),
|
||||
cls('SmackJeeves/GreenKirbyandabunchofotherpeopledoinstuff'),
|
||||
cls('SmackJeeves/GuardianGhost'),
|
||||
cls('SmackJeeves/Harfang'),
|
||||
cls('SmackJeeves/HIPS'),
|
||||
cls('SmackJeeves/HitandMiss'),
|
||||
cls('SmackJeeves/HotChocolate'),
|
||||
cls('SmackJeeves/Hybristorific'),
|
||||
cls('SmackJeeves/Ianua'),
|
||||
|
@ -492,6 +503,7 @@ class Removed(Scraper):
|
|||
cls('SmackJeeves/LastLivingSouls'),
|
||||
cls('SmackJeeves/LatchkeyKingdom'),
|
||||
cls('SmackJeeves/LegendsofMobiusBookOne'),
|
||||
cls('SmackJeeves/LetsBreakitforReals'),
|
||||
cls('SmackJeeves/LiliBleu'),
|
||||
cls('SmackJeeves/LoveTwister'),
|
||||
cls('SmackJeeves/MagicalGirlAlice'),
|
||||
|
@ -503,31 +515,46 @@ class Removed(Scraper):
|
|||
cls('SmackJeeves/MixupofallMixups'),
|
||||
cls('SmackJeeves/MomthegamestorerippedusoffAGAIN'),
|
||||
cls('SmackJeeves/MoonlitDawnAMythicalTale'),
|
||||
cls('SmackJeeves/MUTE', 'mov'),
|
||||
cls('SmackJeeves/MyBoyfriendisaMobBoss'),
|
||||
cls('SmackJeeves/MyTrollLife'),
|
||||
cls('SmackJeeves/MyTwoCentsPlusTax'),
|
||||
cls('SmackJeeves/NeoCrystalAdventures'),
|
||||
cls('SmackJeeves/NihilWandasJourney'),
|
||||
cls('SmackJeeves/OddContact'),
|
||||
cls('SmackJeeves/Ohman', 'brk'),
|
||||
cls('SmackJeeves/OneFrameGags'),
|
||||
cls('SmackJeeves/Pahantekija'),
|
||||
cls('SmackJeeves/Paradox'),
|
||||
cls('SmackJeeves/Paripety'),
|
||||
cls('SmackJeeves/Perinto'),
|
||||
cls('SmackJeeves/PerplexingMagnoliaDisruption', 'mov'),
|
||||
cls('SmackJeeves/PlatonicBoyfriends'),
|
||||
cls('SmackJeeves/Plotlessnesses'),
|
||||
cls('SmackJeeves/PokemonGleamingCrystal', 'mis'),
|
||||
cls('SmackJeeves/PokemonMysteryDungeonTeamCrystal'),
|
||||
cls('SmackJeeves/PRAGUERACE'),
|
||||
cls('SmackJeeves/PumpkinFlower'),
|
||||
cls('SmackJeeves/Razor'),
|
||||
cls('SmackJeeves/RedVelvetRequiem'),
|
||||
cls('SmackJeeves/RuderiQuest'),
|
||||
cls('SmackJeeves/SAKANA'),
|
||||
cls('SmackJeeves/SenoireDelirium'),
|
||||
cls('SmackJeeves/SerendipityAnEquestrianTale'),
|
||||
cls('SmackJeeves/ShacklesInstallment02'),
|
||||
cls('SmackJeeves/SimonSues'),
|
||||
cls('SmackJeeves/SimplySarah'),
|
||||
cls('SmackJeeves/SomebodyShootMe'),
|
||||
cls('SmackJeeves/SonicUniverseAsk'),
|
||||
cls('SmackJeeves/SoulGuardian'),
|
||||
cls('SmackJeeves/SpaghettiAndMeatballs'),
|
||||
cls('SmackJeeves/SparkStory', 'mis'),
|
||||
cls('SmackJeeves/Spidersilk', 'mov'),
|
||||
cls('SmackJeeves/Symbios'),
|
||||
cls('SmackJeeves/TechnicolorLondon'),
|
||||
cls('SmackJeeves/TeKscloset'),
|
||||
cls('SmackJeeves/TheAttackoftheRecoloursSeason1'),
|
||||
cls('SmackJeeves/TheAvianStories'),
|
||||
cls('SmackJeeves/TheCurtandTonyShow'),
|
||||
cls('SmackJeeves/TheDarkAgeofMobius'),
|
||||
cls('SmackJeeves/TheHobbitbic'),
|
||||
|
@ -535,6 +562,7 @@ class Removed(Scraper):
|
|||
cls('SmackJeeves/TheKeyToReality'),
|
||||
cls('SmackJeeves/TheLostland'),
|
||||
cls('SmackJeeves/TheMewExperiment'),
|
||||
cls('SmackJeeves/TheMoistTouch'),
|
||||
cls('SmackJeeves/TheRandomObscureFairyTaleNoOnesEverReallyHeardOf'),
|
||||
cls('SmackJeeves/TheSomewhereOther'),
|
||||
cls('SmackJeeves/TheWastelands', 'mis'),
|
||||
|
@ -687,6 +715,7 @@ class Renamed(Scraper):
|
|||
cls('SmackJeeves/CityFolk', 'ComicFury/CityFolk'),
|
||||
cls('SmackJeeves/DoomsdayMyDear', 'DoomsdayMyDear'),
|
||||
cls('SmackJeeves/ForestHill', 'ForestHill'),
|
||||
cls('SmackJeeves/Katran', 'ComicFury/KATRAN'),
|
||||
cls('SmackJeeves/Magience', 'ComicFury/Magience'),
|
||||
cls('SmackJeeves/RiversideExtras', 'RiversideExtras'),
|
||||
cls('SmackJeeves/StarTrip', 'StarTrip'),
|
||||
|
|
|
@ -16,20 +16,21 @@ class SmackJeeves(_ParserScraper):
|
|||
imageSearch = re.compile("comicData:[^']*'([^']*)'", re.DOTALL)
|
||||
help = 'Index format: n'
|
||||
|
||||
def __init__(self, name, id, adult=False, endOfLife=False):
|
||||
def __init__(self, name, id, adult=False, endOfLife=False, last=None):
|
||||
super(SmackJeeves, self).__init__('SmackJeeves/' + name)
|
||||
self._comicid = id
|
||||
self.url = self.baseUrl + 'articleList?titleNo={}'.format(id)
|
||||
self.stripUrl = self.baseUrl + 'detail?titleNo={}&articleNo=%s'.format(id)
|
||||
self.firstStripUrl = self.stripUrl % 1
|
||||
self.adult = adult
|
||||
self.endOfLife = endOfLife
|
||||
self.endOfLife = endOfLife or last
|
||||
self.lastid = (last or 0) - 1
|
||||
|
||||
def starter(self):
|
||||
response = self.session.post(self.apiBase + 'articleList',
|
||||
params={'titleNo': self._comicid})
|
||||
response.raise_for_status()
|
||||
return response.json()['result']['list'][-1]['articleUrl']
|
||||
return response.json()['result']['list'][self.lastid]['articleUrl']
|
||||
|
||||
def fetchUrls(self, url, data, urlsearch):
|
||||
if urlsearch != self.imageSearch:
|
||||
|
@ -94,11 +95,9 @@ class SmackJeeves(_ParserScraper):
|
|||
cls('AtArmsLength', 39242),
|
||||
cls('Atlaswebcomic', 140920),
|
||||
cls('Autophobia', 82522),
|
||||
cls('Aware', 132405),
|
||||
cls('AyaTakeo', 60325),
|
||||
cls('AYuriCollab', 126727, adult=True),
|
||||
cls('BabysittingFourDemons', 5992),
|
||||
cls('Babywhatsyoursign', 123587),
|
||||
cls('BadassRiz', 130263),
|
||||
cls('BallandChain', 99503),
|
||||
cls('Bard', 51147),
|
||||
|
@ -170,7 +169,6 @@ class SmackJeeves(_ParserScraper):
|
|||
cls('Dasien', 1779, adult=True),
|
||||
cls('DavidDoesntGetIt', 115176),
|
||||
cls('DBON', 76205),
|
||||
cls('DeadtoDay', 137063),
|
||||
cls('DeathNoteIridescent', 87149),
|
||||
cls('DEGAF', 120316),
|
||||
cls('DEMENTED', 104334, adult=True),
|
||||
|
@ -183,10 +181,8 @@ class SmackJeeves(_ParserScraper):
|
|||
cls('Diaz', 129935),
|
||||
cls('DigimonSaviors', 43521),
|
||||
cls('DigimonTamersMiraiProject', 119257),
|
||||
cls('DigisRandomSpriteshack', 47936),
|
||||
cls('DigitalInsanity', 144866),
|
||||
cls('DoItYourself', 103195),
|
||||
cls('DontDie', 140541),
|
||||
cls('DoodleBeans', 17473, adult=True),
|
||||
cls('DoodlingAround', 117225),
|
||||
cls('Dragonet', 119508),
|
||||
|
@ -201,7 +197,6 @@ class SmackJeeves(_ParserScraper):
|
|||
cls('EnkeltenKentta', 78834, adult=True),
|
||||
cls('Enthrall', 113047, adult=True),
|
||||
cls('EntreEuxDeux', 97894, endOfLife=True),
|
||||
cls('Entuthrie', 114564, adult=True),
|
||||
cls('Eorah', 135499, adult=True),
|
||||
cls('Equsopia', 127680),
|
||||
cls('ERAConvergence', 63750, endOfLife=True),
|
||||
|
@ -218,7 +213,6 @@ class SmackJeeves(_ParserScraper):
|
|||
cls('FallenAngelslove', 113208),
|
||||
cls('FarOutMantic', 48658),
|
||||
cls('FarOutThere', 89199),
|
||||
cls('FatetheAnthologyofKaienandhisfuckingmagicfriends', 140771),
|
||||
cls('FeralGentry', 119912),
|
||||
cls('FinalArcanum', 51191),
|
||||
cls('FireredLisasReise', 109151),
|
||||
|
@ -241,12 +235,9 @@ class SmackJeeves(_ParserScraper):
|
|||
cls('GigisNuzlockeRuns', 157589),
|
||||
cls('Gloomverse', 75498),
|
||||
cls('Gnoph', 138286, endOfLife=True),
|
||||
cls('GoldenSunGenerationsAftermathVolume1', 17885),
|
||||
cls('GoldenSunGenerationsColossoVolume6', 17945),
|
||||
cls('GoodGame', 48048, endOfLife=True),
|
||||
cls('GoodnightMrsGoose', 73655),
|
||||
cls('Grayscale', 100295, adult=True),
|
||||
cls('GuardianGhost', 118429),
|
||||
cls('GuardiansoftheGalaxialSpaceways', 70286),
|
||||
cls('Habibahssong', 137453),
|
||||
cls('HarvestMoonParadiseFound', 105422, endOfLife=True),
|
||||
|
@ -254,12 +245,10 @@ class SmackJeeves(_ParserScraper):
|
|||
cls('HatShop', 71816),
|
||||
cls('Helix', 121066),
|
||||
cls('HeltonShelton', 124804),
|
||||
cls('HendrixAndJulian', 155252, adult=True),
|
||||
cls('Hephaestus', 79909),
|
||||
cls('HereBeVoodoo', 133691, adult=True),
|
||||
cls('HiddenStrengthAWhiteNuzlocke', 116170),
|
||||
cls('Hinata', 134703),
|
||||
cls('HitandMiss', 82248),
|
||||
cls('Holocrash', 64113, endOfLife=True),
|
||||
cls('HolyBlasphemy', 91250),
|
||||
cls('HolyCrap', 13235),
|
||||
|
@ -287,7 +276,6 @@ class SmackJeeves(_ParserScraper):
|
|||
cls('Justmyluck', 169106),
|
||||
cls('KaitoShuno', 100498, adult=True),
|
||||
cls('KasaKeira', 78059),
|
||||
cls('Katran', 79778),
|
||||
cls('KazanatoFuneralPlanningService', 24026),
|
||||
cls('KCNO', 103010, endOfLife=True),
|
||||
cls('KezroChroniclesPhantomOps', 104447),
|
||||
|
@ -319,7 +307,6 @@ class SmackJeeves(_ParserScraper):
|
|||
cls('LesCendresdelHiver', 131767),
|
||||
cls('LethalDose', 115067, adult=True),
|
||||
cls('LetLoveRule', 177216),
|
||||
cls('LetsBreakitforReals', 127909),
|
||||
cls('LicensedHeroes', 123974),
|
||||
cls('LifeAsACutOut', 86222),
|
||||
cls('LifeAsItWas', 117747),
|
||||
|
@ -338,7 +325,7 @@ class SmackJeeves(_ParserScraper):
|
|||
cls('LoveroftheSunandMoon', 121615),
|
||||
cls('LsEmpire', 33751),
|
||||
cls('LuffinpuffandEric', 83450),
|
||||
cls('LumasParadise', 46770),
|
||||
cls('LumasParadise', 46770, last=174),
|
||||
cls('MagicalMisfits', 138166),
|
||||
cls('Magipunk', 131204),
|
||||
cls('Manifestedpart1', 140208),
|
||||
|
@ -368,7 +355,6 @@ class SmackJeeves(_ParserScraper):
|
|||
cls('MorphE', 122406),
|
||||
cls('Mortifer', 17271),
|
||||
cls('MrFactory', 119931),
|
||||
cls('MUTE', 32960, endOfLife=True),
|
||||
cls('MyFakeHeart', 19105),
|
||||
cls('MySisterTheDragon', 9612, endOfLife=True),
|
||||
cls('MySparklingPrincesama', 105626),
|
||||
|
@ -376,10 +362,8 @@ class SmackJeeves(_ParserScraper):
|
|||
cls('MysticanDreams', 130154),
|
||||
cls('MythsOfUnovaAWhiteNuzlockeRunHardMode', 101385),
|
||||
cls('MYth', 36430),
|
||||
cls('MyTwoCentsPlusTax', 38595),
|
||||
cls('Nah', 88350),
|
||||
cls('Negligence', 66718),
|
||||
cls('NeoCrystalAdventures', 2060),
|
||||
cls('NeonGlow', 63747),
|
||||
cls('NeverTheHero', 132200),
|
||||
cls('Nexus', 115357),
|
||||
|
@ -397,7 +381,6 @@ class SmackJeeves(_ParserScraper):
|
|||
cls('ObsidianHeart', 130610),
|
||||
cls('October20th', 122344),
|
||||
cls('OddPlaceOddTime', 60037),
|
||||
cls('Ohman', 143967),
|
||||
cls('OldElastikid', 97110),
|
||||
cls('OneRainyDay', 111352, adult=True),
|
||||
cls('Onlyonelovesong', 86555),
|
||||
|
@ -407,17 +390,14 @@ class SmackJeeves(_ParserScraper):
|
|||
cls('Outbreak', 134869),
|
||||
cls('OutofKey', 123152),
|
||||
cls('OverSync', 114781),
|
||||
cls('Pahantekija', 139286),
|
||||
cls('Panacea', 79601, adult=True),
|
||||
cls('PantsParty', 70630),
|
||||
cls('PanzerDragonandEnigmaCompleteEdition', 20039),
|
||||
cls('Paradox', 29667, adult=True),
|
||||
cls('Pause', 135615),
|
||||
cls('PencilviewUpdatesMondayscough', 33041),
|
||||
cls('PerplexingMagnoliaDisruption', 104698),
|
||||
cls('PeterPan', 119991),
|
||||
cls('Phantomland', 106943),
|
||||
cls('PhotoShootNarusasuDoujinshi', 110984, adult=True),
|
||||
cls('PhotoShootNarusasuDoujinshi', 110984, adult=True, last=187),
|
||||
cls('PlasticKings', 132377),
|
||||
cls('PlayTime', 68440),
|
||||
cls('PleaseBeMyBoytoy', 26922),
|
||||
|
@ -427,7 +407,6 @@ class SmackJeeves(_ParserScraper):
|
|||
cls('PokemonBeta', 76962, endOfLife=True),
|
||||
cls('PokemonCrystalDoubleNuzlockeChallenge', 102420, endOfLife=True),
|
||||
cls('PokemonLANDSKY', 120046),
|
||||
cls('PokemonMysteryDungeonTeamCrystal', 122892),
|
||||
cls('PokemonNoRakuen', 22822),
|
||||
cls('PokemonParallel', 86143),
|
||||
cls('PokemonSAKOHJU', 120395),
|
||||
|
@ -449,7 +428,6 @@ class SmackJeeves(_ParserScraper):
|
|||
cls('RANDOM', 99296),
|
||||
cls('RareCandyTreatment', 83853),
|
||||
cls('RavenWolf', 97826),
|
||||
cls('RedVelvetRequiem', 127363),
|
||||
cls('Regina', 129902),
|
||||
cls('ReidyAndFriendsShowcase', 45097),
|
||||
cls('RemoteAngel', 46191),
|
||||
|
@ -460,7 +438,6 @@ class SmackJeeves(_ParserScraper):
|
|||
cls('RottenApple', 133069),
|
||||
cls('RoyalIcing', 81321),
|
||||
cls('RubyNation', 107039),
|
||||
cls('RuderiQuest', 111648),
|
||||
cls('RuneSpark', 140886),
|
||||
cls('RUScrewed', 144524),
|
||||
cls('RyuManwebcomicVersion', 135497, endOfLife=True),
|
||||
|
@ -488,7 +465,6 @@ class SmackJeeves(_ParserScraper):
|
|||
cls('ShroudofLight', 121119),
|
||||
cls('Signifikat', 12217, adult=True),
|
||||
cls('SimpleBear', 120596),
|
||||
cls('SimplySarah', 33343),
|
||||
cls('Sire', 58167),
|
||||
cls('Skeptical', 108904),
|
||||
cls('Slackmatic', 144286),
|
||||
|
@ -496,7 +472,6 @@ class SmackJeeves(_ParserScraper):
|
|||
cls('SlipstreamSingularity', 66132),
|
||||
cls('SmallPressAdventures', 70273),
|
||||
cls('SocksMittensandScarfs', 132813),
|
||||
cls('SomebodyShootMe', 121562),
|
||||
cls('SomethingLikeaPhenomenon', 176182, adult=True),
|
||||
cls('SonicAuthorAdventII', 5867),
|
||||
cls('SonicBoom', 111375),
|
||||
|
@ -507,9 +482,7 @@ class SmackJeeves(_ParserScraper):
|
|||
cls('SOSRadio', 107498),
|
||||
cls('SouthernCross', 94541),
|
||||
cls('SovereignTheMostAmazingComicEver', 129062),
|
||||
cls('SpaghettiAndMeatballs', 130401, adult=True),
|
||||
cls('SparElricsExtras', 125431),
|
||||
cls('SparkStory', 131006),
|
||||
cls('Spellcross', 125055),
|
||||
cls('SpiderWings', 36314),
|
||||
cls('SplitScreen', 125293, adult=True, endOfLife=True),
|
||||
|
@ -545,7 +518,6 @@ class SmackJeeves(_ParserScraper):
|
|||
cls('TheAntihero', 64167),
|
||||
cls('TheArchipelago', 142266),
|
||||
cls('Theatrics', 93578),
|
||||
cls('TheAvianStories', 145492),
|
||||
cls('TheBattleInTheSky', 143006),
|
||||
cls('TheBookOfNosferatu', 97421),
|
||||
cls('TheBrideoftheShark', 61393, adult=True),
|
||||
|
@ -579,7 +551,6 @@ class SmackJeeves(_ParserScraper):
|
|||
cls('TheLoneSwordsman', 18218),
|
||||
cls('TheMadMan', 83928),
|
||||
cls('TheMegaManandSonicSpriteShowcase', 75689),
|
||||
cls('TheMoistTouch', 135862),
|
||||
cls('TheNightSurfers', 120705),
|
||||
cls('TheNocheComicSeries', 133439, adult=True),
|
||||
cls('TheNOMEDSEGA', 127642),
|
||||
|
@ -640,7 +611,7 @@ class SmackJeeves(_ParserScraper):
|
|||
cls('WildWingBoysKoathArc', 50240),
|
||||
cls('WildWingBoys', 203574),
|
||||
cls('WingsOverEthereal', 37417),
|
||||
cls('WingsTurnedtoDust', 113033),
|
||||
cls('WingsTurnedtoDust', 113033, last=417),
|
||||
cls('Wootlabs', 101636),
|
||||
cls('XXMoralityXx', 132016),
|
||||
cls('YadotCakeShop', 106995, adult=True),
|
||||
|
|
Loading…
Reference in a new issue