dosage/dosagelib/plugins/t.py

251 lines
8.6 KiB
Python
Raw Normal View History

# -*- coding: utf-8 -*-
2016-10-28 22:21:41 +00:00
# Copyright (C) 2004-2008 Tristan Seligmann and Jonathan Jacobs
2014-01-05 15:50:57 +00:00
# Copyright (C) 2012-2014 Bastian Kleineidam
2017-02-13 21:41:17 +00:00
# Copyright (C) 2015-2017 Tobias Gruetzmacher
2012-06-20 19:58:13 +00:00
from __future__ import absolute_import, division, print_function
2016-05-06 23:50:10 +00:00
2019-07-14 06:06:31 +00:00
from re import compile, escape, MULTILINE
try:
from functools import cached_property
except ImportError:
from cached_property import cached_property
2016-05-06 23:50:10 +00:00
2015-04-26 09:52:31 +00:00
from ..scraper import _BasicScraper, _ParserScraper
2017-02-13 21:41:17 +00:00
from ..helpers import indirectStarter, xpath_class
from ..util import tagre
2017-05-21 23:17:05 +00:00
from .common import _ComicControlScraper, _TumblrScraper, _WordPressScraper, _WPNavi
2012-06-20 19:58:13 +00:00
2016-10-31 05:57:47 +00:00
class TheBrads(_ParserScraper):
url = 'http://bradcolbow.com/archive/'
imageSearch = '//div[%s]//img' % xpath_class('entry')
prevSearch = '//a[%s]' % xpath_class('prev')
2013-02-06 21:27:40 +00:00
multipleImagesPerStrip = True
2017-05-21 23:17:05 +00:00
class TheDevilsPanties(_WPNavi):
url = 'http://thedevilspanties.com/'
stripUrl = url + 'archives/%s'
2013-04-10 21:57:09 +00:00
firstStripUrl = stripUrl % '300'
2012-12-08 20:30:51 +00:00
help = 'Index format: number'
class TheDreamlandChronicles(_WordPressScraper):
url = 'http://www.thedreamlandchronicles.com/'
2015-04-26 09:52:31 +00:00
class TheGamerCat(_ParserScraper):
url = "http://www.thegamercat.com/"
stripUrl = url + "comic/%s/"
firstStripUrl = stripUrl % "06102011"
css = True
imageSearch = '#comic img'
prevSearch = '.comic-nav-previous'
help = 'Index format: stripname'
2013-04-25 19:20:48 +00:00
2014-01-31 03:32:07 +00:00
class TheGentlemansArmchair(_WordPressScraper):
url = 'http://thegentlemansarmchair.com/'
2014-02-18 20:00:43 +00:00
class TheLandscaper(_BasicScraper):
2016-05-06 23:50:10 +00:00
stripUrl = 'http://landscaper.visual-assault.net/comic/%s'
url = stripUrl % 'latest'
2014-02-18 20:00:43 +00:00
firstStripUrl = stripUrl % '1'
imageSearch = compile(tagre("img", "src",
r'(/comics/comic/comic_page/[^"]+)'))
prevSearch = compile(tagre("a", "href", r'(/comic/[^"]+)') +
'‹ Previous')
2014-02-18 20:00:43 +00:00
help = 'Index format: name'
class TheMelvinChronicles(_WordPressScraper):
url = 'http://melvin.jeaniebottle.com/'
2016-05-06 23:50:10 +00:00
class TheNoob(_WordPressScraper):
url = 'http://thenoobcomic.com/'
stripUrl = url + 'comic/%s/'
2013-04-10 21:57:09 +00:00
firstStripUrl = stripUrl % '1'
2016-05-06 23:50:10 +00:00
help = 'Index format: n (unpadded)'
2012-06-20 19:58:13 +00:00
class TheOrderOfTheStick(_BasicScraper):
url = 'http://www.giantitp.com/'
stripUrl = url + 'comics/oots%s.html'
2013-04-10 21:57:09 +00:00
firstStripUrl = stripUrl % '0001'
2012-12-07 23:45:18 +00:00
imageSearch = compile(r'<IMG src="(/comics/images/[^"]+)">')
2012-06-20 19:58:13 +00:00
prevSearch = compile(r'<A href="(/comics/oots\d{4}\.html)"><IMG src="/Images/redesign/ComicNav_Back.gif"')
latestSearch = compile(r'<A href="(/comics/oots\d{4}\.html)"')
2012-06-20 19:58:13 +00:00
help = 'Index format: n (unpadded)'
starter = indirectStarter
2012-06-20 19:58:13 +00:00
def namer(self, image_url, page_url):
return page_url.rsplit('/', 1)[-1][:-5]
2012-06-20 19:58:13 +00:00
2019-06-30 17:46:39 +00:00
class TheRockCocks(_BasicScraper):
url = 'http://rockcocks.slipshine.net/'
rurl = escape(url)
stripUrl = url + 'comics/%s'
firstStripUrl = stripUrl % "page-1-nsfw-track-1-start"
imageSearch = compile(tagre("img", "src", r'(%scomics/[^"]+)' % rurl, after='id="cc-comic"'))
prevSearch = compile(tagre("a", "href", r'(%scomic/[^"]+)' % rurl, after='rel="prev"'))
adult = True
2016-05-06 23:50:10 +00:00
class TheThinHLine(_TumblrScraper):
url = 'http://thinhline.tumblr.com/'
2016-05-06 23:50:10 +00:00
firstStripUrl = url + 'post/4177372348/thl-1-a-cats-got-his-tongue-click-on-the'
imageSearch = '//img[@id="content-image"]/@data-src'
prevSearch = '//div[@id="pagination"]/a[text()=">"]'
latestSearch = '//a[@class="timestamp"]'
adult = True
2016-05-06 23:50:10 +00:00
indirectImageSearch = '//div[@id="post"]//a[not(@rel) and img]'
def getComicStrip(self, url, data):
"""The comic strip image is in a separate page."""
2016-05-06 23:50:10 +00:00
subPage = self.fetchUrl(url, data, self.indirectImageSearch)
pageData = self.getPage(subPage)
return super(TheThinHLine, self).getComicStrip(subPage, pageData)
2017-05-21 22:30:31 +00:00
class TheWhiteboard(_ParserScraper):
2019-06-13 06:42:12 +00:00
BROKEN_PAGE_MIDDLE = compile(r'</body></html>\n<')
url = 'http://www.the-whiteboard.com/'
2019-06-13 06:42:12 +00:00
stripUrl = url + 'auto%s.html'
firstStripUrl = stripUrl % 'wb001'
imageSearch = '//img[contains(@src, "auto")]'
prevSearch = '//a[.//img[contains(@src, "previous")]]'
2017-05-21 22:30:31 +00:00
def _parse_page(self, data):
2019-06-13 06:42:12 +00:00
# Ugly hack to fix broken HTML
2017-05-21 22:30:31 +00:00
data = self.BROKEN_PAGE_MIDDLE.sub('<', data)
return super(TheWhiteboard, self)._parse_page(data)
2016-05-06 23:50:10 +00:00
class TheWotch(_WordPressScraper):
url = 'http://www.thewotch.com/'
2016-05-06 23:50:10 +00:00
firstStripUrl = url + '?comic=enter-the-wotch'
2012-06-20 19:58:13 +00:00
2013-02-06 21:08:36 +00:00
class ThisIsIndexed(_BasicScraper):
url = 'http://thisisindexed.com/'
rurl = escape(url)
2013-02-06 21:08:36 +00:00
stripUrl = url + 'page/%s'
imageSearch = compile(tagre("img", "src", r'(%swp-content/uploads/\d+/\d+/card[^"]+)' % rurl))
2013-02-06 21:08:36 +00:00
multipleImagesPerStrip = True
prevSearch = compile(tagre("div", "class", "nav-previous") +
2013-04-11 16:27:43 +00:00
tagre("a", "href", r'(%spage/\d+/)[^"]*' % rurl))
2013-02-06 21:08:36 +00:00
help = 'Index format: number'
2016-05-06 23:50:10 +00:00
class ThreePanelSoul(_ComicControlScraper):
url = 'http://threepanelsoul.com/'
2016-05-06 23:50:10 +00:00
firstStripUrl = url + 'comic/a-test-comic'
2016-05-06 23:50:10 +00:00
class ToonHole(_WordPressScraper):
2016-05-16 21:16:29 +00:00
url = 'http://toonhole.com/'
2016-10-31 05:57:47 +00:00
firstStripUrl = url + 'comic/toon-hole-coming-soon-2010/'
2013-02-06 21:08:36 +00:00
def shouldSkipUrl(self, url, data):
2016-10-31 05:57:47 +00:00
return url in (self.url + "comic/if-game-of-thrones-was-animated/",)
2013-03-07 17:22:24 +00:00
2013-02-06 21:08:36 +00:00
2012-12-08 20:30:51 +00:00
class TracyAndTristan(_BasicScraper):
url = 'http://tandt.thecomicseries.com/'
rurl = escape(url)
stripUrl = url + 'comics/%s'
imageSearch = compile(tagre("img", "src", r'(%simages/comics/[^"]+)' % rurl))
2012-12-08 20:30:51 +00:00
prevSearch = compile(tagre("a", "href", r'(/comics/\d+)', after="prev"))
help = 'Index format: number'
2013-04-25 19:23:31 +00:00
class TrippingOverYou(_BasicScraper):
url = 'http://www.trippingoveryou.com/'
stripUrl = url + 'comic/%s'
firstStripUrl = stripUrl % 'wiggle-room'
imageSearch = compile(tagre("img", "src", r'([^"]+/comics/[^"]+)'))
prevSearch = compile(r'<a class="cc-prev" rel="prev" href="(.+?)">')
help = 'Index format: stripname'
class TumbleDryComics(_WordPressScraper):
url = 'http://tumbledrycomics.com/'
stripUrl = url + 'comic/%s/'
firstStripUrl = stripUrl % 'we-need-to-get-high-jpg'
textSearch = '//div[@id="comic"]//img/@alt'
multipleImagesPerStrip = True
adult = True
help = 'Index format: name'
def namer(self, image_url, page_url):
# Most images have the date they were posted in the filename
# For those that don't we can get the month and year from the image url
parts = image_url.rsplit('/', 3)
year = parts[1]
month = parts[2]
filename = parts[3]
if not filename.startswith(year):
filename = year + "-" + month + "-" + filename
return filename
2019-07-14 06:06:31 +00:00
class Turnoff(_ParserScraper):
name = 'turnoff'
url = 'https://turnoff.us/'
imageSearch = '//article[%s]//img' % xpath_class('post-content')
prevSearch = '//div[%s]//a' % xpath_class('prev')
stripUrl = url + 'geek/%s'
firstStripUrl = stripUrl % 'tcp-buddies'
multipleImagesPerStrip = True
@cached_property
def comics_order(self):
# Neither the images nor the pages contain information about dates or indices.
# However we can extract the order of the images from the JavaScript.
html = self.session.get(self.url).text
list_regex = compile(r"""^\s*"/geek/(.*)",\s*$""", flags=MULTILINE)
return list(reversed(list_regex.findall(html)))
def namer(self, image_url, page_url):
comic_name = page_url.split('/')[-1]
try:
index = self.comics_order.index(comic_name) + 1
except ValueError:
index = len(self.comics_order)
file_name = image_url.split('/')[-1]
return "%03d-%s" % (index, file_name)
2013-04-25 19:23:31 +00:00
class TwoGuysAndGuy(_BasicScraper):
url = 'http://www.twogag.com/'
rurl = escape(url)
stripUrl = url + 'archives/%s'
firstStripUrl = stripUrl % '4'
imageSearch = compile(tagre('img', 'src', r'(%scomics/\d{4}-\d{2}-\d{2}[^"]*)' % rurl))
prevSearch = compile(tagre('a', 'href', r'(%sarchives/\d+)' % rurl,
after='title="Previous"'))
2013-04-25 19:23:31 +00:00
help = 'Index format: number'
adult = True
2016-10-13 22:14:53 +00:00
class Twokinds(_ParserScraper):
url = 'http://twokinds.keenspot.com/'
2017-04-15 23:06:41 +00:00
stripUrl = url + 'comic/%s/'
firstStripUrl = stripUrl % '1'
imageSearch = '//article[%s]//img' % xpath_class('comic')
prevSearch = '//a[%s]' % xpath_class('navprev')
help = 'Index format: n (unpadded)'
2016-10-13 22:14:53 +00:00
class TwoLumps(_BasicScraper):
url = 'http://www.twolumps.net/'
stripUrl = url + 'd/%s.html'
imageSearch = compile(tagre("img", "src", r'(/comics/[^"]+)'))
prevSearch = compile(tagre("a", "href", r'(/d/\d+\.html)', after="prev"))
help = 'Index format: yyyymmdd'