From c3f32dfef7ea67b317f8bb56a422c604f7dd69ee Mon Sep 17 00:00:00 2001 From: Tobias Gruetzmacher Date: Thu, 21 Apr 2016 08:20:49 +0200 Subject: [PATCH] Refactor: Make namer a method. When #42 is realized, the naming of files might differ between comic modules, so the namer's logical location is the instance, not the class. --- dosagelib/helpers.py | 16 +++++------- dosagelib/plugins/a.py | 6 ++--- dosagelib/plugins/b.py | 15 +++++------ dosagelib/plugins/c.py | 17 +++++------- dosagelib/plugins/clonemanga.py | 5 ++-- dosagelib/plugins/comicfury.py | 9 +++---- dosagelib/plugins/d.py | 19 ++++++-------- dosagelib/plugins/drunkduck.py | 7 +++-- dosagelib/plugins/e.py | 7 +++-- dosagelib/plugins/g.py | 5 ++-- dosagelib/plugins/gocomics.py | 5 ++-- dosagelib/plugins/h.py | 7 +++-- dosagelib/plugins/l.py | 7 +++-- dosagelib/plugins/m.py | 5 ++-- dosagelib/plugins/n.py | 5 ++-- dosagelib/plugins/o.py | 7 +++-- dosagelib/plugins/p.py | 7 +++-- dosagelib/plugins/s.py | 44 ++++++++++++++------------------ dosagelib/plugins/smackjeeves.py | 3 +-- dosagelib/plugins/snafu.py | 3 +-- dosagelib/plugins/t.py | 12 ++++----- dosagelib/plugins/v.py | 5 ++-- dosagelib/plugins/w.py | 7 +++-- dosagelib/plugins/wlpcomics.py | 3 +-- dosagelib/plugins/x.py | 3 +-- dosagelib/plugins/z.py | 10 +++----- dosagelib/scraper.py | 3 +-- 27 files changed, 100 insertions(+), 142 deletions(-) diff --git a/dosagelib/helpers.py b/dosagelib/helpers.py index 67e13f58a..b630393d2 100644 --- a/dosagelib/helpers.py +++ b/dosagelib/helpers.py @@ -8,22 +8,20 @@ from __future__ import absolute_import, division, print_function from .util import getQueryParams -def queryNamer(paramName, usePageUrl=False): +def queryNamer(param, use_page_url=False): """Get name from URL query part.""" - @classmethod - def _namer(cls, imageUrl, pageUrl): + def _namer(self, image_url, page_url): """Get URL query part.""" - url = pageUrl if usePageUrl else imageUrl - return getQueryParams(url)[paramName][0] + url = page_url if use_page_url else image_url + return getQueryParams(url)[param][0] return _namer -def regexNamer(regex, usePageUrl=False): +def regexNamer(regex, use_page_url=False): """Get name from regular expression.""" - @classmethod - def _namer(cls, imageUrl, pageUrl): + def _namer(self, image_url, page_url): """Get first regular expression group.""" - url = pageUrl if usePageUrl else imageUrl + url = page_url if use_page_url else image_url mo = regex.search(url) if mo: return mo.group(1) diff --git a/dosagelib/plugins/a.py b/dosagelib/plugins/a.py index 21021b414..0bd6e36b8 100644 --- a/dosagelib/plugins/a.py +++ b/dosagelib/plugins/a.py @@ -28,8 +28,7 @@ class AbstruseGoose(_BasicScraper): help = 'Index format: n (unpadded)' textSearch = compile(tagre("img", "title", r'([^"]+)')) - @classmethod - def namer(cls, image_url, page_url): + def namer(self, image_url, page_url): index = int(page_url.rstrip('/').split('/')[-1]) name = image_url.split('/')[-1].split('.')[0] return 'c%03d-%s' % (index, name) @@ -300,8 +299,7 @@ class AstronomyPOTD(_ParserScraper): """Skip pages without images.""" return data.xpath('//iframe') # videos - @classmethod - def namer(cls, image_url, page_url): + def namer(self, image_url, page_url): return '%s-%s' % (page_url.split('/')[-1].split('.')[0][2:], image_url.split('/')[-1].split('.')[0]) diff --git a/dosagelib/plugins/b.py b/dosagelib/plugins/b.py index 2927cf03b..ac638140f 100644 --- a/dosagelib/plugins/b.py +++ b/dosagelib/plugins/b.py @@ -76,11 +76,10 @@ class Beetlebum(_BasicScraper): help = 'Index format: yyyy/mm/dd/striptitle' lang = 'de' - @classmethod - def namer(cls, imageUrl, pageUrl): - indexes = tuple(pageUrl.rstrip('/').split('/')[-4:]) + def namer(self, image_url, page_url): + indexes = tuple(page_url.rstrip('/').split('/')[-4:]) name = '%s-%s-%s-%s' % indexes - name = name + '_' + imageUrl.split('/')[-1] + name = name + '_' + image_url.split('/')[-1] return name @@ -154,8 +153,7 @@ class BloomingFaeries(_ParserScraper): imageSearch = '//div[@id="comic"]//img' prevSearch = WP_PREV_SEARCH - @classmethod - def namer(cls, image_url, page_url): + def namer(self, image_url, page_url): return "_".join(image_url.rsplit('/', 3)[1:]) @@ -197,9 +195,8 @@ class BoyOnAStickAndSlither(_BasicScraper): "Next page") help = 'Index format: n (unpadded)' - @classmethod - def namer(cls, imageUrl, pageUrl): - return pageUrl.rsplit('/')[-1] + def namer(self, image_url, page_url): + return page_url.rsplit('/')[-1] class BratHalla(_WordPressScraper): diff --git a/dosagelib/plugins/c.py b/dosagelib/plugins/c.py index 064f24c48..09639ce39 100755 --- a/dosagelib/plugins/c.py +++ b/dosagelib/plugins/c.py @@ -57,9 +57,8 @@ class Carciphona(_BasicScraper): r'(view\.php\?page=[0-9]+[^"]*)')) starter = indirectStarter - @classmethod - def namer(cls, imageUrl, pageUrl): - ip = imageUrl.split('/') + def namer(self, image_url, page_url): + ip = image_url.split('/') return "volume_%s_page_%s" % (ip[-2], ip[-1]) @@ -262,9 +261,8 @@ class CorydonCafe(_ParserScraper): latestSearch = '//ul//a' help = 'Index format: yyyy/stripname' - @classmethod - def namer(cls, imageUrl, pageUrl): - return pageUrl.split('/')[-1].split('.')[0] + def namer(self, image_url, page_url): + return page_url.split('/')[-1].split('.')[0] class CourtingDisaster(_WordPressScraper): @@ -349,10 +347,9 @@ class CyanideAndHappiness(_BasicScraper): """Skip pages without images.""" return "/comics/play-button.png" in data[0] - @classmethod - def namer(cls, imageUrl, pageUrl): - imgname = imageUrl.split('/')[-1] + def namer(self, image_url, page_url): + imgname = image_url.split('/')[-1] # only get the first 100 chars for the image name imgname = imgname[:100] - imgnum = pageUrl.split('/')[-2] + imgnum = page_url.split('/')[-2] return '%s_%s' % (imgnum, imgname) diff --git a/dosagelib/plugins/clonemanga.py b/dosagelib/plugins/clonemanga.py index a1585332d..ffaa75f90 100644 --- a/dosagelib/plugins/clonemanga.py +++ b/dosagelib/plugins/clonemanga.py @@ -24,9 +24,8 @@ def add(name, shortName, imageFolder=None, lastStrip=None): if imageFolder is None: imageFolder = shortName - @classmethod - def namer(cls, imageUrl, pageUrl): - return '%03d' % int(getQueryParams(pageUrl)['page'][0]) + def namer(self, image_url, page_url): + return '%03d' % int(getQueryParams(page_url)['page'][0]) def _starter(self): # first, try hopping to previous and next comic diff --git a/dosagelib/plugins/comicfury.py b/dosagelib/plugins/comicfury.py index 15026d974..3b03babc4 100644 --- a/dosagelib/plugins/comicfury.py +++ b/dosagelib/plugins/comicfury.py @@ -22,12 +22,11 @@ class _ComicFury(_ParserScraper): help = 'Index format: n' starter = bounceStarter - @classmethod - def namer(cls, imageUrl, pageUrl): - parts = pageUrl.split('/') - path, ext = os.path.splitext(imageUrl) + def namer(self, image_url, page_url): + parts = page_url.split('/') + path, ext = os.path.splitext(image_url) num = parts[-1] - return "%s_%s%s" % (cls.__name__[2:], num, ext) + return "%s_%s%s" % (self.__class__.__name__[2:], num, ext) @property def url(self): diff --git a/dosagelib/plugins/d.py b/dosagelib/plugins/d.py index 569d4cb0a..b948cd523 100644 --- a/dosagelib/plugins/d.py +++ b/dosagelib/plugins/d.py @@ -34,10 +34,9 @@ class DamnLol(_BasicScraper): help = 'Index format: stripname-number' starter = bounceStarter - @classmethod - def namer(cls, imageUrl, pageUrl): - ext = imageUrl.rsplit('.', 1)[1] - path = pageUrl.rsplit('/', 1)[1][:-5] + def namer(self, image_url, page_url): + ext = image_url.rsplit('.', 1)[1] + path = page_url.rsplit('/', 1)[1][:-5] stripname, number = path.rsplit('-', 1) return '%s-%s.%s' % (number, stripname, ext) @@ -136,10 +135,9 @@ class DieselSweeties(_BasicScraper): tagre("img", "src", r'(?:http://www\.dieselsweeties\.com/ximages/blackbackarrow160.png|/ximages/prev\.gif)')) help = 'Index format: n (unpadded)' - @classmethod - def namer(cls, imageUrl, pageUrl): - index = int(imageUrl.split('/')[-1].split('.')[0]) - return 'sw%02d' % (index,) + def namer(self, image_url, page_url): + index = int(image_url.split('/')[-1].split('.')[0]) + return 'sw%02d' % index class Dilbert(_BasicScraper): @@ -154,9 +152,8 @@ class Dilbert(_BasicScraper): after="Click to see")) help = 'Index format: yyyy-mm-dd' - @classmethod - def namer(cls, imageUrl, pageUrl): - name = pageUrl.rsplit("/", 1)[1] + def namer(self, image_url, page_url): + name = page_url.rsplit("/", 1)[1] return "%s" % name diff --git a/dosagelib/plugins/drunkduck.py b/dosagelib/plugins/drunkduck.py index 6a9df6823..e789e5f86 100644 --- a/dosagelib/plugins/drunkduck.py +++ b/dosagelib/plugins/drunkduck.py @@ -24,10 +24,9 @@ def add(name, path): classname = '_DrunkDuck_%s' % name _url = 'http://www.theduckwebcomics.com/%s/' % path - @classmethod - def _namer(cls, imageUrl, pageUrl): - index = int(pageUrl.rstrip('/').split('/')[-1]) - ext = imageUrl.rsplit('.')[-1] + def _namer(self, image_url, page_url): + index = int(page_url.rstrip('/').split('/')[-1]) + ext = image_url.rsplit('.')[-1] return '%d.%s' % (index, ext) def _starter(self): diff --git a/dosagelib/plugins/e.py b/dosagelib/plugins/e.py index 993f8495b..a91517c14 100644 --- a/dosagelib/plugins/e.py +++ b/dosagelib/plugins/e.py @@ -26,13 +26,12 @@ class EarthsongSaga(_ParserScraper): return [x.replace('earthsongsaga.com/../', 'earthsongsaga.com/') for x in urls] - @classmethod - def namer(cls, imageUrl, pageUrl): + def namer(self, image_url, page_url): imgmatch = compile(r'images/vol(\d+)/ch(\d+)/(\d+)\.\w+$', - IGNORECASE).search(imageUrl) + IGNORECASE).search(image_url) if not imgmatch: imgmatch = compile(r'images/vol(\d+)/ch(\d+)/ch(\d+)cover\.\w+$', - IGNORECASE).search(imageUrl) + IGNORECASE).search(image_url) suffix = "cover" else: suffix = "" diff --git a/dosagelib/plugins/g.py b/dosagelib/plugins/g.py index f7d26c892..c2125e719 100644 --- a/dosagelib/plugins/g.py +++ b/dosagelib/plugins/g.py @@ -146,9 +146,8 @@ class GoneWithTheBlastwave(_BasicScraper): r'.*]*prev_button\.gif') nextSearch = compile(r'.*]*next_button\.gif') help = 'Index format: n (unpadded)' - namer = queryNamer('comicid', usePageUrl=True) + namer = queryNamer('comicid', use_page_url=True) class Pimpette(_ParserScraper): diff --git a/dosagelib/plugins/s.py b/dosagelib/plugins/s.py index db084e3a8..55d5d825a 100644 --- a/dosagelib/plugins/s.py +++ b/dosagelib/plugins/s.py @@ -197,10 +197,9 @@ class SexyLosers(_BasicScraper): help = 'Index format: nnn' starter = indirectStarter - @classmethod - def namer(cls, imageUrl, pageUrl): - index = pageUrl.split('/')[-1].split('.')[0] - title = imageUrl.split('/')[-1].split('.')[0] + def namer(self, image_url, page_url): + index = page_url.split('/')[-1].split('.')[0] + title = image_url.split('/')[-1].split('.')[0] return index + '-' + title @@ -228,9 +227,8 @@ class ShermansLagoon(_BasicScraper): prevSearch = compile(r'id="previouscomic" class="button white">