From e92a3fb3a1e3fc500cf26446309a0f4ed8d44505 Mon Sep 17 00:00:00 2001 From: Tobias Gruetzmacher Date: Mon, 13 Oct 2014 21:39:13 +0200 Subject: [PATCH] New feature: Comic modules ca be "disabled". This is modeled parallel to the "adult" feature, except the user can't override it via the command line. Each comic module can override the classmethod getDisabledReasons and give the user a reason why this module is disabled. The user can see the reason in the comic list (-l or --singlelist) and the comic module refuses to run, showing the same message. This is currently used to disable modules that use the _ParserScraper if the LXML python module is missing. --- dosage | 31 ++++++++++++++++++++++--------- dosagelib/director.py | 24 +++++++++++++++++------- dosagelib/scraper.py | 40 ++++++++++++++++++++++++---------------- 3 files changed, 63 insertions(+), 32 deletions(-) diff --git a/dosage b/dosage index 5fc7f447d..cd1f0cb4f 100755 --- a/dosage +++ b/dosage @@ -136,7 +136,7 @@ def displayHelp(options): """Print help for comic strips.""" errors = 0 try: - for scraperobj in director.getScrapers(options.comic, options.basepath): + for scraperobj in director.getScrapers(options.comic, options.basepath, listing=True): errors += displayComicHelp(scraperobj) except ValueError as msg: out.exception(msg) @@ -239,12 +239,17 @@ def doList(columnList=True, verbose=False): out.info(u'Available comic scrapers:') out.info(u'Comics tagged with [%s] require age confirmation with the --adult option.' % TAG_ADULT) out.info(u'Non-english comics are tagged with [%s].' % TAG_LANG) - scrapers = sorted(director.getAllScrapers(), key=lambda s: s.getName()) + scrapers = sorted(director.getAllScrapers(listing=True), key=lambda s: s.getName()) if columnList: - num = doColumnList(scrapers) + num, disabled = doColumnList(scrapers) else: - num = doSingleList(scrapers, verbose=verbose) + num, disabled = doSingleList(scrapers, verbose=verbose) out.info(u'%d supported comics.' % num) + if disabled: + out.info('') + out.info(u'Some comics are disabled, they are tagged with [%s:REASON], where REASON is one of:' % TAG_DISABLED) + for k in disabled: + out.info(u' %-10s %s' % (k, disabled[k])) if page: pydoc.pager(fd.getvalue()) return 0 @@ -254,38 +259,46 @@ def doList(columnList=True, verbose=False): def doSingleList(scrapers, verbose=False): """Get list of scraper names, one per line.""" + disabled = {} for num, scraperobj in enumerate(scrapers): if verbose: displayComicHelp(scraperobj) else: - out.info(getScraperName(scraperobj)) - return num + out.info(getScraperName(scraperobj, reasons=disabled)) + return num, disabled def doColumnList(scrapers): """Get list of scraper names with multiple names per line.""" + disabled = {} screenWidth = get_columns(sys.stdout) # limit name length so at least two columns are there limit = (screenWidth // 2) - 8 - names = [getScraperName(scraperobj, limit=limit) for scraperobj in scrapers] + names = [getScraperName(scraperobj, limit=limit, reasons=disabled) for scraperobj in scrapers] num = len(names) maxlen = max(len(name) for name in names) namesPerLine = max(screenWidth // (maxlen + 1), 1) while names: out.info(u''.join(name.ljust(maxlen) for name in names[:namesPerLine])) del names[:namesPerLine] - return num + return num, disabled TAG_ADULT = "adult" TAG_LANG = "lang" +TAG_DISABLED = "dis" -def getScraperName(scraperobj, limit=None): +def getScraperName(scraperobj, limit=None, reasons=None): """Get comic scraper name.""" tags = [] if scraperobj.adult: tags.append(TAG_ADULT) if scraperobj.lang != "en": tags.append("%s:%s" % (TAG_LANG, scraperobj.lang)) + disabled = scraperobj.getDisabledReasons() + if disabled: + reasons.update(disabled) + for reason in disabled: + tags.append("%s:%s" % (TAG_DISABLED, reason)) if tags: suffix = " [" + ", ".join(tags) + "]" else: diff --git a/dosagelib/director.py b/dosagelib/director.py index 43ba32f8b..ecd2b3acb 100644 --- a/dosagelib/director.py +++ b/dosagelib/director.py @@ -189,12 +189,12 @@ def finish(): out.warn("Waiting for download threads to finish.") -def getAllScrapers(): +def getAllScrapers(listing=False): """Get all scrapers.""" - return getScrapers(['@@']) + return getScrapers(['@@'], listing=listing) -def getScrapers(comics, basepath=None, adult=True, multiple_allowed=False): +def getScrapers(comics, basepath=None, adult=True, multiple_allowed=False, listing=False): """Get scraper objects for the given comics.""" if '@' in comics: # only scrapers whose directory already exists @@ -203,12 +203,12 @@ def getScrapers(comics, basepath=None, adult=True, multiple_allowed=False): for scraperclass in scraper.get_scraperclasses(): dirname = getDirname(scraperclass.getName()) if os.path.isdir(os.path.join(basepath, dirname)): - if shouldRunScraper(scraperclass, adult): + if shouldRunScraper(scraperclass, adult, listing): yield scraperclass() elif '@@' in comics: # all scrapers for scraperclass in scraper.get_scraperclasses(): - if shouldRunScraper(scraperclass, adult): + if shouldRunScraper(scraperclass, adult, listing): yield scraperclass() else: # get only selected comic scrapers @@ -229,20 +229,30 @@ def getScrapers(comics, basepath=None, adult=True, multiple_allowed=False): indexes = None scraperclasses = scraper.find_scraperclasses(name, multiple_allowed=multiple_allowed) for scraperclass in scraperclasses: - if shouldRunScraper(scraperclass, adult): + if shouldRunScraper(scraperclass, adult, listing): scraperobj = scraperclass(indexes=indexes) if scraperobj not in scrapers: scrapers.add(scraperobj) yield scraperobj -def shouldRunScraper(scraperclass, adult=True): +def shouldRunScraper(scraperclass, adult=True, listing=False): + if listing: + return True if not adult and scraperclass.adult: warn_adult(scraperclass) return False + reasons = scraperclass.getDisabledReasons() + if reasons: + warn_disabled(scraperclass, reasons) + return False return True def warn_adult(scraperclass): """Print warning about adult content.""" out.warn(u"skipping adult comic %s; use the --adult option to confirm your age" % scraperclass.getName()) + +def warn_disabled(scraperclass, reasons): + """Print warning about disabled comic modules.""" + out.warn(u"Skipping comic %s: %s" % (scraperclass.getName(), ' '.join(reasons.values()))) diff --git a/dosagelib/scraper.py b/dosagelib/scraper.py index 2d56b531e..0a4ea33ad 100644 --- a/dosagelib/scraper.py +++ b/dosagelib/scraper.py @@ -10,6 +10,13 @@ try: from urllib.parse import urljoin except ImportError: from urlparse import urljoin + +try: + from lxml import html + from lxml.html.defs import link_attrs as html_link_attrs +except ImportError: + html = None + from . import loader, configuration, util from .util import (getPageContent, makeSequence, get_system_uid, urlopen, getDirname, unescape, tagre, normaliseURL) @@ -308,6 +315,16 @@ class Scraper(object): def fetchText(cls, url, data, textSearch, optional): raise ValueError("No implementation for fetchText!") + @classmethod + def getDisabledReasons(cls): + """ + Get a dict of reasons why this comic module is disabled. The key is a + short (unique) identifier, the value is a string explaining why the + module is deactivated. If the module is not disabled, just return an + empty dict. + """ + return {} + class _BasicScraper(Scraper): """ @@ -389,23 +406,8 @@ class _ParserScraper(Scraper): of the HTML element and returns that. """ - @classmethod - def xpath(cls, expr): - return expr - - @classmethod - def css(cls, expr, attr=None): - return expr - @classmethod def getPage(cls, url): - try: - from lxml import html - except ImportError: - raise ValueError(u"Skipping comic %s: Needs lxml (python-lxml) installed." % cls.getName()) - from lxml.html.defs import link_attrs - cls.link_attrs = link_attrs - cls.html = html tree = html.document_fromstring(getPageContent(url, cls.session)) tree.make_links_absolute(url) return tree @@ -418,7 +420,7 @@ class _ParserScraper(Scraper): for search in searches: for match in data.xpath(search): try: - for attrib in cls.link_attrs: + for attrib in html_link_attrs: if attrib in match.attrib: searchUrls.append(match.get(attrib)) except AttributeError: @@ -450,6 +452,12 @@ class _ParserScraper(Scraper): else: return None + @classmethod + def getDisabledReasons(cls): + res = {} + if html is None: + res['lxml'] = u"This module needs the lxml (python-lxml) python module which is not installed." + return res def find_scraperclasses(comic, multiple_allowed=False): """Get a list comic scraper classes. Can return more than one entries if