#!/usr/bin/env python # -*- coding: iso-8859-1 -*- # Dosage, the webcomic downloader # Copyright (C) 2004-2005 Tristan Seligmann and Jonathan Jacobs # Copyright (C) 2012-2013 Bastian Kleineidam from __future__ import print_function import sys import os import optparse from collections import OrderedDict from dosagelib import events, scraper from dosagelib.output import out from dosagelib.util import internal_error, getDirname, strlimit from dosagelib.ansicolor import get_columns from dosagelib.configuration import App, Freeware, Copyright, SupportUrl def setupOptions(): """Construct option parser. @return: new option parser @rtype optparse.OptionParser """ usage = 'usage: %prog [options] comicModule [comicModule ...]' parser = optparse.OptionParser(usage=usage) parser.add_option('-v', '--verbose', action='count', dest='verbose', default=0, help='provides verbose output, use multiple times for more verbosity') parser.add_option('-n', '--numstrips', action='store', dest='numstrips', type='int', default=0, help='traverse and retrieve the given number of comic strips; use --all to retrieve all comic strips') parser.add_option('-a', '--all', action='store_true', dest='all', default=None, help='traverse and retrieve all comic strips') parser.add_option('-c', '--continue', action='store_true', dest='cont', default=None, help='traverse and retrieve comic strips until an existing one is found') parser.add_option('-b', '--basepath', action='store', dest='basepath', default='Comics', help='set the path to create invidivual comic directories in, default is Comics', metavar='PATH') parser.add_option('--baseurl', action='store', dest='baseurl', default=None, help='the base URL of your comics directory (for RSS, HTML, etc.); this should correspond to --base-path', metavar='PATH') parser.add_option('-l', '--list', action='store_const', const=1, dest='list', help='list available comic modules') parser.add_option('--singlelist', action='store_const', const=2, dest='list', help='list available comic modules in a single list') parser.add_option('-V', '--version', action='store_true', dest='version', help='display the version number') parser.add_option('-m', '--modulehelp', action='store_true', dest='modhelp', help='display help for comic modules') parser.add_option('-t', '--timestamps', action='store_true', dest='timestamps', default=False, help='print timestamps for all output at any info level') parser.add_option('-o', '--output', action='store', dest='output', choices=events.getHandlers(), help='output formatting for downloaded comics') parser.add_option('--adult', action='store_true', dest='adult', default=False, help='confirms that you are old enough to view adult content') try: import optcomplete optcomplete.autocomplete(parser) except ImportError: pass return parser def displayVersion(): """Display application name, version, copyright and license.""" print(App) print(Copyright) print(Freeware) print("For support see", SupportUrl) return 0 def setOutputInfo(options): """Set global output level and timestamp option.""" out.level = 0 out.level += options.verbose out.timestamps = options.timestamps def saveComicStrip(strip, basepath): """Save a comic strip which can consist of multiple images.""" errors = 0 allskipped = True for image in strip.getImages(): try: filename, saved = image.save(basepath) if saved: allskipped = False except IOError as msg: out.error('Could not save %s: %s' % (image.filename, msg)) errors += 1 return errors, allskipped def displayHelp(comics): """Print help for comic strips.""" try: for scraperobj in getScrapers(comics): displayComicHelp(scraperobj) except ValueError as msg: out.error(msg) return 1 return 0 def displayComicHelp(scraperobj): """Print description and help for a comic.""" out.context = scraperobj.get_name() try: if scraperobj.description: for line in scraperobj.description.splitlines(): out.info(line) if scraperobj.help: for line in scraperobj.help.splitlines(): out.info(line) finally: out.context = '' def getComics(options, comics): """Retrieve given comics.""" errors = 0 if options.output: events.installHandler(options.output, options.basepath, options.baseurl) events.getHandler().start() try: for scraperobj in getScrapers(comics, options.basepath, options.adult): errors += getStrips(scraperobj, options) except ValueError as msg: out.error(msg) errors += 1 finally: out.context = '' events.getHandler().end() return errors def getStrips(scraperobj, options): """Get all strips from a scraper.""" errors = 0 out.context = scraperobj.get_name() if options.all: strips = scraperobj.getAllStrips() elif options.numstrips: strips = scraperobj.getAllStrips(options.numstrips) else: strips = scraperobj.getCurrentStrips() try: for strip in strips: _errors, skipped = saveComicStrip(strip, options.basepath) errors += _errors if skipped and options.cont: # stop when retrieval skipped an image for one comic strip out.info("Stop retrieval because image file already exists") break except (ValueError, IOError) as msg: out.error(msg) errors += 1 return errors def run(options, comics): """Execute comic commands.""" setOutputInfo(options) if options.version: return displayVersion() if options.list: return doList(options.list == 1) if len(comics) <= 0: out.warn('No comics specified, bailing out!') return 1 if options.modhelp: return displayHelp(comics) return getComics(options, comics) def doList(columnList): """List available comics.""" out.info('Available comic scrapers:') out.info('Comics marked with [A] require age confirmation with the --adult option.') scrapers = getScrapers(['@@']) try: if columnList: num = doColumnList(scrapers) else: num = doSingleList(scrapers) out.info('%d supported comics.' % num) except IOError: pass return 0 def doSingleList(scrapers): """Get list of scraper names, one per line.""" for num, scraperobj in enumerate(scrapers): print(getScraperName(scraperobj)) return num def doColumnList(scrapers): """Get list of scraper names with multiple names per line.""" screenWidth = get_columns(sys.stdout) # limit name length so at least two columns are there limit = (screenWidth / 2) - 8 names = [getScraperName(scraperobj, limit=limit) for scraperobj in scrapers] num = len(names) maxlen = max(len(name) for name in names) namesPerLine = max(int(screenWidth / (maxlen + 1)), 1) while names: print(''.join(name.ljust(maxlen) for name in names[:namesPerLine])) del names[:namesPerLine] return num def getScraperName(scraperobj, limit=None): """Get comic scraper name.""" suffix = " [A]" if scraperobj.adult else "" name = scraperobj.get_name() if limit is not None: name = strlimit(name, limit) return name + suffix def getScrapers(comics, basepath=None, adult=True): """Get scraper objects for the given comics.""" if '@' in comics: # only scrapers whose directory already exists if len(comics) > 1: out.warn("using '@' as comic name ignores all other specified comics.") for scraperclass in scraper.get_scraperclasses(): if not adult and scraperclass.adult: warn_adult(scraperclass) continue dirname = getDirname(scraperclass.get_name()) if os.path.isdir(os.path.join(basepath, dirname)): yield scraperclass() elif '@@' in comics: # all scrapers for scraperclass in scraper.get_scraperclasses(): if not adult and scraperclass.adult: warn_adult(scraperclass) continue yield scraperclass() else: # get only selected comic scrapers # store them in an ordered set to eliminate duplicates scrapers = OrderedDict() for comic in comics: if ':' in comic: name, index = comic.split(':', 1) indexes = index.split(',') else: name = comic indexes = None # XXX if matchmulti, search for a list of scrapers scraperclass = scraper.get_scraper(name) if not adult and scraperclass.adult: warn_adult(scraperclass) continue scraperobj = scraperclass(indexes=indexes) if scraperobj not in scrapers: scrapers[scraperobj] = True for scraperobj in scrapers: yield scraperobj def warn_adult(scraperclass): """Print warning about adult content.""" out.warn("skipping adult comic %s; use the --adult option to confirm your age" % scraperclass.get_name()) def main(): """Parse options and execute commands.""" if sys.argv[0].endswith("mainline"): out.warn("the 'mainline' program is deprecated, please use the new 'dosage' program") try: parser = setupOptions() options, args = parser.parse_args() # eliminate duplicate comic names comics = set(args) res = run(options, comics) except KeyboardInterrupt: print("Aborted.") res = 1 except Exception: internal_error() res = 2 return res def profile(): """Profile the loading of all scrapers.""" import cProfile cProfile.run("scraper.get_scraperclasses()", "dosage.prof") def viewprof(): """View profile stats.""" import pstats stats = pstats.Stats("dosage.prof") stats.strip_dirs().sort_stats("cumulative").print_stats(100) if __name__ == '__main__': sys.exit(main()) #profile() #viewprof()