289 lines
10 KiB
Python
Executable file
289 lines
10 KiB
Python
Executable file
#!/usr/bin/env python
|
|
# -*- coding: iso-8859-1 -*-
|
|
# Dosage, the webcomic downloader
|
|
# Copyright (C) 2004-2005 Tristan Seligmann and Jonathan Jacobs
|
|
# Copyright (C) 2012-2013 Bastian Kleineidam
|
|
from __future__ import print_function
|
|
import sys
|
|
import os
|
|
import optparse
|
|
from collections import OrderedDict
|
|
|
|
from dosagelib import events, scraper
|
|
from dosagelib.output import out
|
|
from dosagelib.util import internal_error, getDirname, strlimit
|
|
from dosagelib.ansicolor import get_columns
|
|
from dosagelib.configuration import App, Freeware, Copyright, SupportUrl
|
|
|
|
def setupOptions():
|
|
"""Construct option parser.
|
|
@return: new option parser
|
|
@rtype optparse.OptionParser
|
|
"""
|
|
usage = 'usage: %prog [options] comicModule [comicModule ...]'
|
|
parser = optparse.OptionParser(usage=usage)
|
|
parser.add_option('-v', '--verbose', action='count', dest='verbose', default=0, help='provides verbose output, use multiple times for more verbosity')
|
|
parser.add_option('-n', '--numstrips', action='store', dest='numstrips', type='int', default=0, help='traverse and retrieve the given number of comic strips; use --all to retrieve all comic strips')
|
|
parser.add_option('-a', '--all', action='store_true', dest='all', default=None, help='traverse and retrieve all comic strips')
|
|
parser.add_option('-c', '--continue', action='store_true', dest='cont', default=None, help='traverse and retrieve comic strips until an existing one is found')
|
|
parser.add_option('-b', '--basepath', action='store', dest='basepath', default='Comics', help='set the path to create invidivual comic directories in, default is Comics', metavar='PATH')
|
|
parser.add_option('--baseurl', action='store', dest='baseurl', default=None, help='the base URL of your comics directory (for RSS, HTML, etc.); this should correspond to --base-path', metavar='PATH')
|
|
parser.add_option('-l', '--list', action='store_const', const=1, dest='list', help='list available comic modules')
|
|
parser.add_option('--singlelist', action='store_const', const=2, dest='list', help='list available comic modules in a single list')
|
|
parser.add_option('-V', '--version', action='store_true', dest='version', help='display the version number')
|
|
parser.add_option('-m', '--modulehelp', action='store_true', dest='modhelp', help='display help for comic modules')
|
|
parser.add_option('-t', '--timestamps', action='store_true', dest='timestamps', default=False, help='print timestamps for all output at any info level')
|
|
parser.add_option('-o', '--output', action='store', dest='output', choices=events.getHandlers(), help='output formatting for downloaded comics')
|
|
parser.add_option('--adult', action='store_true', dest='adult', default=False, help='confirms that you are old enough to view adult content')
|
|
parser.add_option('--multimatch', action='store_true', dest='multimatch', default=False, help='')
|
|
try:
|
|
import optcomplete
|
|
optcomplete.autocomplete(parser)
|
|
except ImportError:
|
|
pass
|
|
return parser
|
|
|
|
|
|
def displayVersion():
|
|
"""Display application name, version, copyright and license."""
|
|
print(App)
|
|
print(Copyright)
|
|
print(Freeware)
|
|
print("For support see", SupportUrl)
|
|
return 0
|
|
|
|
|
|
def setOutputInfo(options):
|
|
"""Set global output level and timestamp option."""
|
|
out.level = 0
|
|
out.level += options.verbose
|
|
out.timestamps = options.timestamps
|
|
|
|
|
|
def saveComicStrip(strip, basepath):
|
|
"""Save a comic strip which can consist of multiple images."""
|
|
errors = 0
|
|
allskipped = True
|
|
for image in strip.getImages():
|
|
try:
|
|
filename, saved = image.save(basepath)
|
|
if saved:
|
|
allskipped = False
|
|
except IOError as msg:
|
|
out.error('Could not save image at %s to %s: %s' % (image.referrer, image.filename, msg))
|
|
errors += 1
|
|
return errors, allskipped
|
|
|
|
|
|
def displayHelp(comics):
|
|
"""Print help for comic strips."""
|
|
try:
|
|
for scraperobj in getScrapers(comics):
|
|
displayComicHelp(scraperobj)
|
|
except ValueError as msg:
|
|
out.error(msg)
|
|
return 1
|
|
return 0
|
|
|
|
|
|
def displayComicHelp(scraperobj):
|
|
"""Print description and help for a comic."""
|
|
out.context = scraperobj.get_name()
|
|
try:
|
|
if scraperobj.description:
|
|
for line in scraperobj.description.splitlines():
|
|
out.info(line)
|
|
if scraperobj.help:
|
|
for line in scraperobj.help.splitlines():
|
|
out.info(line)
|
|
finally:
|
|
out.context = ''
|
|
|
|
|
|
def getComics(options, comics):
|
|
"""Retrieve given comics."""
|
|
errors = 0
|
|
if options.output:
|
|
events.installHandler(options.output, options.basepath, options.baseurl)
|
|
events.getHandler().start()
|
|
try:
|
|
for scraperobj in getScrapers(comics, options.basepath, options.adult, options.multimatch):
|
|
errors += getStrips(scraperobj, options)
|
|
except ValueError as msg:
|
|
out.error(msg)
|
|
errors += 1
|
|
finally:
|
|
out.context = ''
|
|
events.getHandler().end()
|
|
return errors
|
|
|
|
|
|
def getStrips(scraperobj, options):
|
|
"""Get all strips from a scraper."""
|
|
errors = 0
|
|
out.context = scraperobj.get_name()
|
|
if options.all:
|
|
strips = scraperobj.getAllStrips()
|
|
elif options.numstrips:
|
|
strips = scraperobj.getAllStrips(options.numstrips)
|
|
else:
|
|
strips = scraperobj.getCurrentStrips()
|
|
try:
|
|
for strip in strips:
|
|
_errors, skipped = saveComicStrip(strip, options.basepath)
|
|
errors += _errors
|
|
if skipped and options.cont:
|
|
# stop when retrieval skipped an image for one comic strip
|
|
out.info("Stop retrieval because image file already exists")
|
|
break
|
|
except (ValueError, IOError) as msg:
|
|
out.error(msg)
|
|
errors += 1
|
|
return errors
|
|
|
|
|
|
def run(options, comics):
|
|
"""Execute comic commands."""
|
|
setOutputInfo(options)
|
|
if options.version:
|
|
return displayVersion()
|
|
if options.list:
|
|
return doList(options.list == 1)
|
|
if len(comics) <= 0:
|
|
out.warn('No comics specified, bailing out!')
|
|
return 1
|
|
if options.modhelp:
|
|
return displayHelp(comics)
|
|
return getComics(options, comics)
|
|
|
|
|
|
def doList(columnList):
|
|
"""List available comics."""
|
|
out.info('Available comic scrapers:')
|
|
out.info('Comics marked with [A] require age confirmation with the --adult option.')
|
|
scrapers = sorted(getScrapers(['@@']), key=lambda s: s.get_name())
|
|
try:
|
|
if columnList:
|
|
num = doColumnList(scrapers)
|
|
else:
|
|
num = doSingleList(scrapers)
|
|
out.info('%d supported comics.' % num)
|
|
except IOError:
|
|
pass
|
|
return 0
|
|
|
|
|
|
def doSingleList(scrapers):
|
|
"""Get list of scraper names, one per line."""
|
|
for num, scraperobj in enumerate(scrapers):
|
|
print(getScraperName(scraperobj))
|
|
return num
|
|
|
|
|
|
def doColumnList(scrapers):
|
|
"""Get list of scraper names with multiple names per line."""
|
|
screenWidth = get_columns(sys.stdout)
|
|
# limit name length so at least two columns are there
|
|
limit = (screenWidth / 2) - 8
|
|
names = [getScraperName(scraperobj, limit=limit) for scraperobj in scrapers]
|
|
num = len(names)
|
|
maxlen = max(len(name) for name in names)
|
|
namesPerLine = max(int(screenWidth / (maxlen + 1)), 1)
|
|
while names:
|
|
print(''.join(name.ljust(maxlen) for name in names[:namesPerLine]))
|
|
del names[:namesPerLine]
|
|
return num
|
|
|
|
|
|
def getScraperName(scraperobj, limit=None):
|
|
"""Get comic scraper name."""
|
|
suffix = " [A]" if scraperobj.adult else ""
|
|
name = scraperobj.get_name()
|
|
if limit is not None:
|
|
name = strlimit(name, limit)
|
|
return name + suffix
|
|
|
|
|
|
def getScrapers(comics, basepath=None, adult=True, multiple_allowed=False):
|
|
"""Get scraper objects for the given comics."""
|
|
if '@' in comics:
|
|
# only scrapers whose directory already exists
|
|
if len(comics) > 1:
|
|
out.warn("using '@' as comic name ignores all other specified comics.")
|
|
for scraperclass in scraper.get_scraperclasses():
|
|
if not adult and scraperclass.adult:
|
|
warn_adult(scraperclass)
|
|
continue
|
|
dirname = getDirname(scraperclass.get_name())
|
|
if os.path.isdir(os.path.join(basepath, dirname)):
|
|
yield scraperclass()
|
|
elif '@@' in comics:
|
|
# all scrapers
|
|
for scraperclass in scraper.get_scraperclasses():
|
|
if not adult and scraperclass.adult:
|
|
warn_adult(scraperclass)
|
|
continue
|
|
yield scraperclass()
|
|
else:
|
|
# get only selected comic scrapers
|
|
# store them in an ordered set to eliminate duplicates
|
|
scrapers = OrderedDict()
|
|
for comic in comics:
|
|
if ':' in comic:
|
|
name, index = comic.split(':', 1)
|
|
indexes = index.split(',')
|
|
else:
|
|
name = comic
|
|
indexes = None
|
|
for scraperclass in scraper.find_scraperclasses(name, multiple_allowed=multiple_allowed):
|
|
if not adult and scraperclass.adult:
|
|
warn_adult(scraperclass)
|
|
continue
|
|
scraperobj = scraperclass(indexes=indexes)
|
|
if scraperobj not in scrapers:
|
|
scrapers[scraperobj] = True
|
|
for scraperobj in scrapers:
|
|
yield scraperobj
|
|
|
|
|
|
def warn_adult(scraperclass):
|
|
"""Print warning about adult content."""
|
|
out.warn("skipping adult comic %s; use the --adult option to confirm your age" % scraperclass.get_name())
|
|
|
|
|
|
def main():
|
|
"""Parse options and execute commands."""
|
|
if sys.argv[0].endswith("mainline"):
|
|
out.warn("the 'mainline' program is deprecated, please use the new 'dosage' program")
|
|
try:
|
|
parser = setupOptions()
|
|
options, args = parser.parse_args()
|
|
# eliminate duplicate comic names
|
|
comics = set(args)
|
|
res = run(options, comics)
|
|
except KeyboardInterrupt:
|
|
print("Aborted.")
|
|
res = 1
|
|
except Exception:
|
|
internal_error()
|
|
res = 2
|
|
return res
|
|
|
|
|
|
def profile():
|
|
"""Profile the loading of all scrapers."""
|
|
import cProfile
|
|
cProfile.run("scraper.get_scraperclasses()", "dosage.prof")
|
|
|
|
|
|
def viewprof():
|
|
"""View profile stats."""
|
|
import pstats
|
|
stats = pstats.Stats("dosage.prof")
|
|
stats.strip_dirs().sort_stats("cumulative").print_stats(100)
|
|
|
|
|
|
if __name__ == '__main__':
|
|
sys.exit(main())
|
|
#profile()
|
|
#viewprof()
|