From 6634a39d6100b838c1f18bd5267adfd8ff090a08 Mon Sep 17 00:00:00 2001 From: Bastian Kleineidam Date: Thu, 24 Jan 2013 21:42:04 +0100 Subject: [PATCH] Generate individual comic pages. --- Makefile | 2 +- scripts/mktestpage.json | 1 + scripts/mktestpage.py | 260 +++++++++++++++++++++++++++++++--------- 3 files changed, 207 insertions(+), 56 deletions(-) create mode 100644 scripts/mktestpage.json diff --git a/Makefile b/Makefile index e94166286..70ed98c57 100644 --- a/Makefile +++ b/Makefile @@ -49,7 +49,7 @@ upload: dist/$(ARCHIVE_SOURCE).asc dist/$(ARCHIVE_WIN32).asc testresults: - scripts/mktestpage.py testresults.txt > $(HOMEPAGE)/content/testresults.html + scripts/mktestpage.py testresults.txt $(HOMEPAGE)/content homepage: # update metadata diff --git a/scripts/mktestpage.json b/scripts/mktestpage.json new file mode 100644 index 000000000..833602439 --- /dev/null +++ b/scripts/mktestpage.json @@ -0,0 +1 @@ +{"Arcamax_BabyBlues": {"adult": false, "description": "", "error": "AssertionError: Arcamax/BabyBlues http://www.arcamax.com/thefunnies/babyblues/ strip URL 'http://www.arcamax.com/thef\\", "name": "Arcamax/BabyBlues", "since": "1.9", "status": "error", "url": "http://www.arcamax.com/thefunnies/babyblues/"}, "BiggerThanCheeses": {"adult": false, "description": "", "error": null, "name": "BiggerThanCheeses", "since": "1.8", "status": "ok", "url": "http://www.biggercheese.com/"}, "Chester5000XYV": {"adult": false, "description": "", "error": null, "name": "Chester5000XYV", "since": "1.8", "status": "ok", "url": "http://jessfink.com/Chester5000XYV/"}, "DrunkDuck_Civil_Servitude": {"adult": false, "description": "", "error": null, "name": "DrunkDuck/Civil_Servitude", "since": "1.9", "status": "orphan", "url": "http://www.drunkduck.com/Civil_Servitude/4994879/"}, "DrunkDuck_Dasien": {"adult": false, "description": "", "error": "AssertionError: DrunkDuck/Dasien http://www.drunkduck.com/Dasien/5343187/ could not save http://media.drunkduck.com.s3.amazonaws.com:80/users/Neilsama/comics/Dasien/web/Chapter%2B7.jpg to /tmp/tmpNpJZd4: [Errno Unable to retrieve URL.] http://media.drunkduck.com.s3.amazonaws.com:80/users/Neilsama/comics/Dasien/web/Chapter%2B7.jpg: IOError('URL retrieval of http://media.drunkduck.com.s3.amazonaws.com:80/users/Neilsama/comics/Dasien/web/Chapter%2B7.jpg failed: 403 Client Error: Forbidden',)", "name": "DrunkDuck/Dasien", "since": "1.9", "status": "orphan", "url": "http://www.drunkduck.com/Dasien/5343187/"}, "DrunkDuck_Explorers_Of_the_Unknown": {"adult": false, "description": "", "error": "AssertionError: DrunkDuck/Explorers_Of_the_Unknown http://www.drunkduck.com/Explorers_Of_the_Unknown/5395556/ could n\\", "name": "DrunkDuck/Explorers_Of_the_Unknown", "since": "1.8", "status": "error", "url": "http://www.drunkduck.com/Explorers_Of_the_Unknown/5395556/"}, "DrunkDuck_Metal_Breakdown": {"adult": false, "description": "", "error": null, "name": "DrunkDuck/Metal_Breakdown", "since": "1.8", "status": "ok", "url": "http://www.drunkduck.com/Metal_Breakdown/5386007/"}, "NobodyScores": {"adult": false, "description": "", "error": null, "name": "NobodyScores", "since": "1.8", "status": "ok", "url": "http://nobodyscores.loosenutstudio.com/"}, "SmackJeeves_ERRORERROR": {"adult": false, "description": "A finnish teenage boy Tomi is a normal nerd. He plays videogames and gets average grades, but also is terribly bored with his life. But one day he accidentally gets inside of his computer and meets there a girl, who calls herself princess Jooda. And there is a profecy about Tomi, sword and Viruses. Warnings: If you are under 13 years old, I do not recommend this comic to you. Contains bad language, blood and somewhat sexual themes. ps. sorry for style changes, i practise haha Updates 2-4 pages a week.", "error": null, "name": "SmackJeeves/ERRORERROR", "since": "1.8", "status": "ok", "url": "http://errorerror.smackjeeves.com/comics/1664476/ee297/"}, "SmackJeeves_SoulGuardian": {"adult": false, "description": "One day humans will create viruses with the intention of infecting themselves. With a virus strong enough, a person would even be able to travel across space and time...they would have the power to change their destiny, but at what cost? READ: Right to left UPDATES: Wednesday", "error": null, "name": "SmackJeeves/SoulGuardian", "since": "1.8", "status": "ok", "url": "http://soulguardian.smackjeeves.com/comics/1664888/nav-11-96/"}, "WebcomicsNation_AgnesQuill": {"adult": false, "description": "", "error": null, "name": "WebcomicsNation/AgnesQuill", "since": "1.9", "status": "orphan", "url": "http://www.webcomicsnation.com/daveroman/agnes/"}} \ No newline at end of file diff --git a/scripts/mktestpage.py b/scripts/mktestpage.py index 949fa3f81..b2533e645 100755 --- a/scripts/mktestpage.py +++ b/scripts/mktestpage.py @@ -7,37 +7,98 @@ import time import cgi sys.path.append(os.path.join(os.path.dirname(__file__), "..")) from dosagelib.scraper import get_scrapers +from dosagelib.configuration import Version as DosageVersion +from scriptutil import load_result, save_result -htmltemplate = """ +json_file = __file__.replace(".py", ".json") + +class Status: + ok = "ok" + error = "error" + orphan = "orphan" + +indextemplate = """ --- extends: base.j2 -title: Dosage by Bastian Kleineidam -description: a commandline webcomic downloader and archiver +title: Dosage comic list +description: a list of comic strips supported by Dosage --- {%% block js %%} - + {%% endblock js %%} {%% block content %%} -
-

Dosage test results from %(date)s

-

Note that it is almost impossible to get a 100%% OK test run -due to temporary site failures.

-
+

Dosage comic list

+
%(content)s
+ +
{%% endblock content %%} """ +comic_template = """ +--- +extends: base.j2 +title: Dosage comic %(name)s +--- +{%% block content %%} +
+ +

Dosage comic %(name)s

+ + + + + + + + + + + + + + + + +
Description%(description)s
Website%(url)s
Adult content%(adult)s
Available sinceDosage v%(since)s
Status%(status)s on %(date)s
+
+ +
+{%% endblock content %%} +""" + +entrytemplate_url = """ +%(name)s +
+""" + +entrytemplate_nourl = """ +%(name)s +""" + def get_mtime (filename): """Return modification time of filename.""" @@ -49,75 +110,164 @@ def strdate(t): return time.strftime("%d.%m.%Y", time.localtime(t)) -def get_test_name(line): - """Get scraper name from test output line.""" +def get_testscraper(line): + """Get scraper from test output line.""" classname = line.split('::')[1][4:] for scraper in get_scrapers(): if scraper.__name__ == classname: - try: - url = scraper.starter() - except Exception: - url = None - return scraper.get_name(), url + return scraper raise ValueError("Scraper %r not found" % classname) -def get_test(line): - """Get test name from test output line.""" - name, url = get_test_name(line) - result = "OK" if line.startswith(". ") else "FAILED" - return [name, url, result, ""] - - -def get_content(filename): - """Get HTML content for test output.""" - tests = [] +def get_testinfo(filename, modified): + """Maintains a static list of comics which users can vote on. + The original set of comic strips is stored in a JSON file which gets + updated from the test results. + If a comic strip stored in JSON is not found in the test results, it is + orphaned. + @return: {name -> { + "status": Status.*, + "url": string or None, + "description": string or None, + "error": string or None, + } + } + """ + if os.path.isfile(json_file): + testinfo = load_result(json_file) + else: + testinfo = {} with open(filename, "r") as f: print("Tests parsed: 0", end=" ", file=sys.stderr) num_tests = 0 - add_reason = False + add_error = False + keys = [] for line in f: if line.startswith((". ", "F ")) and "test_comics" in line: - add_reason = line.startswith("F ") + add_error = line.startswith("F ") num_tests += 1 - try: - tests.append(get_test(line)) - except Exception as msg: - print("WARNING:", msg, file=sys.stderr) - continue - elif add_reason and line.startswith(" E "): - reason = line[3:].strip() - tests[-1][-1] = reason + key, entry = get_testentry(line) + keys.append(key) + update_testentry(key, entry, testinfo) + elif add_error and line.startswith(" E "): + entry["error"] = line[3:].strip() if num_tests % 5 == 0: print(num_tests, end=" ", file=sys.stderr) - tests.sort() - res = [] - for name, url, result, reason in tests: - css = result.lower() - if len(name) > 40: - name = name[:37] + "..." - if url: - args = quote_all(url, reason, css, name) - inner = '%s' % args + orphan_entries(keys, testinfo) + save_result(testinfo, json_file) + return testinfo + + +def get_testentry(line): + """Get one test entry.""" + scraper = get_testscraper(line) + key = scraper.__name__ + name = scraper.get_name() + if len(name) > 40: + name = name[:37] + "..." + entry = { + "status": Status.ok if line.startswith(". ") else Status.error, + "name": name, + "url": None, + "description": scraper.description, + "error": None, + "adult": scraper.adult, + } + try: + entry["url"] = scraper.starter() + except Exception as msg: + print("WARNING:", msg, file=sys.stderr) + return key, entry + + +def orphan_entries(keys, testinfo): + """Mark all entries that are in testinfo but not in keys as orphaned.""" + for key, entry in testinfo.items(): + if key not in keys: + entry["status"] = Status.orphan + + +def update_testentry(key, entry, testinfo): + if key not in testinfo: + # add dosage version for this comic + # XXX replace this after next release + if key.startswith("Arcamax") or key in ("AmazingSuperPowers", "PandyLand"): + entry["since"] = DosageVersion else: - args = quote_all(reason, css, name) - inner = '%s' % args - res.append('
%s
' % inner) + entry["since"] = "1.8" + else: + entry["since"] = testinfo[key]["since"] + testinfo[key] = entry + + +def get_html_index(testinfo): + """Get HTML content for test output index.""" + res = [] + for key in sorted(testinfo.keys()): + entry = testinfo[key] + css = entry["status"] + url = "comics/%s.html" % key + if entry["error"]: + title = entry["error"] + elif entry["description"]: + title = entry["description"] + else: + title = entry["name"] + args = { + "url": quote(url), + "title": quote(title), + "css": quote(css), + "name": quote(entry["name"]), + } + template = entrytemplate_url if url else entrytemplate_nourl + entryhtml = template % args + res.append('
%s
' % entryhtml) return os.linesep.join(res) -def quote_all(*args): - """CGI-escape all arguments for.""" - return tuple(cgi.escape(x, quote=True) for x in args) +def write_html(testinfo, outputdir, modified): + """Write index page and all comic pages.""" + content = get_html_index(testinfo) + date = strdate(modified) + args = {"date": quote(date), "content": content} + fname = os.path.join(outputdir, "comic_index.html") + with open(fname, 'w') as fp: + fp.write(indextemplate % args) + comicdir = os.path.join(outputdir, "comics") + if not os.path.isdir(comicdir): + os.mkdir(comicdir) + for key, entry in testinfo.items(): + write_html_comic(key, entry, comicdir, date) + + +def write_html_comic(key, entry, outputdir, date): + """Write a comic page.""" + args = { + "url": quote(entry["url"]), + "name": quote(entry["name"]), + "adult": quote("yes" if entry["adult"] else "no"), + "since": quote(entry["since"]), + "description": quote(entry["description"]), + "status": quote(entry["status"]), + "date": quote(date), + } + fname = os.path.join(outputdir, key+".html") + with open(fname, 'w') as fp: + fp.write(comic_template % args) + + +def quote(arg): + """CGI-escape argument.""" + return cgi.escape(arg, quote=True) def main(args): """Generate HTML output for test result.""" filename = args[0] + outputdir = args[1] modified = get_mtime(filename) - content = get_content(filename) - attrs = {"date": strdate(modified), "content": content} - print(htmltemplate % attrs) + testinfo = get_testinfo(filename, modified) + write_html(testinfo, outputdir, modified) return 0