Generate individual comic pages.
This commit is contained in:
Bastian Kleineidam
parent
4b35d332dc
commit
6634a39d61
2
Makefile
2
Makefile
|
|
@ -49,7 +49,7 @@ upload:
|
|||
dist/$(ARCHIVE_SOURCE).asc dist/$(ARCHIVE_WIN32).asc
|
||||
|
||||
testresults:
|
||||
scripts/mktestpage.py testresults.txt > $(HOMEPAGE)/content/testresults.html
|
||||
scripts/mktestpage.py testresults.txt $(HOMEPAGE)/content
|
||||
|
||||
homepage:
|
||||
# update metadata
|
||||
|
|
|
|||
1
scripts/mktestpage.json
Normal file
1
scripts/mktestpage.json
Normal file
|
|
@ -0,0 +1 @@
|
|||
{"Arcamax_BabyBlues": {"adult": false, "description": "", "error": "AssertionError: Arcamax/BabyBlues http://www.arcamax.com/thefunnies/babyblues/ strip URL 'http://www.arcamax.com/thef\\", "name": "Arcamax/BabyBlues", "since": "1.9", "status": "error", "url": "http://www.arcamax.com/thefunnies/babyblues/"}, "BiggerThanCheeses": {"adult": false, "description": "", "error": null, "name": "BiggerThanCheeses", "since": "1.8", "status": "ok", "url": "http://www.biggercheese.com/"}, "Chester5000XYV": {"adult": false, "description": "", "error": null, "name": "Chester5000XYV", "since": "1.8", "status": "ok", "url": "http://jessfink.com/Chester5000XYV/"}, "DrunkDuck_Civil_Servitude": {"adult": false, "description": "", "error": null, "name": "DrunkDuck/Civil_Servitude", "since": "1.9", "status": "orphan", "url": "http://www.drunkduck.com/Civil_Servitude/4994879/"}, "DrunkDuck_Dasien": {"adult": false, "description": "", "error": "AssertionError: DrunkDuck/Dasien http://www.drunkduck.com/Dasien/5343187/ could not save http://media.drunkduck.com.s3.amazonaws.com:80/users/Neilsama/comics/Dasien/web/Chapter%2B7.jpg to /tmp/tmpNpJZd4: [Errno Unable to retrieve URL.] http://media.drunkduck.com.s3.amazonaws.com:80/users/Neilsama/comics/Dasien/web/Chapter%2B7.jpg: IOError('URL retrieval of http://media.drunkduck.com.s3.amazonaws.com:80/users/Neilsama/comics/Dasien/web/Chapter%2B7.jpg failed: 403 Client Error: Forbidden',)", "name": "DrunkDuck/Dasien", "since": "1.9", "status": "orphan", "url": "http://www.drunkduck.com/Dasien/5343187/"}, "DrunkDuck_Explorers_Of_the_Unknown": {"adult": false, "description": "", "error": "AssertionError: DrunkDuck/Explorers_Of_the_Unknown http://www.drunkduck.com/Explorers_Of_the_Unknown/5395556/ could n\\", "name": "DrunkDuck/Explorers_Of_the_Unknown", "since": "1.8", "status": "error", "url": "http://www.drunkduck.com/Explorers_Of_the_Unknown/5395556/"}, "DrunkDuck_Metal_Breakdown": {"adult": false, "description": "", "error": null, "name": "DrunkDuck/Metal_Breakdown", "since": "1.8", "status": "ok", "url": "http://www.drunkduck.com/Metal_Breakdown/5386007/"}, "NobodyScores": {"adult": false, "description": "", "error": null, "name": "NobodyScores", "since": "1.8", "status": "ok", "url": "http://nobodyscores.loosenutstudio.com/"}, "SmackJeeves_ERRORERROR": {"adult": false, "description": "A finnish teenage boy Tomi is a normal nerd. He plays videogames and gets average grades, but also is terribly bored with his life. But one day he accidentally gets inside of his computer and meets there a girl, who calls herself princess Jooda. And there is a profecy about Tomi, sword and Viruses. Warnings: If you are under 13 years old, I do not recommend this comic to you. Contains bad language, blood and somewhat sexual themes. ps. sorry for style changes, i practise haha Updates 2-4 pages a week.", "error": null, "name": "SmackJeeves/ERRORERROR", "since": "1.8", "status": "ok", "url": "http://errorerror.smackjeeves.com/comics/1664476/ee297/"}, "SmackJeeves_SoulGuardian": {"adult": false, "description": "One day humans will create viruses with the intention of infecting themselves. With a virus strong enough, a person would even be able to travel across space and time...they would have the power to change their destiny, but at what cost? READ: Right to left UPDATES: Wednesday", "error": null, "name": "SmackJeeves/SoulGuardian", "since": "1.8", "status": "ok", "url": "http://soulguardian.smackjeeves.com/comics/1664888/nav-11-96/"}, "WebcomicsNation_AgnesQuill": {"adult": false, "description": "", "error": null, "name": "WebcomicsNation/AgnesQuill", "since": "1.9", "status": "orphan", "url": "http://www.webcomicsnation.com/daveroman/agnes/"}}
|
||||
|
|
@ -7,37 +7,98 @@ import time
|
|||
import cgi
|
||||
sys.path.append(os.path.join(os.path.dirname(__file__), ".."))
|
||||
from dosagelib.scraper import get_scrapers
|
||||
from dosagelib.configuration import Version as DosageVersion
|
||||
from scriptutil import load_result, save_result
|
||||
|
||||
htmltemplate = """
|
||||
json_file = __file__.replace(".py", ".json")
|
||||
|
||||
class Status:
|
||||
ok = "ok"
|
||||
error = "error"
|
||||
orphan = "orphan"
|
||||
|
||||
indextemplate = """
|
||||
---
|
||||
extends: base.j2
|
||||
title: Dosage by Bastian Kleineidam
|
||||
description: a commandline webcomic downloader and archiver
|
||||
title: Dosage comic list
|
||||
description: a list of comic strips supported by Dosage
|
||||
---
|
||||
{%% block js %%}
|
||||
<script src="media/js/masonry.min.js"></script>
|
||||
<script src="{{ media_url('js/masonry.min.js') }}"></script>
|
||||
{%% endblock js %%}
|
||||
|
||||
{%% block content %%}
|
||||
<div class="inner clearfix">
|
||||
<section id="main-content">
|
||||
|
||||
<h2>Dosage test results from %(date)s</h2>
|
||||
<p>Note that it is almost impossible to get a 100%% OK test run
|
||||
due to temporary site failures.</p>
|
||||
<div id="testresults">
|
||||
<h2>Dosage comic list</h2>
|
||||
<div id="comics">
|
||||
%(content)s
|
||||
</div>
|
||||
<script type="text/javascript">
|
||||
(function() {
|
||||
var po = document.createElement('script'); po.type = 'text/javascript'; po.async = true;
|
||||
po.src = 'https://apis.google.com/js/plusone.js';
|
||||
var s = document.getElementsByTagName('script')[0]; s.parentNode.insertBefore(po, s);
|
||||
})();
|
||||
</script>
|
||||
<script>
|
||||
window.onload = function() {
|
||||
var wall = new Masonry(document.getElementById('testresults'), {
|
||||
var wall = new Masonry(document.getElementById('comics'), {
|
||||
columnWidth: 240
|
||||
});
|
||||
};
|
||||
</script>
|
||||
</section>
|
||||
{%% endblock content %%}
|
||||
"""
|
||||
|
||||
comic_template = """
|
||||
---
|
||||
extends: base.j2
|
||||
title: Dosage comic %(name)s
|
||||
---
|
||||
{%% block content %%}
|
||||
<section id="main-content">
|
||||
|
||||
<h2>Dosage comic %(name)s</h2>
|
||||
<table class="comicinfo">
|
||||
<tr>
|
||||
<th>Description</th><td>%(description)s</td>
|
||||
</tr>
|
||||
<tr>
|
||||
<th>Website</th><td><a href="%(url)s">%(url)s</a></td>
|
||||
</tr>
|
||||
<tr>
|
||||
<th>Adult content</th><td>%(adult)s</td>
|
||||
</tr>
|
||||
<tr>
|
||||
<th>Available since</th><td>Dosage v%(since)s</td>
|
||||
</tr>
|
||||
<tr>
|
||||
<th>Status</th><td>%(status)s on %(date)s</td>
|
||||
</tr>
|
||||
</table>
|
||||
<div class="g-plusone" data-size="standard" data-annotation="inline" data-width="300"></div>
|
||||
<script type="text/javascript">
|
||||
(function() {
|
||||
var po = document.createElement('script'); po.type = 'text/javascript'; po.async = true;
|
||||
po.src = 'https://apis.google.com/js/plusone.js';
|
||||
var s = document.getElementsByTagName('script')[0]; s.parentNode.insertBefore(po, s);
|
||||
})();
|
||||
</script>
|
||||
</section>
|
||||
{%% endblock content %%}
|
||||
"""
|
||||
|
||||
entrytemplate_url = """
|
||||
<a href="%(url)s" title="%(title)s" class="%(css)s">%(name)s</a>
|
||||
<div class="g-plusone" data-size="medium" data-annotation="bubble" data-href="%(url)s"></div>
|
||||
"""
|
||||
|
||||
entrytemplate_nourl = """
|
||||
<span title="%(title)s" class="%(css)s">%(name)s</span>
|
||||
"""
|
||||
|
||||
|
||||
def get_mtime (filename):
|
||||
"""Return modification time of filename."""
|
||||
|
|
@ -49,75 +110,164 @@ def strdate(t):
|
|||
return time.strftime("%d.%m.%Y", time.localtime(t))
|
||||
|
||||
|
||||
def get_test_name(line):
|
||||
"""Get scraper name from test output line."""
|
||||
def get_testscraper(line):
|
||||
"""Get scraper from test output line."""
|
||||
classname = line.split('::')[1][4:]
|
||||
for scraper in get_scrapers():
|
||||
if scraper.__name__ == classname:
|
||||
try:
|
||||
url = scraper.starter()
|
||||
except Exception:
|
||||
url = None
|
||||
return scraper.get_name(), url
|
||||
return scraper
|
||||
raise ValueError("Scraper %r not found" % classname)
|
||||
|
||||
|
||||
def get_test(line):
|
||||
"""Get test name from test output line."""
|
||||
name, url = get_test_name(line)
|
||||
result = "OK" if line.startswith(". ") else "FAILED"
|
||||
return [name, url, result, ""]
|
||||
|
||||
|
||||
def get_content(filename):
|
||||
"""Get HTML content for test output."""
|
||||
tests = []
|
||||
def get_testinfo(filename, modified):
|
||||
"""Maintains a static list of comics which users can vote on.
|
||||
The original set of comic strips is stored in a JSON file which gets
|
||||
updated from the test results.
|
||||
If a comic strip stored in JSON is not found in the test results, it is
|
||||
orphaned.
|
||||
@return: {name -> {
|
||||
"status": Status.*,
|
||||
"url": string or None,
|
||||
"description": string or None,
|
||||
"error": string or None,
|
||||
}
|
||||
}
|
||||
"""
|
||||
if os.path.isfile(json_file):
|
||||
testinfo = load_result(json_file)
|
||||
else:
|
||||
testinfo = {}
|
||||
with open(filename, "r") as f:
|
||||
print("Tests parsed: 0", end=" ", file=sys.stderr)
|
||||
num_tests = 0
|
||||
add_reason = False
|
||||
add_error = False
|
||||
keys = []
|
||||
for line in f:
|
||||
if line.startswith((". ", "F ")) and "test_comics" in line:
|
||||
add_reason = line.startswith("F ")
|
||||
add_error = line.startswith("F ")
|
||||
num_tests += 1
|
||||
try:
|
||||
tests.append(get_test(line))
|
||||
except Exception as msg:
|
||||
print("WARNING:", msg, file=sys.stderr)
|
||||
continue
|
||||
elif add_reason and line.startswith(" E "):
|
||||
reason = line[3:].strip()
|
||||
tests[-1][-1] = reason
|
||||
key, entry = get_testentry(line)
|
||||
keys.append(key)
|
||||
update_testentry(key, entry, testinfo)
|
||||
elif add_error and line.startswith(" E "):
|
||||
entry["error"] = line[3:].strip()
|
||||
if num_tests % 5 == 0:
|
||||
print(num_tests, end=" ", file=sys.stderr)
|
||||
tests.sort()
|
||||
res = []
|
||||
for name, url, result, reason in tests:
|
||||
css = result.lower()
|
||||
if len(name) > 40:
|
||||
name = name[:37] + "..."
|
||||
if url:
|
||||
args = quote_all(url, reason, css, name)
|
||||
inner = '<a href="%s" title="%s" class="%s">%s</a>' % args
|
||||
orphan_entries(keys, testinfo)
|
||||
save_result(testinfo, json_file)
|
||||
return testinfo
|
||||
|
||||
|
||||
def get_testentry(line):
|
||||
"""Get one test entry."""
|
||||
scraper = get_testscraper(line)
|
||||
key = scraper.__name__
|
||||
name = scraper.get_name()
|
||||
if len(name) > 40:
|
||||
name = name[:37] + "..."
|
||||
entry = {
|
||||
"status": Status.ok if line.startswith(". ") else Status.error,
|
||||
"name": name,
|
||||
"url": None,
|
||||
"description": scraper.description,
|
||||
"error": None,
|
||||
"adult": scraper.adult,
|
||||
}
|
||||
try:
|
||||
entry["url"] = scraper.starter()
|
||||
except Exception as msg:
|
||||
print("WARNING:", msg, file=sys.stderr)
|
||||
return key, entry
|
||||
|
||||
|
||||
def orphan_entries(keys, testinfo):
|
||||
"""Mark all entries that are in testinfo but not in keys as orphaned."""
|
||||
for key, entry in testinfo.items():
|
||||
if key not in keys:
|
||||
entry["status"] = Status.orphan
|
||||
|
||||
|
||||
def update_testentry(key, entry, testinfo):
|
||||
if key not in testinfo:
|
||||
# add dosage version for this comic
|
||||
# XXX replace this after next release
|
||||
if key.startswith("Arcamax") or key in ("AmazingSuperPowers", "PandyLand"):
|
||||
entry["since"] = DosageVersion
|
||||
else:
|
||||
args = quote_all(reason, css, name)
|
||||
inner = '<span title="%s" class="%s">%s</span>' % args
|
||||
res.append('<div class="item">%s</div>' % inner)
|
||||
entry["since"] = "1.8"
|
||||
else:
|
||||
entry["since"] = testinfo[key]["since"]
|
||||
testinfo[key] = entry
|
||||
|
||||
|
||||
def get_html_index(testinfo):
|
||||
"""Get HTML content for test output index."""
|
||||
res = []
|
||||
for key in sorted(testinfo.keys()):
|
||||
entry = testinfo[key]
|
||||
css = entry["status"]
|
||||
url = "comics/%s.html" % key
|
||||
if entry["error"]:
|
||||
title = entry["error"]
|
||||
elif entry["description"]:
|
||||
title = entry["description"]
|
||||
else:
|
||||
title = entry["name"]
|
||||
args = {
|
||||
"url": quote(url),
|
||||
"title": quote(title),
|
||||
"css": quote(css),
|
||||
"name": quote(entry["name"]),
|
||||
}
|
||||
template = entrytemplate_url if url else entrytemplate_nourl
|
||||
entryhtml = template % args
|
||||
res.append('<div class="item">%s</div>' % entryhtml)
|
||||
return os.linesep.join(res)
|
||||
|
||||
|
||||
def quote_all(*args):
|
||||
"""CGI-escape all arguments for."""
|
||||
return tuple(cgi.escape(x, quote=True) for x in args)
|
||||
def write_html(testinfo, outputdir, modified):
|
||||
"""Write index page and all comic pages."""
|
||||
content = get_html_index(testinfo)
|
||||
date = strdate(modified)
|
||||
args = {"date": quote(date), "content": content}
|
||||
fname = os.path.join(outputdir, "comic_index.html")
|
||||
with open(fname, 'w') as fp:
|
||||
fp.write(indextemplate % args)
|
||||
comicdir = os.path.join(outputdir, "comics")
|
||||
if not os.path.isdir(comicdir):
|
||||
os.mkdir(comicdir)
|
||||
for key, entry in testinfo.items():
|
||||
write_html_comic(key, entry, comicdir, date)
|
||||
|
||||
|
||||
def write_html_comic(key, entry, outputdir, date):
|
||||
"""Write a comic page."""
|
||||
args = {
|
||||
"url": quote(entry["url"]),
|
||||
"name": quote(entry["name"]),
|
||||
"adult": quote("yes" if entry["adult"] else "no"),
|
||||
"since": quote(entry["since"]),
|
||||
"description": quote(entry["description"]),
|
||||
"status": quote(entry["status"]),
|
||||
"date": quote(date),
|
||||
}
|
||||
fname = os.path.join(outputdir, key+".html")
|
||||
with open(fname, 'w') as fp:
|
||||
fp.write(comic_template % args)
|
||||
|
||||
|
||||
def quote(arg):
|
||||
"""CGI-escape argument."""
|
||||
return cgi.escape(arg, quote=True)
|
||||
|
||||
|
||||
def main(args):
|
||||
"""Generate HTML output for test result."""
|
||||
filename = args[0]
|
||||
outputdir = args[1]
|
||||
modified = get_mtime(filename)
|
||||
content = get_content(filename)
|
||||
attrs = {"date": strdate(modified), "content": content}
|
||||
print(htmltemplate % attrs)
|
||||
testinfo = get_testinfo(filename, modified)
|
||||
write_html(testinfo, outputdir, modified)
|
||||
return 0
|
||||
|
||||
|
||||
|
|
|
|||
Loading…
Reference in a new issue