dmoonfire/dosage
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Activity

Generate individual comic pages.

Browse source
This commit is contained in:
Bastian Kleineidam 2013-01-24 21:42:04 +01:00
parent 4b35d332dc
commit 6634a39d61
3 changed files with 207 additions and 56 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

2
Makefile
View file

@ -49,7 +49,7 @@ upload:
dist/$(ARCHIVE_SOURCE).asc dist/$(ARCHIVE_WIN32).asc
testresults:
scripts/mktestpage.py testresults.txt > $(HOMEPAGE)/content/testresults.html
scripts/mktestpage.py testresults.txt $(HOMEPAGE)/content
homepage:
# update metadata

1
scripts/mktestpage.json Normal file
View file

@ -0,0 +1 @@
{"Arcamax_BabyBlues": {"adult": false, "description": "", "error": "AssertionError: Arcamax/BabyBlues http://www.arcamax.com/thefunnies/babyblues/ strip URL 'http://www.arcamax.com/thef\\", "name": "Arcamax/BabyBlues", "since": "1.9", "status": "error", "url": "http://www.arcamax.com/thefunnies/babyblues/"}, "BiggerThanCheeses": {"adult": false, "description": "", "error": null, "name": "BiggerThanCheeses", "since": "1.8", "status": "ok", "url": "http://www.biggercheese.com/"}, "Chester5000XYV": {"adult": false, "description": "", "error": null, "name": "Chester5000XYV", "since": "1.8", "status": "ok", "url": "http://jessfink.com/Chester5000XYV/"}, "DrunkDuck_Civil_Servitude": {"adult": false, "description": "", "error": null, "name": "DrunkDuck/Civil_Servitude", "since": "1.9", "status": "orphan", "url": "http://www.drunkduck.com/Civil_Servitude/4994879/"}, "DrunkDuck_Dasien": {"adult": false, "description": "", "error": "AssertionError: DrunkDuck/Dasien http://www.drunkduck.com/Dasien/5343187/ could not save http://media.drunkduck.com.s3.amazonaws.com:80/users/Neilsama/comics/Dasien/web/Chapter%2B7.jpg to /tmp/tmpNpJZd4: [Errno Unable to retrieve URL.] http://media.drunkduck.com.s3.amazonaws.com:80/users/Neilsama/comics/Dasien/web/Chapter%2B7.jpg: IOError('URL retrieval of http://media.drunkduck.com.s3.amazonaws.com:80/users/Neilsama/comics/Dasien/web/Chapter%2B7.jpg failed: 403 Client Error: Forbidden',)", "name": "DrunkDuck/Dasien", "since": "1.9", "status": "orphan", "url": "http://www.drunkduck.com/Dasien/5343187/"}, "DrunkDuck_Explorers_Of_the_Unknown": {"adult": false, "description": "", "error": "AssertionError: DrunkDuck/Explorers_Of_the_Unknown http://www.drunkduck.com/Explorers_Of_the_Unknown/5395556/ could n\\", "name": "DrunkDuck/Explorers_Of_the_Unknown", "since": "1.8", "status": "error", "url": "http://www.drunkduck.com/Explorers_Of_the_Unknown/5395556/"}, "DrunkDuck_Metal_Breakdown": {"adult": false, "description": "", "error": null, "name": "DrunkDuck/Metal_Breakdown", "since": "1.8", "status": "ok", "url": "http://www.drunkduck.com/Metal_Breakdown/5386007/"}, "NobodyScores": {"adult": false, "description": "", "error": null, "name": "NobodyScores", "since": "1.8", "status": "ok", "url": "http://nobodyscores.loosenutstudio.com/"}, "SmackJeeves_ERRORERROR": {"adult": false, "description": "A finnish teenage boy Tomi is a normal nerd. He plays videogames and gets average grades, but also is terribly bored with his life. But one day he accidentally gets inside of his computer and meets there a girl, who calls herself princess Jooda. And there is a profecy about Tomi, sword and Viruses. Warnings: If you are under 13 years old, I do not recommend this comic to you. Contains bad language, blood and somewhat sexual themes. ps. sorry for style changes, i practise haha Updates 2-4 pages a week.", "error": null, "name": "SmackJeeves/ERRORERROR", "since": "1.8", "status": "ok", "url": "http://errorerror.smackjeeves.com/comics/1664476/ee297/"}, "SmackJeeves_SoulGuardian": {"adult": false, "description": "One day humans will create viruses with the intention of infecting themselves. With a virus strong enough, a person would even be able to travel across space and time...they would have the power to change their destiny, but at what cost? READ: Right to left UPDATES: Wednesday", "error": null, "name": "SmackJeeves/SoulGuardian", "since": "1.8", "status": "ok", "url": "http://soulguardian.smackjeeves.com/comics/1664888/nav-11-96/"}, "WebcomicsNation_AgnesQuill": {"adult": false, "description": "", "error": null, "name": "WebcomicsNation/AgnesQuill", "since": "1.9", "status": "orphan", "url": "http://www.webcomicsnation.com/daveroman/agnes/"}}

260
scripts/mktestpage.py
View file

@ -7,37 +7,98 @@ import time
import cgi
sys.path.append(os.path.join(os.path.dirname(__file__), ".."))
from dosagelib.scraper import get_scrapers
from dosagelib.configuration import Version as DosageVersion
from scriptutil import load_result, save_result
htmltemplate = """
json_file = __file__.replace(".py", ".json")
class Status:
ok = "ok"
error = "error"
orphan = "orphan"
indextemplate = """
---
extends: base.j2
title: Dosage by Bastian Kleineidam
description: a commandline webcomic downloader and archiver
title: Dosage comic list
description: a list of comic strips supported by Dosage
---
{%% block js %%}
<script src="media/js/masonry.min.js"></script>
<script src="{{ media_url('js/masonry.min.js') }}"></script>
{%% endblock js %%}
{%% block content %%}
<div class="inner clearfix">
<section id="main-content">
<h2>Dosage test results from %(date)s</h2>
<p>Note that it is almost impossible to get a 100%% OK test run
due to temporary site failures.</p>
<div id="testresults">
<h2>Dosage comic list</h2>
<div id="comics">
%(content)s
</div>
<script type="text/javascript">
(function() {
var po = document.createElement('script'); po.type = 'text/javascript'; po.async = true;
po.src = 'https://apis.google.com/js/plusone.js';
var s = document.getElementsByTagName('script')[0]; s.parentNode.insertBefore(po, s);
})();
</script>
<script>
window.onload = function() {
var wall = new Masonry(document.getElementById('testresults'), {
var wall = new Masonry(document.getElementById('comics'), {
columnWidth: 240
});
};
</script>
</section>
{%% endblock content %%}
"""
comic_template = """
---
extends: base.j2
title: Dosage comic %(name)s
---
{%% block content %%}
<section id="main-content">
<h2>Dosage comic %(name)s</h2>
<table class="comicinfo">
<tr>
<th>Description</th><td>%(description)s</td>
</tr>
<tr>
<th>Website</th><td><a href="%(url)s">%(url)s</a></td>
</tr>
<tr>
<th>Adult content</th><td>%(adult)s</td>
</tr>
<tr>
<th>Available since</th><td>Dosage v%(since)s</td>
</tr>
<tr>
<th>Status</th><td>%(status)s on %(date)s</td>
</tr>
</table>
<div class="g-plusone" data-size="standard" data-annotation="inline" data-width="300"></div>
<script type="text/javascript">
(function() {
var po = document.createElement('script'); po.type = 'text/javascript'; po.async = true;
po.src = 'https://apis.google.com/js/plusone.js';
var s = document.getElementsByTagName('script')[0]; s.parentNode.insertBefore(po, s);
})();
</script>
</section>
{%% endblock content %%}
"""
entrytemplate_url = """
<a href="%(url)s" title="%(title)s" class="%(css)s">%(name)s</a>
<div class="g-plusone" data-size="medium" data-annotation="bubble" data-href="%(url)s"></div>
"""
entrytemplate_nourl = """
<span title="%(title)s" class="%(css)s">%(name)s</span>
"""
def get_mtime (filename):
"""Return modification time of filename."""
@ -49,75 +110,164 @@ def strdate(t):
return time.strftime("%d.%m.%Y", time.localtime(t))
def get_test_name(line):
"""Get scraper name from test output line."""
def get_testscraper(line):
"""Get scraper from test output line."""
classname = line.split('::')[1][4:]
for scraper in get_scrapers():
if scraper.__name__ == classname:
try:
url = scraper.starter()
except Exception:
url = None
return scraper.get_name(), url
return scraper
raise ValueError("Scraper %r not found" % classname)
def get_test(line):
"""Get test name from test output line."""
name, url = get_test_name(line)
result = "OK" if line.startswith(". ") else "FAILED"
return [name, url, result, ""]
def get_content(filename):
"""Get HTML content for test output."""
tests = []
def get_testinfo(filename, modified):
"""Maintains a static list of comics which users can vote on.
The original set of comic strips is stored in a JSON file which gets
updated from the test results.
If a comic strip stored in JSON is not found in the test results, it is
orphaned.
@return: {name -> {
"status": Status.*,
"url": string or None,
"description": string or None,
"error": string or None,
}
}
"""
if os.path.isfile(json_file):
testinfo = load_result(json_file)
else:
testinfo = {}
with open(filename, "r") as f:
print("Tests parsed: 0", end=" ", file=sys.stderr)
num_tests = 0
add_reason = False
add_error = False
keys = []
for line in f:
if line.startswith((". ", "F ")) and "test_comics" in line:
add_reason = line.startswith("F ")
add_error = line.startswith("F ")
num_tests += 1
try:
tests.append(get_test(line))
except Exception as msg:
print("WARNING:", msg, file=sys.stderr)
continue
elif add_reason and line.startswith(" E "):
reason = line[3:].strip()
tests[-1][-1] = reason
key, entry = get_testentry(line)
keys.append(key)
update_testentry(key, entry, testinfo)
elif add_error and line.startswith(" E "):
entry["error"] = line[3:].strip()
if num_tests % 5 == 0:
print(num_tests, end=" ", file=sys.stderr)
tests.sort()
res = []
for name, url, result, reason in tests:
css = result.lower()
if len(name) > 40:
name = name[:37] + "..."
if url:
args = quote_all(url, reason, css, name)
inner = '<a href="%s" title="%s" class="%s">%s</a>' % args
orphan_entries(keys, testinfo)
save_result(testinfo, json_file)
return testinfo
def get_testentry(line):
"""Get one test entry."""
scraper = get_testscraper(line)
key = scraper.__name__
name = scraper.get_name()
if len(name) > 40:
name = name[:37] + "..."
entry = {
"status": Status.ok if line.startswith(". ") else Status.error,
"name": name,
"url": None,
"description": scraper.description,
"error": None,
"adult": scraper.adult,
}
try:
entry["url"] = scraper.starter()
except Exception as msg:
print("WARNING:", msg, file=sys.stderr)
return key, entry
def orphan_entries(keys, testinfo):
"""Mark all entries that are in testinfo but not in keys as orphaned."""
for key, entry in testinfo.items():
if key not in keys:
entry["status"] = Status.orphan
def update_testentry(key, entry, testinfo):
if key not in testinfo:
# add dosage version for this comic
# XXX replace this after next release
if key.startswith("Arcamax") or key in ("AmazingSuperPowers", "PandyLand"):
entry["since"] = DosageVersion
else:
args = quote_all(reason, css, name)
inner = '<span title="%s" class="%s">%s</span>' % args
res.append('<div class="item">%s</div>' % inner)
entry["since"] = "1.8"
else:
entry["since"] = testinfo[key]["since"]
testinfo[key] = entry
def get_html_index(testinfo):
"""Get HTML content for test output index."""
res = []
for key in sorted(testinfo.keys()):
entry = testinfo[key]
css = entry["status"]
url = "comics/%s.html" % key
if entry["error"]:
title = entry["error"]
elif entry["description"]:
title = entry["description"]
else:
title = entry["name"]
args = {
"url": quote(url),
"title": quote(title),
"css": quote(css),
"name": quote(entry["name"]),
}
template = entrytemplate_url if url else entrytemplate_nourl
entryhtml = template % args
res.append('<div class="item">%s</div>' % entryhtml)
return os.linesep.join(res)
def quote_all(*args):
"""CGI-escape all arguments for."""
return tuple(cgi.escape(x, quote=True) for x in args)
def write_html(testinfo, outputdir, modified):
"""Write index page and all comic pages."""
content = get_html_index(testinfo)
date = strdate(modified)
args = {"date": quote(date), "content": content}
fname = os.path.join(outputdir, "comic_index.html")
with open(fname, 'w') as fp:
fp.write(indextemplate % args)
comicdir = os.path.join(outputdir, "comics")
if not os.path.isdir(comicdir):
os.mkdir(comicdir)
for key, entry in testinfo.items():
write_html_comic(key, entry, comicdir, date)
def write_html_comic(key, entry, outputdir, date):
"""Write a comic page."""
args = {
"url": quote(entry["url"]),
"name": quote(entry["name"]),
"adult": quote("yes" if entry["adult"] else "no"),
"since": quote(entry["since"]),
"description": quote(entry["description"]),
"status": quote(entry["status"]),
"date": quote(date),
}
fname = os.path.join(outputdir, key+".html")
with open(fname, 'w') as fp:
fp.write(comic_template % args)
def quote(arg):
"""CGI-escape argument."""
return cgi.escape(arg, quote=True)
def main(args):
"""Generate HTML output for test result."""
filename = args[0]
outputdir = args[1]
modified = get_mtime(filename)
content = get_content(filename)
attrs = {"date": strdate(modified), "content": content}
print(htmltemplate % attrs)
testinfo = get_testinfo(filename, modified)
write_html(testinfo, outputdir, modified)
return 0