Generate individual comic pages.

Makefile

@@ -49,7 +49,7 @@ upload:
 	  dist/$(ARCHIVE_SOURCE).asc dist/$(ARCHIVE_WIN32).asc
 
 testresults:
-	scripts/mktestpage.py testresults.txt > $(HOMEPAGE)/content/testresults.html
+	scripts/mktestpage.py testresults.txt $(HOMEPAGE)/content
 
 homepage:
 # update metadata

scripts/mktestpage.json

@@ -0,0 +1 @@
+{"Arcamax_BabyBlues": {"adult": false, "description": "", "error": "AssertionError: Arcamax/BabyBlues http://www.arcamax.com/thefunnies/babyblues/ strip URL 'http://www.arcamax.com/thef\\", "name": "Arcamax/BabyBlues", "since": "1.9", "status": "error", "url": "http://www.arcamax.com/thefunnies/babyblues/"}, "BiggerThanCheeses": {"adult": false, "description": "", "error": null, "name": "BiggerThanCheeses", "since": "1.8", "status": "ok", "url": "http://www.biggercheese.com/"}, "Chester5000XYV": {"adult": false, "description": "", "error": null, "name": "Chester5000XYV", "since": "1.8", "status": "ok", "url": "http://jessfink.com/Chester5000XYV/"}, "DrunkDuck_Civil_Servitude": {"adult": false, "description": "", "error": null, "name": "DrunkDuck/Civil_Servitude", "since": "1.9", "status": "orphan", "url": "http://www.drunkduck.com/Civil_Servitude/4994879/"}, "DrunkDuck_Dasien": {"adult": false, "description": "", "error": "AssertionError: DrunkDuck/Dasien http://www.drunkduck.com/Dasien/5343187/ could not save http://media.drunkduck.com.s3.amazonaws.com:80/users/Neilsama/comics/Dasien/web/Chapter%2B7.jpg to /tmp/tmpNpJZd4: [Errno Unable to retrieve URL.] http://media.drunkduck.com.s3.amazonaws.com:80/users/Neilsama/comics/Dasien/web/Chapter%2B7.jpg: IOError('URL retrieval of http://media.drunkduck.com.s3.amazonaws.com:80/users/Neilsama/comics/Dasien/web/Chapter%2B7.jpg failed: 403 Client Error: Forbidden',)", "name": "DrunkDuck/Dasien", "since": "1.9", "status": "orphan", "url": "http://www.drunkduck.com/Dasien/5343187/"}, "DrunkDuck_Explorers_Of_the_Unknown": {"adult": false, "description": "", "error": "AssertionError: DrunkDuck/Explorers_Of_the_Unknown http://www.drunkduck.com/Explorers_Of_the_Unknown/5395556/ could n\\", "name": "DrunkDuck/Explorers_Of_the_Unknown", "since": "1.8", "status": "error", "url": "http://www.drunkduck.com/Explorers_Of_the_Unknown/5395556/"}, "DrunkDuck_Metal_Breakdown": {"adult": false, "description": "", "error": null, "name": "DrunkDuck/Metal_Breakdown", "since": "1.8", "status": "ok", "url": "http://www.drunkduck.com/Metal_Breakdown/5386007/"}, "NobodyScores": {"adult": false, "description": "", "error": null, "name": "NobodyScores", "since": "1.8", "status": "ok", "url": "http://nobodyscores.loosenutstudio.com/"}, "SmackJeeves_ERRORERROR": {"adult": false, "description": "A finnish teenage boy Tomi is a normal nerd. He plays videogames and gets average grades, but also is terribly bored with his life. But one day he accidentally gets inside of his computer and meets there a girl, who calls herself princess Jooda. And there is a profecy about Tomi, sword and Viruses. Warnings: If you are under 13 years old, I do not recommend this comic to you. Contains bad language, blood and somewhat sexual themes. ps. sorry for style changes, i practise haha Updates 2-4 pages a week.", "error": null, "name": "SmackJeeves/ERRORERROR", "since": "1.8", "status": "ok", "url": "http://errorerror.smackjeeves.com/comics/1664476/ee297/"}, "SmackJeeves_SoulGuardian": {"adult": false, "description": "One day humans will create viruses with the intention of infecting themselves. With a virus strong enough, a person would even be able to travel across space and time...they would have the power to change their destiny, but at what cost? READ: Right to left UPDATES: Wednesday", "error": null, "name": "SmackJeeves/SoulGuardian", "since": "1.8", "status": "ok", "url": "http://soulguardian.smackjeeves.com/comics/1664888/nav-11-96/"}, "WebcomicsNation_AgnesQuill": {"adult": false, "description": "", "error": null, "name": "WebcomicsNation/AgnesQuill", "since": "1.9", "status": "orphan", "url": "http://www.webcomicsnation.com/daveroman/agnes/"}}

scripts/mktestpage.py

@@ -7,37 +7,98 @@ import time
 import cgi
 sys.path.append(os.path.join(os.path.dirname(__file__), ".."))
 from dosagelib.scraper import get_scrapers
+from dosagelib.configuration import Version as DosageVersion
+from scriptutil import load_result, save_result
 
-htmltemplate = """
+json_file = __file__.replace(".py", ".json")
+
+class Status:
+    ok = "ok"
+    error = "error"
+    orphan = "orphan"
+
+indextemplate = """
 ---
 extends: base.j2
-title: Dosage by Bastian Kleineidam
+title: Dosage comic list
-description: a commandline webcomic downloader and archiver
+description: a list of comic strips supported by Dosage
 ---
 {%% block js %%}
-<script src="media/js/masonry.min.js"></script>
+<script src="{{ media_url('js/masonry.min.js') }}"></script>
 {%% endblock js %%}
 
 {%% block content %%}
-<div class="inner clearfix">
 <section id="main-content">
 
-<h2>Dosage test results from %(date)s</h2>
+<h2>Dosage comic list</h2>
-<p>Note that it is almost impossible to get a 100%% OK test run
+<div id="comics">
-due to temporary site failures.</p>
-<div id="testresults">
 %(content)s
 </div>
+<script type="text/javascript">
+  (function() {
+    var po = document.createElement('script'); po.type = 'text/javascript'; po.async = true;
+    po.src = 'https://apis.google.com/js/plusone.js';
+    var s = document.getElementsByTagName('script')[0]; s.parentNode.insertBefore(po, s);
+  })();
+</script>
 <script>
 window.onload = function() {
-  var wall = new Masonry(document.getElementById('testresults'), {
+  var wall = new Masonry(document.getElementById('comics'), {
     columnWidth: 240
   });
 };
 </script>
+</section>
 {%% endblock content %%}
 """
 
+comic_template = """
+---
+extends: base.j2
+title: Dosage comic %(name)s
+---
+{%% block content %%}
+<section id="main-content">
+
+<h2>Dosage comic %(name)s</h2>
+<table class="comicinfo">
+<tr>
+<th>Description</th><td>%(description)s</td>
+</tr>
+<tr>
+<th>Website</th><td><a href="%(url)s">%(url)s</a></td>
+</tr>
+<tr>
+<th>Adult content</th><td>%(adult)s</td>
+</tr>
+<tr>
+<th>Available since</th><td>Dosage v%(since)s</td>
+</tr>
+<tr>
+<th>Status</th><td>%(status)s on %(date)s</td>
+</tr>
+</table>
+<div class="g-plusone" data-size="standard" data-annotation="inline" data-width="300"></div>
+<script type="text/javascript">
+  (function() {
+    var po = document.createElement('script'); po.type = 'text/javascript'; po.async = true;
+    po.src = 'https://apis.google.com/js/plusone.js';
+    var s = document.getElementsByTagName('script')[0]; s.parentNode.insertBefore(po, s);
+  })();
+</script>
+</section>
+{%% endblock content %%}
+"""
+
+entrytemplate_url = """
+<a href="%(url)s" title="%(title)s" class="%(css)s">%(name)s</a>
+<div class="g-plusone" data-size="medium" data-annotation="bubble" data-href="%(url)s"></div>
+"""
+
+entrytemplate_nourl = """
+<span title="%(title)s" class="%(css)s">%(name)s</span>
+"""
+
 
 def get_mtime (filename):
     """Return modification time of filename."""
@@ -49,75 +110,164 @@ def strdate(t):
     return time.strftime("%d.%m.%Y", time.localtime(t))
 
 
-def get_test_name(line):
+def get_testscraper(line):
-    """Get scraper name from test output line."""
+    """Get scraper from test output line."""
     classname = line.split('::')[1][4:]
     for scraper in get_scrapers():
         if scraper.__name__ == classname:
-            try:
+            return scraper
-                url = scraper.starter()
-            except Exception:
-                url = None
-            return scraper.get_name(), url
     raise ValueError("Scraper %r not found" % classname)
 
 
-def get_test(line):
+def get_testinfo(filename, modified):
-    """Get test name from test output line."""
+    """Maintains a static list of comics which users can vote on.
-    name, url = get_test_name(line)
+    The original set of comic strips is stored in a JSON file which gets
-    result = "OK" if line.startswith(". ") else "FAILED"
+    updated from the test results.
-    return [name, url, result, ""]
+    If a comic strip stored in JSON is not found in the test results, it is
-
+    orphaned.
-
+    @return: {name -> {
-def get_content(filename):
+                "status": Status.*,
-    """Get HTML content for test output."""
+                "url": string or None,
-    tests = []
+                "description": string or None,
+                "error": string or None,
+               }
+             }
+    """
+    if os.path.isfile(json_file):
+        testinfo = load_result(json_file)
+    else:
+        testinfo = {}
     with open(filename, "r") as f:
         print("Tests parsed: 0", end=" ", file=sys.stderr)
         num_tests = 0
-        add_reason = False
+        add_error = False
+        keys = []
         for line in f:
             if line.startswith((". ", "F ")) and "test_comics" in line:
-                add_reason = line.startswith("F ")
+                add_error = line.startswith("F ")
                 num_tests += 1
-                try:
+                key, entry = get_testentry(line)
-                    tests.append(get_test(line))
+                keys.append(key)
-                except Exception as msg:
+                update_testentry(key, entry, testinfo)
-                    print("WARNING:", msg, file=sys.stderr)
+            elif add_error and line.startswith(" E "):
-                    continue
+                entry["error"] = line[3:].strip()
-            elif add_reason and line.startswith(" E "):
-                reason = line[3:].strip()
-                tests[-1][-1] = reason
             if num_tests % 5 == 0:
                 print(num_tests, end=" ", file=sys.stderr)
-    tests.sort()
+    orphan_entries(keys, testinfo)
-    res = []
+    save_result(testinfo, json_file)
-    for name, url, result, reason in tests:
+    return testinfo
-        css = result.lower()
+
-        if len(name) > 40:
+
-            name = name[:37] + "..."
+def get_testentry(line):
-        if url:
+    """Get one test entry."""
-            args = quote_all(url, reason, css, name)
+    scraper = get_testscraper(line)
-            inner = '<a href="%s" title="%s" class="%s">%s</a>' % args
+    key = scraper.__name__
+    name = scraper.get_name()
+    if len(name) > 40:
+        name = name[:37] + "..."
+    entry = {
+        "status": Status.ok if line.startswith(". ") else Status.error,
+        "name": name,
+        "url": None,
+        "description": scraper.description,
+        "error": None,
+        "adult": scraper.adult,
+    }
+    try:
+        entry["url"] = scraper.starter()
+    except Exception as msg:
+        print("WARNING:", msg, file=sys.stderr)
+    return key, entry
+
+
+def orphan_entries(keys, testinfo):
+    """Mark all entries that are in testinfo but not in keys as orphaned."""
+    for key, entry in testinfo.items():
+        if key not in keys:
+            entry["status"] = Status.orphan
+
+
+def update_testentry(key, entry, testinfo):
+    if key not in testinfo:
+        # add dosage version for this comic
+        # XXX replace this after next release
+        if key.startswith("Arcamax") or key in ("AmazingSuperPowers", "PandyLand"):
+            entry["since"] = DosageVersion
         else:
-            args = quote_all(reason, css, name)
+            entry["since"] = "1.8"
-            inner = '<span title="%s" class="%s">%s</span>' % args
+    else:
-        res.append('<div class="item">%s</div>' % inner)
+        entry["since"] = testinfo[key]["since"]
+    testinfo[key] = entry
+
+
+def get_html_index(testinfo):
+    """Get HTML content for test output index."""
+    res = []
+    for key in sorted(testinfo.keys()):
+        entry = testinfo[key]
+        css = entry["status"]
+        url = "comics/%s.html" % key
+        if entry["error"]:
+            title = entry["error"]
+        elif entry["description"]:
+            title = entry["description"]
+        else:
+            title = entry["name"]
+        args = {
+            "url": quote(url),
+            "title": quote(title),
+            "css": quote(css),
+            "name": quote(entry["name"]),
+        }
+        template = entrytemplate_url if url else entrytemplate_nourl
+        entryhtml = template % args
+        res.append('<div class="item">%s</div>' % entryhtml)
     return os.linesep.join(res)
 
 
-def quote_all(*args):
+def write_html(testinfo, outputdir, modified):
-    """CGI-escape all arguments for."""
+    """Write index page and all comic pages."""
-    return tuple(cgi.escape(x, quote=True) for x in args)
+    content = get_html_index(testinfo)
+    date = strdate(modified)
+    args = {"date": quote(date), "content": content}
+    fname = os.path.join(outputdir, "comic_index.html")
+    with open(fname, 'w') as fp:
+        fp.write(indextemplate % args)
+    comicdir = os.path.join(outputdir, "comics")
+    if not os.path.isdir(comicdir):
+        os.mkdir(comicdir)
+    for key, entry in testinfo.items():
+        write_html_comic(key, entry, comicdir, date)
+
+
+def write_html_comic(key, entry, outputdir, date):
+    """Write a comic page."""
+    args = {
+        "url": quote(entry["url"]),
+        "name": quote(entry["name"]),
+        "adult": quote("yes" if entry["adult"] else "no"),
+        "since": quote(entry["since"]),
+        "description": quote(entry["description"]),
+        "status": quote(entry["status"]),
+        "date": quote(date),
+    }
+    fname = os.path.join(outputdir, key+".html")
+    with open(fname, 'w') as fp:
+        fp.write(comic_template % args)
+
+
+def quote(arg):
+    """CGI-escape argument."""
+    return cgi.escape(arg, quote=True)
 
 
 def main(args):
     """Generate HTML output for test result."""
     filename = args[0]
+    outputdir = args[1]
     modified = get_mtime(filename)
-    content = get_content(filename)
+    testinfo = get_testinfo(filename, modified)
-    attrs = {"date": strdate(modified), "content": content}
+    write_html(testinfo, outputdir, modified)
-    print(htmltemplate % attrs)
     return 0