dosage/scripts/drunkduck.py

#!/usr/bin/env python
# Copyright (C) 2012 Bastian Kleineidam
"""
Script to get drunkduck comics and save the info in a JSON file for further processing.
"""
from __future__ import print_function
import re
import sys
import os
import json
sys.path.append(os.path.join(os.path.dirname(__file__), ".."))
from dosagelib.util import tagre, getPageContent
from scriptutil import contains_case_insensitive

json_file = __file__.replace(".py", ".json")

# names of comics to exclude
exclude_comics = [
    "Twonks_and_Plonkers", # broken images, no real content
    "U_Chuu_No_Hoshi_Hotoshi_Tsuko", # broken images
    "Red_Dog_Venue", # start page is broken
    "Monster_Lover", # start page is broken
    "Legacy_of_Blaze", # broken images
    "Dead_Strangers", # broken images
    "Crack", # broken images
    "Iron_Wolf", # broken images
]


def handle_url(url, url_matcher, num_matcher, res):
    """Parse one search result page."""
    try:
        data, baseUrl = getPageContent(url)
    except IOError as msg:
        print("ERROR:", msg, file=sys.stderr)
        return
    for match in url_matcher.finditer(data):
        comicurl = match.group(1)
        name = comicurl[:-1].rsplit('/')[-1]
        if contains_case_insensitive(res, name):
            # we cannot handle two comics that only differ in case
            print("WARN: skipping possible duplicate", name, file=sys.stderr)
            continue
        if name in exclude_comics:
            continue
        # find out how many images this comic has
        end = match.end(1)
        mo = num_matcher.search(data[end:])
        if not mo:
            print("ERROR:", repr(data[end:end+300], file=sys.stderr))
            continue
        num = int(mo.group(1))
        res[name] = num


def save_result(res):
    """Save result to file."""
    with open(json_file, 'wb') as f:
        json.dump(res, f, sort_keys=True)


def get_results():
    """Parse all search result pages."""
    base = "http://www.drunkduck.com/search/?page=%d&search=&type=0&type=1&last_update="
    href = re.compile(tagre("a", "href", r'(/[^"]+/)', before="size24 yanone blue"))
    num = re.compile(r'(\d+) pages?</span>')
    # store info in a dictionary {name -> number of comics}
    res = {}
    # a search for an empty string returned 825 result pages
    result_pages = 825
    print("Parsing", result_pages, "search result pages...", file=sys.stderr)
    for i in range(1, result_pages + 1):
        print(i, file=sys.stderr, end=" ")
        handle_url(base % i, href, num, res)
    save_result(res)


def print_results(min_strips):
    """Print all comics that have at least the given number of minimum comic strips."""
    with open(json_file, "rb") as f:
        comics = json.load(f)
    for name, num in sorted(comics.items()):
        if name in exclude_comics:
            continue
        if num >= min_strips:
            print("add('%s')" % name)


if __name__ == '__main__':
    if len(sys.argv) > 1:
        print_results(int(sys.argv[1]))
    else:
        get_results()