dosage/scripts/creators.py

65 lines
1.9 KiB
Python
Raw Normal View History

#!/usr/bin/env python
# Copyright (C) 2012 Bastian Kleineidam
"""
2012-12-12 16:41:29 +00:00
Script to get a list of creators.com comics and save the info in a JSON file for further processing.
"""
from __future__ import print_function
import re
import sys
import os
sys.path.append(os.path.join(os.path.dirname(__file__), ".."))
from dosagelib.util import getPageContent, asciify, unescape, tagre
2012-12-19 19:42:53 +00:00
from scriptutil import contains_case_insensitive, capfirst, save_result, load_result
json_file = __file__.replace(".py", ".json")
url_matcher = re.compile(tagre("a", "href", r'(/comics/[^/]+)\.html') + r'<strong>([^<]+)</strong>')
2012-11-29 05:46:58 +00:00
# names of comics to exclude
exclude_comics = [
]
def handle_url(url, res):
"""Parse one search result page."""
print("Parsing", url, file=sys.stderr)
try:
data, baseUrl = getPageContent(url)
except IOError as msg:
print("ERROR:", msg, file=sys.stderr)
return
for match in url_matcher.finditer(data):
url = match.group(1)
name = unescape(match.group(2))
name = asciify(name.replace('&', 'And').replace('@', 'At'))
2012-12-12 16:41:29 +00:00
name = capfirst(name)
2012-11-29 05:46:58 +00:00
if name in exclude_comics:
continue
if contains_case_insensitive(res, name):
# we cannot handle two comics that only differ in case
2012-12-19 19:42:53 +00:00
print("INFO: skipping possible duplicate", name, file=sys.stderr)
continue
res[name] = url
def get_results():
"""Parse all search result pages."""
# store info in a dictionary {name -> shortname}
res = {}
handle_url('http://www.creators.com/comics/cat-seeall.html', res)
2012-12-19 19:42:53 +00:00
save_result(res, json_file)
def print_results(args):
"""Print comics."""
2012-12-19 19:42:53 +00:00
for name, url in sorted(load_result(json_file).items()):
2012-11-29 05:46:58 +00:00
if name in exclude_comics:
continue
print("add(%r, %r)" % (str(name), str(url)))
if __name__ == '__main__':
if len(sys.argv) > 1:
print_results(sys.argv[1:])
else:
get_results()