dosage/scripts/arcamax.py

92 lines
2.8 KiB
Python
Raw Normal View History

2013-01-23 18:34:11 +00:00
#!/usr/bin/env python
# Copyright (C) 2013 Bastian Kleineidam
"""
Script to get arcamax comics and save the info in a JSON file for further processing.
"""
from __future__ import print_function
import re
import sys
import os
2013-02-12 20:53:57 +00:00
import requests
2013-01-23 18:34:11 +00:00
sys.path.append(os.path.join(os.path.dirname(__file__), ".."))
from dosagelib.util import getPageContent, asciify, unescape
from dosagelib.scraper import get_scraperclasses
2013-01-23 18:34:11 +00:00
from scriptutil import contains_case_insensitive, capfirst, save_result, load_result, truncate_name
json_file = __file__.replace(".py", ".json")
url_matcher = re.compile(r'<li><b><a href="(/thefunnies/[^"]+)">([^<]+)</a>')
# names of comics to exclude
exclude_comics = [
]
2013-02-12 20:53:57 +00:00
def handle_url(url, session, res):
2013-01-23 18:34:11 +00:00
"""Parse one search result page."""
print("Parsing", url, file=sys.stderr)
try:
2013-02-12 20:53:57 +00:00
data, baseUrl = getPageContent(url, session)
2013-01-23 18:34:11 +00:00
except IOError as msg:
print("ERROR:", msg, file=sys.stderr)
return
for match in url_matcher.finditer(data):
shortname = match.group(1)
name = unescape(match.group(2))
name = asciify(name.replace('&', 'And').replace('@', 'At'))
name = capfirst(name)
if name in exclude_comics:
continue
if contains_case_insensitive(res, name):
# we cannot handle two comics that only differ in case
2013-03-12 19:47:11 +00:00
print("INFO: skipping possible duplicate", repr(name), file=sys.stderr)
2013-01-23 18:34:11 +00:00
continue
res[name] = shortname
if not res:
print("ERROR:", "did not match any comics", file=sys.stderr)
def get_results():
"""Parse all search result pages."""
# store info in a dictionary {name -> shortname}
res = {}
2013-02-12 20:53:57 +00:00
session = requests.Session()
handle_url('http://www.arcamax.com/comics', session, res)
2013-01-23 18:34:11 +00:00
save_result(res, json_file)
def has_comic(name):
"""Check if comic name already exists."""
names = [
("Creators/%s" % name).lower(),
("DrunkDuck/%s" % name).lower(),
("GoComics/%s" % name).lower(),
("KeenSpot/%s" % name).lower(),
("ComicGenesis/%s" % name).lower(),
2013-01-23 18:34:11 +00:00
("SmackJeeves/%s" % name).lower(),
]
for scraperclass in get_scraperclasses():
lname = scraperclass.getName().lower()
2013-01-23 18:34:11 +00:00
if lname in names:
return True
return False
def print_results(args):
"""Print all comics that have at least the given number of minimum comic strips."""
for name, shortname in sorted(load_result(json_file).items()):
if name in exclude_comics:
continue
if has_comic(name):
prefix = '#'
else:
prefix = ''
print("%sadd(%r, %r)" % (prefix, str(truncate_name(name)), str(shortname)))
if __name__ == '__main__':
if len(sys.argv) > 1:
print_results(sys.argv[1:])
else:
get_results()