dosage/scripts/gocomics.py

125 lines
4.2 KiB
Python
Raw Normal View History

#!/usr/bin/env python
2014-01-05 16:50:57 +01:00
# Copyright (C) 2012-2014 Bastian Kleineidam
"""
2012-12-12 17:41:29 +01:00
Script to get a list of gocomics and save the info in a JSON file for further processing.
"""
from __future__ import print_function
2013-05-22 22:29:03 +02:00
import codecs
import re
import sys
import os
2013-02-12 21:53:57 +01:00
import requests
sys.path.append(os.path.join(os.path.dirname(__file__), ".."))
from dosagelib.util import tagre, getPageContent, asciify, unescape
2013-01-09 22:20:03 +01:00
from scriptutil import contains_case_insensitive, capfirst, save_result, load_result, truncate_name
json_file = __file__.replace(".py", ".json")
#<a href="/shortname" class="alpha_list updated">name</a>
url_matcher = re.compile(tagre("a", "href", r'(/[^"]+)', after="alpha_list") + r"([^<]+)</a>")
2012-11-29 06:46:58 +01:00
# names of comics to exclude
exclude_comics = [
2013-04-28 19:58:38 +02:00
"Adagio", # too few comics
2013-04-04 18:30:02 +02:00
"AgentGates", # too few comics
2012-12-04 07:02:40 +01:00
"Apocalypseharry", # too few comics
"BatkidandBatrat", # too few comics
"BETWEENTHELINES", # comic unavailable
"Bonner", # missing page
"Buster", # comic unavailabe
2013-03-07 19:54:18 +01:00
"CarteBlanche", # missing images
2013-11-12 18:33:14 +01:00
"Critterdoodles", # missing images
2013-04-28 19:58:38 +02:00
"CountyLine", # too few comics
2013-07-09 22:21:12 +02:00
"Crawdiddy", # comic unavailable
2012-12-04 07:02:40 +01:00
"DALTONDOG", # comic unavailable
"DellAndSteve", # too few comics
"Dilbert", # redirect
"DutchnPals", # too few comics
2013-03-07 23:08:17 +01:00
"EclecticCartoons", # missing images
2013-03-07 18:22:24 +01:00
"FlexandTone", # too few comics
2013-03-07 23:08:17 +01:00
"FrikkFrakkAndFrank", # too few comics
2013-04-04 18:30:02 +02:00
"GOODAndEVIL", # too few comics
2013-04-28 19:58:38 +02:00
"GoodwithCoffee", # too few comics
2012-12-04 07:02:40 +01:00
"InkeeDoodles", # comic unavailable
2013-11-12 18:33:14 +01:00
"JoesBar", # missing images
2013-04-04 18:30:02 +02:00
"KALEECHIKORNERS", # too few comics
2013-04-11 18:27:43 +02:00
"LoveIs", # missing images
2012-12-04 07:02:40 +01:00
"MaggiesComics", # too few comics
2013-07-09 22:21:12 +02:00
"MagicCoffeeHair", # too few comics
"NickGalifianakis", # too few comics
2012-12-04 07:02:40 +01:00
"OfMiceandMud", # too few comics
"OysterWar", # too few comics
2013-04-28 19:58:38 +02:00
"Penguins", # too few comics
2012-12-04 07:02:40 +01:00
"PIGTIMES", # comic unavailable
"PS", # comic unavailable
2013-04-04 18:30:02 +02:00
"Radiowave", # too few comics
2013-03-26 17:33:15 +01:00
"RatchetAndSpin", # too few comics
2012-12-08 00:45:18 +01:00
"RichardsPoorAlmanac", # missing images
2013-07-09 22:21:12 +02:00
"SatchelandDuff", # too few comics
2012-12-04 07:02:40 +01:00
"SherpaAid", # comic unavailable
"Slowpoke", # comic moved
2013-04-04 18:30:02 +02:00
"SpaghettiSandwich", # too few comics
2012-12-04 07:02:40 +01:00
"SparComics", # comic unavailable
2012-12-13 21:05:27 +01:00
"SurvivingSingle", # comic unavailable
2013-11-12 18:33:14 +01:00
"TheBluckwells", # missing images
"TheConjurers", # too few comics
2013-04-04 18:30:02 +02:00
"TheDeadlys", # too few comics
2013-05-25 23:24:33 +02:00
"TheNursesLockerRoom", # too few comics
2013-04-28 19:58:38 +02:00
"Tomversation", # too few comics
2013-03-07 23:08:17 +01:00
"VoicesInTheDark", # too few comics
2013-02-23 09:07:44 +01:00
"WhatTheFrak", # too few comics
"ZeekyZebraandCompany", # too few comics
2012-11-29 06:46:58 +01:00
]
2013-02-12 21:53:57 +01:00
def handle_url(url, session, res):
"""Parse one search result page."""
print("Parsing", url, file=sys.stderr)
try:
2013-02-12 21:53:57 +01:00
data, baseUrl = getPageContent(url, session)
except IOError as msg:
print("ERROR:", msg, file=sys.stderr)
return
for match in url_matcher.finditer(data):
shortname = match.group(1)
name = unescape(match.group(2))
name = asciify(name.replace('&', 'And').replace('@', 'At'))
2012-12-12 17:41:29 +01:00
name = capfirst(name)
2012-11-29 06:46:58 +01:00
if name in exclude_comics:
continue
if contains_case_insensitive(res, name):
# we cannot handle two comics that only differ in case
2013-03-12 20:47:11 +01:00
print("INFO: skipping possible duplicate", repr(name), file=sys.stderr)
continue
res[name] = shortname
def get_results():
"""Parse all search result pages."""
# store info in a dictionary {name -> shortname}
res = {}
2013-02-12 21:53:57 +01:00
session = requests.Session()
handle_url('http://www.gocomics.com/features', session, res)
handle_url('http://www.gocomics.com/explore/editorial_list', session, res)
handle_url('http://www.gocomics.com/explore/sherpa_list', session, res)
2012-12-19 20:42:53 +01:00
save_result(res, json_file)
def print_results(args):
"""Print all comics that have at least the given number of minimum comic strips."""
2013-05-22 22:29:03 +02:00
min_comics, filename = args
with codecs.open(filename, 'a', 'utf-8') as fp:
for name, shortname in sorted(load_result(json_file).items()):
if name in exclude_comics:
continue
fp.write(u"add(%r, %r)\n" % (
str(truncate_name(name)), str(shortname))
)
if __name__ == '__main__':
if len(sys.argv) > 1:
print_results(sys.argv[1:])
else:
get_results()