diff --git a/scripts/scriptutil.py b/scripts/scriptutil.py index 051b14cd6..9dcfde477 100644 --- a/scripts/scriptutil.py +++ b/scripts/scriptutil.py @@ -1,6 +1,7 @@ # Copyright (C) 2012-2013 Bastian Kleineidam import re import json +from dosagelib.util import unescape, unquote, asciify def contains_case_insensitive(adict, akey): """Check if key is in adict. The search is case insensitive.""" @@ -46,3 +47,20 @@ def load_result(json_file): def truncate_name(text): """Ensure the comic name does not exceed 100 characters.""" return text[:100] + + +def format_name(text): + """Format a comic name.""" + name = unescape(text) + name = asciify(name.replace('&', 'And').replace('@', 'At')) + name = capfirst(name) + return name + + +def format_description(text): + """Format a comic description.""" + desc = remove_html_tags(text) + desc = unescape(desc) + desc = unquote(desc) + desc = compact_whitespace(desc).strip() + return desc diff --git a/scripts/smackjeeves.py b/scripts/smackjeeves.py index 796eec30d..345eee138 100755 --- a/scripts/smackjeeves.py +++ b/scripts/smackjeeves.py @@ -10,9 +10,9 @@ import os import urlparse import requests sys.path.append(os.path.join(os.path.dirname(__file__), "..")) -from dosagelib.util import getPageContent, asciify, unescape, tagre, unquote -from dosagelib.scraper import get_scrapers -from scriptutil import contains_case_insensitive, remove_html_tags, capfirst, compact_whitespace, save_result, load_result, truncate_name +from dosagelib.util import getPageContent, tagre +from dosagelib.scraper import get_scraperclasses +from scriptutil import contains_case_insensitive, save_result, load_result, truncate_name, format_name, format_description json_file = __file__.replace(".py", ".json") @@ -225,9 +225,7 @@ def handle_url(url, session, res): for match in page_matcher.finditer(data): page_url = match.group(1) page_url = urlparse.urljoin(url, page_url) - name = unescape(match.group(2)) - name = asciify(name.replace('&', 'And').replace('@', 'At')) - name = capfirst(name) + name = format_name(match.group(2)) if name in exclude_comics: continue if contains_case_insensitive(res, name): @@ -259,10 +257,7 @@ def handle_url(url, session, res): if not mo: print("ERROR matching comic description:", repr(data2[end:end+300]), file=sys.stderr) continue - desc = remove_html_tags(mo.group(1)) - desc = unescape(desc) - desc = unquote(desc) - desc = compact_whitespace(desc).strip() + desc = format_description(mo.group(1)) # search for adult flag adult = adult_matcher.search(data2[end:]) bounce = name not in repeat_comics