From 62bcc5212e52788982d63fbb8b07de5a7dec85bd Mon Sep 17 00:00:00 2001 From: Tobias Gruetzmacher Date: Mon, 6 Jun 2022 00:52:33 +0200 Subject: [PATCH] Remove webtoons auto-updater for now --- dosagelib/plugins/webtoons.py | 6 ++---- scripts/generate_json.sh | 4 ++-- scripts/update_plugins.sh | 4 ++-- scripts/webtoons.py | 39 ----------------------------------- 4 files changed, 6 insertions(+), 47 deletions(-) delete mode 100755 scripts/webtoons.py diff --git a/dosagelib/plugins/webtoons.py b/dosagelib/plugins/webtoons.py index e9c9ccb9d..ed6e85a39 100644 --- a/dosagelib/plugins/webtoons.py +++ b/dosagelib/plugins/webtoons.py @@ -1,6 +1,6 @@ # SPDX-License-Identifier: MIT -# Copyright (C) 2019-2021 Tobias Gruetzmacher -# Copyright (C) 2019-2020 Daniel Ring +# Copyright (C) 2019-2022 Tobias Gruetzmacher +# Copyright (C) 2019-2022 Daniel Ring from ..scraper import _ParserScraper @@ -47,7 +47,6 @@ class WebToons(_ParserScraper): @classmethod def getmodules(cls): # noqa: Allowed to be long return ( - # START AUTOUPDATE cls('1000', 'action/one-thousand', 1217), cls('10thDimensionBoys', 'comedy/10th-dimension-boys', 71), cls('1111Animals', 'comedy/1111-animals', 437), @@ -450,5 +449,4 @@ class WebToons(_ParserScraper): cls('YunaAndKawachan', 'drama/yuna-and-kawachan', 1840), cls('ZeroGame', 'fantasy/zero-game', 1704), cls('ZomCom', 'challenge/zomcom', 70195), - # END AUTOUPDATE ) diff --git a/scripts/generate_json.sh b/scripts/generate_json.sh index 6ff7617c5..7ed034573 100755 --- a/scripts/generate_json.sh +++ b/scripts/generate_json.sh @@ -1,6 +1,6 @@ #!/bin/sh # Copyright (C) 2012-2014 Bastian Kleineidam -# Copyright (C) 2015-2020 Tobias Gruetzmacher +# Copyright (C) 2015-2022 Tobias Gruetzmacher # Copyright (C) 2019-2020 Daniel Ring set -e set -u @@ -9,7 +9,7 @@ d=$(dirname $0) if [ $# -ge 1 ]; then list="$*" else - list="arcamax comicfury comicgenesis comicskingdom creators gocomics keenspot tapastic webcomicfactory webtoons" + list="arcamax comicfury comicgenesis comicskingdom creators gocomics keenspot tapastic webcomicfactory" fi for script in $list; do echo "Executing ${script}.py" diff --git a/scripts/update_plugins.sh b/scripts/update_plugins.sh index 8ce0a9126..3c4c627c0 100755 --- a/scripts/update_plugins.sh +++ b/scripts/update_plugins.sh @@ -1,6 +1,6 @@ #!/bin/sh # Copyright (C) 2012-2014 Bastian Kleineidam -# Copyright (C) 2015-2020 Tobias Gruetzmacher +# Copyright (C) 2015-2022 Tobias Gruetzmacher # Copyright (C) 2019-2020 Daniel Ring set -e set -u @@ -11,7 +11,7 @@ d=$(dirname $0) if [ $# -ge 1 ]; then list="$*" else - list="arcamax comicfury comicgenesis comicskingdom creators gocomics keenspot tapastic webcomicfactory webtoons" + list="arcamax comicfury comicgenesis comicskingdom creators gocomics keenspot tapastic webcomicfactory" fi for script in $list; do target="${d}/../dosagelib/plugins/${script}.py" diff --git a/scripts/webtoons.py b/scripts/webtoons.py deleted file mode 100755 index 043082d00..000000000 --- a/scripts/webtoons.py +++ /dev/null @@ -1,39 +0,0 @@ -#!/usr/bin/env python3 -# SPDX-License-Identifier: MIT -# Copyright (C) 2019-2020 Tobias Gruetzmacher -# Copyright (C) 2019-2020 Daniel Ring -""" -Script to get a list of WebToons comics and save the info in a -JSON file for further processing. -""" -from urllib.parse import urlsplit, parse_qs - -from scriptutil import ComicListUpdater -from dosagelib.util import check_robotstxt - - -class WebToonsUpdater(ComicListUpdater): - def collect_results(self): - # Parse the comic list page - data = self.get_url('https://www.webtoons.com/en/dailySchedule') - - for comiclink in data.xpath('//a[contains(@class, "daily_card_item")]'): - comicurl = comiclink.attrib['href'] - name = comiclink.xpath('.//div[@class="info"]/p[@class="subj"]')[0].text - try: - check_robotstxt(comicurl, self.session) - except IOError as e: - print('[%s] INFO: robots.txt denied: %s' % (name, e)) - continue - - self.add_comic(name, comicurl) - - def get_entry(self, name, url): - shortName = name.replace(' ', '') - titleNum = int(parse_qs(urlsplit(url).query)['title_no'][0]) - url = url.rsplit('/', 1)[0].replace('https://www.webtoons.com/en/', '') - return u"cls('%s', '%s', %d)," % (shortName, url, titleNum) - - -if __name__ == '__main__': - WebToonsUpdater(__file__).run()