From 809796f3d16db471257bbea7f69b3fcaeb9d87c9 Mon Sep 17 00:00:00 2001 From: "D. Moonfire" Date: Sun, 24 Sep 2023 00:29:30 -0500 Subject: [PATCH 01/12] Add ExorcismAcademy --- dosagelib/plugins/e.py | 24 +++++++++++++++++++++++- 1 file changed, 23 insertions(+), 1 deletion(-) diff --git a/dosagelib/plugins/e.py b/dosagelib/plugins/e.py index d423528dd..d4ca493df 100644 --- a/dosagelib/plugins/e.py +++ b/dosagelib/plugins/e.py @@ -4,7 +4,7 @@ # SPDX-FileCopyrightText: © 2015 Tobias Gruetzmacher # SPDX-FileCopyrightText: © 2019 Daniel Ring import os -from re import compile, IGNORECASE +from re import compile, sub, IGNORECASE from ..helpers import bounceStarter, indirectStarter from ..scraper import ParserScraper, _BasicScraper, _ParserScraper @@ -214,6 +214,28 @@ class Evon(WordPressScraper): adult = True +class ExorcismAcademy(ParserScraper): + url = 'https://ea.asmodrawscomics.com/' + stripUrl = url + 'comic/%s/' + firstStripUrl = stripUrl % 'title-page' + imageSearch = '//div[contains(@class, "webcomic-image")]//img[contains(@class, "size-full")]' + prevSearch = '//a[contains(@class, "previous-webcomic-link")]' + multipleImagesPerStrip = True + adult = True + + def namer(self, image_url, page_url): + def repl(m): + return "{0}-{1}".format(m.group(2).zfill(4), m.group(1)) + + indexes = tuple(image_url.rstrip('/').split('/')[-3:]) + day = sub(r'^(.+?)-?(?:Pg-(\d+))', repl, indexes[2]) + name = "{year}-{month}-{day}".format( + year = indexes[0], + month = indexes[1], + day = day) + return name + + class ExploitationNow(WordPressNavi): url = 'http://www.exploitationnow.com/' firstStripUrl = url + '2000-07-07/9' From 72b468015f61af6c261c58e2bdde1cb2acd8b971 Mon Sep 17 00:00:00 2001 From: "D. Moonfire" Date: Sun, 24 Sep 2023 00:27:48 -0500 Subject: [PATCH 02/12] Add CassiopeiaQuinn --- dosagelib/plugins/c.py | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/dosagelib/plugins/c.py b/dosagelib/plugins/c.py index ababfcb93..f3005fdfb 100644 --- a/dosagelib/plugins/c.py +++ b/dosagelib/plugins/c.py @@ -9,7 +9,7 @@ from typing import List from ..scraper import _BasicScraper, _ParserScraper, ParserScraper from ..helpers import bounceStarter, indirectStarter, joinPathPartsNamer from ..util import tagre -from .common import WordPressScraper, WordPressNavi, WordPressWebcomic +from .common import ComicControlScraper, WordPressScraper, WordPressNavi, WordPressWebcomic class CampComic(_ParserScraper): @@ -87,6 +87,11 @@ class CaseyAndAndy(_BasicScraper): help = 'Index format: number' +class CassiopeiaQuinn(ComicControlScraper): + url = 'https://www.cassiopeiaquinn.com/' + firstStripUrl = url + 'comic/the-prize-cover' + + class CasuallyKayla(_BasicScraper): url = 'http://casuallykayla.com/' stripUrl = url + '?p=%s' From cf63e39cebc903a7213aef305ba1871ffbdf145e Mon Sep 17 00:00:00 2001 From: Tobias Gruetzmacher Date: Sun, 8 Oct 2023 20:41:39 +0200 Subject: [PATCH 03/12] Fix CyanideAndHappiness (fixes #227) --- dosagelib/plugins/c.py | 34 +++++++++++----------------------- 1 file changed, 11 insertions(+), 23 deletions(-) diff --git a/dosagelib/plugins/c.py b/dosagelib/plugins/c.py index f3005fdfb..c596ede60 100644 --- a/dosagelib/plugins/c.py +++ b/dosagelib/plugins/c.py @@ -1,8 +1,8 @@ # SPDX-License-Identifier: MIT -# Copyright (C) 2004-2008 Tristan Seligmann and Jonathan Jacobs -# Copyright (C) 2012-2014 Bastian Kleineidam -# Copyright (C) 2015-2022 Tobias Gruetzmacher -# Copyright (C) 2019-2020 Daniel Ring +# SPDX-FileCopyrightText: © 2004 Tristan Seligmann and Jonathan Jacobs +# SPDX-FileCopyrightText: © 2012 Bastian Kleineidam +# SPDX-FileCopyrightText: © 2015 Tobias Gruetzmacher +# SPDX-FileCopyrightText: © 2019 Daniel Ring from re import compile, escape from typing import List @@ -460,25 +460,13 @@ class CutLoose(_ParserScraper): return '%s-%s-%s_%s' % (postDate[1], postDate[2], postDate[3], filename) -class CyanideAndHappiness(_BasicScraper): - url = 'https://explosm.net/comics/' - stripUrl = url + '%s/' - firstStripUrl = stripUrl % '15' - imageSearch = compile(tagre("img", "src", r'(.*files.explosm.net/[^/]+/[^"]+)', before="main-comic")) - prevSearch = compile(tagre("a", "href", r'(/comics/\d+/)', after="nav-previous")) - nextSearch = compile(tagre("a", "href", r"(/comics/\d+/)", after="nav-next")) - help = 'Index format: n (unpadded)' - - def shouldSkipUrl(self, url, data): - """Skip pages without images.""" - return "/comics/play-button.png" in data[0] - - def namer(self, image_url, page_url): - imgname = image_url.split('/')[-1] - # only get the first 100 chars for the image name - imgname = imgname[:100] - imgnum = page_url.split('/')[-2] - return '%s_%s' % (imgnum, imgname) +class CyanideAndHappiness(ParserScraper): + url = 'https://explosm.net/' + imageSearch = '//div[@id="comic"]//div[contains(@class,"ComicImage")]/span//img' + prevSearch = '//div[@type="comic"]//a[*[local-name()="svg" and @rotate="180deg"]]' + nextSearch = '//div[@type="comic"]//a[*[local-name()="svg" and @rotate="0deg"]]' + starter = bounceStarter + namer = joinPathPartsNamer((), range(-4, 0)) class CynWolf(_ParserScraper): From 77aba8adec5c72eeef07ac207b8e96116e78d65a Mon Sep 17 00:00:00 2001 From: Tobias Gruetzmacher Date: Wed, 11 Oct 2023 00:54:22 +0200 Subject: [PATCH 04/12] Remove certificate "pinning" for ComicsKingdom (fixes #291) They have switched to short-lived Google certificates, let's hope they have automated their certificate setup enough so that chain issues won't happen again... --- dosagelib/data/godaddy-bundle-g2-2031.pem | 51 ----------------------- dosagelib/plugins/comicskingdom.py | 8 ---- 2 files changed, 59 deletions(-) delete mode 100644 dosagelib/data/godaddy-bundle-g2-2031.pem diff --git a/dosagelib/data/godaddy-bundle-g2-2031.pem b/dosagelib/data/godaddy-bundle-g2-2031.pem deleted file mode 100644 index a83df0f00..000000000 --- a/dosagelib/data/godaddy-bundle-g2-2031.pem +++ /dev/null @@ -1,51 +0,0 @@ ------BEGIN CERTIFICATE----- -MIIE0DCCA7igAwIBAgIBBzANBgkqhkiG9w0BAQsFADCBgzELMAkGA1UEBhMCVVMx -EDAOBgNVBAgTB0FyaXpvbmExEzARBgNVBAcTClNjb3R0c2RhbGUxGjAYBgNVBAoT -EUdvRGFkZHkuY29tLCBJbmMuMTEwLwYDVQQDEyhHbyBEYWRkeSBSb290IENlcnRp -ZmljYXRlIEF1dGhvcml0eSAtIEcyMB4XDTExMDUwMzA3MDAwMFoXDTMxMDUwMzA3 -MDAwMFowgbQxCzAJBgNVBAYTAlVTMRAwDgYDVQQIEwdBcml6b25hMRMwEQYDVQQH -EwpTY290dHNkYWxlMRowGAYDVQQKExFHb0RhZGR5LmNvbSwgSW5jLjEtMCsGA1UE -CxMkaHR0cDovL2NlcnRzLmdvZGFkZHkuY29tL3JlcG9zaXRvcnkvMTMwMQYDVQQD -EypHbyBEYWRkeSBTZWN1cmUgQ2VydGlmaWNhdGUgQXV0aG9yaXR5IC0gRzIwggEi -MA0GCSqGSIb3DQEBAQUAA4IBDwAwggEKAoIBAQC54MsQ1K92vdSTYuswZLiBCGzD -BNliF44v/z5lz4/OYuY8UhzaFkVLVat4a2ODYpDOD2lsmcgaFItMzEUz6ojcnqOv -K/6AYZ15V8TPLvQ/MDxdR/yaFrzDN5ZBUY4RS1T4KL7QjL7wMDge87Am+GZHY23e -cSZHjzhHU9FGHbTj3ADqRay9vHHZqm8A29vNMDp5T19MR/gd71vCxJ1gO7GyQ5HY -pDNO6rPWJ0+tJYqlxvTV0KaudAVkV4i1RFXULSo6Pvi4vekyCgKUZMQWOlDxSq7n -eTOvDCAHf+jfBDnCaQJsY1L6d8EbyHSHyLmTGFBUNUtpTrw700kuH9zB0lL7AgMB -AAGjggEaMIIBFjAPBgNVHRMBAf8EBTADAQH/MA4GA1UdDwEB/wQEAwIBBjAdBgNV -HQ4EFgQUQMK9J47MNIMwojPX+2yz8LQsgM4wHwYDVR0jBBgwFoAUOpqFBxBnKLbv -9r0FQW4gwZTaD94wNAYIKwYBBQUHAQEEKDAmMCQGCCsGAQUFBzABhhhodHRwOi8v -b2NzcC5nb2RhZGR5LmNvbS8wNQYDVR0fBC4wLDAqoCigJoYkaHR0cDovL2NybC5n -b2RhZGR5LmNvbS9nZHJvb3QtZzIuY3JsMEYGA1UdIAQ/MD0wOwYEVR0gADAzMDEG -CCsGAQUFBwIBFiVodHRwczovL2NlcnRzLmdvZGFkZHkuY29tL3JlcG9zaXRvcnkv -MA0GCSqGSIb3DQEBCwUAA4IBAQAIfmyTEMg4uJapkEv/oV9PBO9sPpyIBslQj6Zz -91cxG7685C/b+LrTW+C05+Z5Yg4MotdqY3MxtfWoSKQ7CC2iXZDXtHwlTxFWMMS2 -RJ17LJ3lXubvDGGqv+QqG+6EnriDfcFDzkSnE3ANkR/0yBOtg2DZ2HKocyQetawi -DsoXiWJYRBuriSUBAA/NxBti21G00w9RKpv0vHP8ds42pM3Z2Czqrpv1KrKQ0U11 -GIo/ikGQI31bS/6kA1ibRrLDYGCD+H1QQc7CoZDDu+8CL9IVVO5EFdkKrqeKM+2x -LXY2JtwE65/3YR8V3Idv7kaWKK2hJn0KCacuBKONvPi8BDAB ------END CERTIFICATE----- ------BEGIN CERTIFICATE----- -MIIDxTCCAq2gAwIBAgIBADANBgkqhkiG9w0BAQsFADCBgzELMAkGA1UEBhMCVVMx -EDAOBgNVBAgTB0FyaXpvbmExEzARBgNVBAcTClNjb3R0c2RhbGUxGjAYBgNVBAoT -EUdvRGFkZHkuY29tLCBJbmMuMTEwLwYDVQQDEyhHbyBEYWRkeSBSb290IENlcnRp -ZmljYXRlIEF1dGhvcml0eSAtIEcyMB4XDTA5MDkwMTAwMDAwMFoXDTM3MTIzMTIz -NTk1OVowgYMxCzAJBgNVBAYTAlVTMRAwDgYDVQQIEwdBcml6b25hMRMwEQYDVQQH -EwpTY290dHNkYWxlMRowGAYDVQQKExFHb0RhZGR5LmNvbSwgSW5jLjExMC8GA1UE -AxMoR28gRGFkZHkgUm9vdCBDZXJ0aWZpY2F0ZSBBdXRob3JpdHkgLSBHMjCCASIw -DQYJKoZIhvcNAQEBBQADggEPADCCAQoCggEBAL9xYgjx+lk09xvJGKP3gElY6SKD -E6bFIEMBO4Tx5oVJnyfq9oQbTqC023CYxzIBsQU+B07u9PpPL1kwIuerGVZr4oAH -/PMWdYA5UXvl+TW2dE6pjYIT5LY/qQOD+qK+ihVqf94Lw7YZFAXK6sOoBJQ7Rnwy -DfMAZiLIjWltNowRGLfTshxgtDj6AozO091GB94KPutdfMh8+7ArU6SSYmlRJQVh -GkSBjCypQ5Yj36w6gZoOKcUcqeldHraenjAKOc7xiID7S13MMuyFYkMlNAJWJwGR -tDtwKj9useiciAF9n9T521NtYJ2/LOdYq7hfRvzOxBsDPAnrSTFcaUaz4EcCAwEA -AaNCMEAwDwYDVR0TAQH/BAUwAwEB/zAOBgNVHQ8BAf8EBAMCAQYwHQYDVR0OBBYE -FDqahQcQZyi27/a9BUFuIMGU2g/eMA0GCSqGSIb3DQEBCwUAA4IBAQCZ21151fmX -WWcDYfF+OwYxdS2hII5PZYe096acvNjpL9DbWu7PdIxztDhC2gV7+AJ1uP2lsdeu -9tfeE8tTEH6KRtGX+rcuKxGrkLAngPnon1rpN5+r5N9ss4UXnT3ZJE95kTXWXwTr -gIOrmgIttRD02JDHBHNA7XIloKmf7J6raBKZV8aPEjoJpL1E/QYVN8Gb5DKj7Tjo -2GTzLH4U/ALqn83/B2gX2yKQOC16jdFU8WnjXzPKej17CuPKf1855eJ1usV2GDPO -LPAvTK33sefOT6jEm0pUBsV/fdUID+Ic/n4XuKxe9tQWskMJDE32p2u0mYRlynqI -4uJEvlz36hz1 ------END CERTIFICATE----- diff --git a/dosagelib/plugins/comicskingdom.py b/dosagelib/plugins/comicskingdom.py index 5ced8fa49..818a37fa7 100644 --- a/dosagelib/plugins/comicskingdom.py +++ b/dosagelib/plugins/comicskingdom.py @@ -25,14 +25,6 @@ class ComicsKingdom(ParserScraper): if lang: self.lang = lang - # slightly iffy hack taken from certifi - # We need or own certificate bundle since ComicsKingdom screws up their - # TLS setup from time to time, this should "fix" it) - self.cert_ctx = as_file(files('dosagelib.data') / 'godaddy-bundle-g2-2031.pem') - self.session.add_host_options('comicskingdom.com', { - 'verify': str(self.cert_ctx.__enter__()), - }) - @classmethod def getmodules(cls): # noqa: CFQ001 return ( From bafe3ecb316b8c63bc2ad3faab087e1e7fbaace2 Mon Sep 17 00:00:00 2001 From: "D. Moonfire" Date: Sun, 24 Sep 2023 00:23:37 -0500 Subject: [PATCH 05/12] Add Alfie --- dosagelib/plugins/a.py | 30 +++++++++++++++++++++++++++++- 1 file changed, 29 insertions(+), 1 deletion(-) diff --git a/dosagelib/plugins/a.py b/dosagelib/plugins/a.py index 32a5b42ce..e3fc866ca 100644 --- a/dosagelib/plugins/a.py +++ b/dosagelib/plugins/a.py @@ -3,7 +3,7 @@ # Copyright (C) 2012-2014 Bastian Kleineidam # Copyright (C) 2015-2022 Tobias Gruetzmacher # Copyright (C) 2019-2020 Daniel Ring -from re import compile, escape, MULTILINE +from re import compile, escape, sub, MULTILINE from ..util import tagre from ..scraper import BasicScraper, ParserScraper, _BasicScraper, _ParserScraper @@ -136,6 +136,34 @@ class ALessonIsLearned(_BasicScraper): help = 'Index format: nnn' +class Alfie(WordPressScraper): + url = 'https://buttsmithy.com/' + stripUrl = url + 'archives/comic/%s' + firstStripUrl = stripUrl % 'p1' + adult = True + starter = bounceStarter + + def namer(self, image_url, page_url): + def repl(m): + return "{0}".format(m.group(1).zfill(4)) + + name = sub('^p-?(\d+)', repl, page_url.split('/')[-1]) + + # Some of the first 1k pages were inconsistently named. + renames = {"/comic/p145": "0145-1", "/comic/p-145": "0145-2", + "/comic/268": "0268", "/comic/1132": "0313", + "/comic/1169": "0319", "/comic/1186": "0324", + "/comic/1404": "0378", "/comic/0338-2": "0339", + "/comic/0369-2": "0469", "/comic/2080": "0517", + "/comic/o-525": "0525", "/comic/p-361": "0553", + "/comic/p-668-2": "0678", "/comic/p-670-2": "0670", + "/comic/p-679-2": "0690", "/comic/3140": "0805"} + for rename in renames: + if rename in page_url: + name = renames[rename] + return name + + class Alice(WordPressScraper): url = 'https://web.archive.org/web/20210115132313/http://www.alicecomics.com/' latestSearch = '//a[text()="Latest Alice!"]' From 25549a36ab91b2ab9c1568619f37e460162c0af5 Mon Sep 17 00:00:00 2001 From: Tobias Gruetzmacher Date: Thu, 26 Oct 2023 22:25:25 +0200 Subject: [PATCH 06/12] Jenkins: Use modern coverage step --- Jenkinsfile | 8 +++----- tests/modules/Jenkinsfile | 8 +++----- 2 files changed, 6 insertions(+), 10 deletions(-) diff --git a/Jenkinsfile b/Jenkinsfile index 776b5f612..4a2ee6258 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -55,11 +55,9 @@ pys.each { py -> def buildVer = findFiles(glob: 'dist/*.tar.gz')[0].name.replaceFirst(/\.tar\.gz$/, '') currentBuild.description = buildVer - publishCoverage calculateDiffForChangeRequests: true, - sourceFileResolver: sourceFiles('STORE_LAST_BUILD'), - adapters: [ - coberturaAdapter('.tox/reports/*/coverage.xml') - ] + recordCoverage sourceCodeEncoding: 'UTF-8', tools: [ + [parser: 'COBERTURA', pattern: '.tox/reports/*/coverage.xml'] + ] recordIssues sourceCodeEncoding: 'UTF-8', referenceJobName: 'dosage/master', diff --git a/tests/modules/Jenkinsfile b/tests/modules/Jenkinsfile index 7fbe276b5..4f157ebc1 100644 --- a/tests/modules/Jenkinsfile +++ b/tests/modules/Jenkinsfile @@ -16,11 +16,9 @@ node { stage('Report') { junit 'junit.xml' - publishCoverage calculateDiffForChangeRequests: true, - sourceFileResolver: sourceFiles('STORE_LAST_BUILD'), - adapters: [ - coberturaAdapter('coverage.xml') - ] + recordCoverage sourceCodeEncoding: 'UTF-8', tools: [ + [parser: 'COBERTURA', pattern: 'coverage.xml'] + ] } stage('Allure Report') { From e64e8c7d53b1ed57db04b5e0d55a3debd7aa2e99 Mon Sep 17 00:00:00 2001 From: Tobias Gruetzmacher Date: Sun, 29 Oct 2023 19:05:20 +0100 Subject: [PATCH 07/12] Simplify update checker Motivated by the distutils removal, this replaces version parsing by a "good enough" version comparison algorithm... --- dosagelib/cmd.py | 19 +++++++---------- dosagelib/updater.py | 49 ++++++++++++++++++++++++-------------------- pyproject.toml | 4 ++-- tests/test_dosage.py | 10 ++++++++- 4 files changed, 45 insertions(+), 37 deletions(-) diff --git a/dosagelib/cmd.py b/dosagelib/cmd.py index b6c766b7d..134d3749f 100644 --- a/dosagelib/cmd.py +++ b/dosagelib/cmd.py @@ -1,7 +1,7 @@ # SPDX-License-Identifier: MIT -# Copyright (C) 2004-2008 Tristan Seligmann and Jonathan Jacobs -# Copyright (C) 2012-2014 Bastian Kleineidam -# Copyright (C) 2015-2022 Tobias Gruetzmacher +# SPDX-FileCopyrightText: © 2004 Tristan Seligmann and Jonathan Jacobs +# SPDX-FileCopyrightText: © 2012 Bastian Kleineidam +# SPDX-FileCopyrightText: © 2015 Tobias Gruetzmacher import argparse import os import platform @@ -124,8 +124,8 @@ def display_version(verbose): if verbose: # search for updates from .updater import check_update - result, value = check_update() - if result: + try: + value = check_update() if value: version, url = value if url is None: @@ -139,13 +139,8 @@ def display_version(verbose): attrs = {'version': version, 'app': AppName, 'url': url, 'currentversion': __version__} print(text % attrs) - else: - if value is None: - value = 'invalid update file syntax' - text = ('An error occured while checking for an ' - 'update of %(app)s: %(error)s.') - attrs = {'error': value, 'app': AppName} - print(text % attrs) + except (IOError, KeyError) as err: + print(f'An error occured while checking for an update of {AppName}: {err!r}') return 0 diff --git a/dosagelib/updater.py b/dosagelib/updater.py index 99218489d..3ef4ac512 100644 --- a/dosagelib/updater.py +++ b/dosagelib/updater.py @@ -1,10 +1,10 @@ # SPDX-License-Identifier: MIT -# Copyright (C) 2004-2008 Tristan Seligmann and Jonathan Jacobs -# Copyright (C) 2012-2014 Bastian Kleineidam -# Copyright (C) 2015-2020 Tobias Gruetzmacher +# SPDX-FileCopyrightText: © 2004 Tristan Seligmann and Jonathan Jacobs +# SPDX-FileCopyrightText: © 2012 Bastian Kleineidam +# SPDX-FileCopyrightText: © 2015 Tobias Gruetzmacher import os - -from distutils.version import LooseVersion +import re +from typing import Any import dosagelib from . import http @@ -18,35 +18,34 @@ EXTPRIO = { } -def check_update(): +def check_update() -> None | tuple[str, None | str]: """Return the following values: - (False, errmsg) - online version could not be determined - (True, None) - user has newest version - (True, (version, url string)) - update available - (True, (version, None)) - current version is newer than online version + throws exception - online version could not be determined + None - user has newest version + (version, url string) - update available + (version, None) - current version is newer than online version """ version, value = get_online_version() - if version is None: - # value is an error message - return False, value if version == dosagelib.__version__: # user has newest version - return True, None + return None if is_newer_version(version): # value is an URL linking to the update package - return True, (version, value) + return (version, value) # user is running a local or development version - return True, (version, None) + return (version, None) -def asset_key(asset): +def asset_key(asset: dict[str, Any]) -> int: return EXTPRIO.get(os.path.splitext(asset['browser_download_url'])[1], 99) -def get_online_version(): +def get_online_version() -> tuple[str, None | str]: """Download update info and parse it.""" - page = http.default_session.get(UPDATE_URL).json() - version = page.get('tag_name', None) + response = http.default_session.get(UPDATE_URL) + response.raise_for_status() + page = response.json() + version = page['tag_name'] url = None try: @@ -58,6 +57,12 @@ def get_online_version(): return version, url -def is_newer_version(version): +def version_nums(ver: str) -> tuple[int, ...]: + """Extract all numeric "sub-parts" of a version string. Not very exact, but + works for our use case.""" + return tuple(int(s) for s in re.split(r'\D+', ver + '0')) + + +def is_newer_version(version) -> bool: """Check if given version is newer than current version.""" - return LooseVersion(version) > LooseVersion(dosagelib.__version__) + return version_nums(version) > version_nums(dosagelib.__version__) diff --git a/pyproject.toml b/pyproject.toml index 923f77874..af7c92ea0 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -52,16 +52,16 @@ dev = [ "pytest-cov", "pytest-xdist", "responses", - "setup-cfg-fmt", ] lint = [ - "flake8<6", + "flake8~=6.0", "flake8-2020", "flake8-breakpoint", "flake8-bugbear", "flake8-coding", "flake8-commas", "flake8-comprehensions", + "flake8-deprecated", "flake8-eradicate", "flake8-fixme", "flake8-functions", diff --git a/tests/test_dosage.py b/tests/test_dosage.py index 9b810a79e..b0c4d4179 100644 --- a/tests/test_dosage.py +++ b/tests/test_dosage.py @@ -82,7 +82,15 @@ class TestDosage: json={}) cmd_ok('--version', '-v') out = capfd.readouterr().out - assert 'invalid update file' in out + assert 'KeyError' in out + + @responses.activate + def test_update_rate_limit(self, capfd): + responses.add(responses.GET, re.compile(r'https://api\.github\.com/'), + status=403) + cmd_ok('--version', '-v') + out = capfd.readouterr().out + assert 'HTTPError' in out def test_display_help(self): for option in ("-h", "--help"): From e0cf40a3a6abe5b66fbdaefa2f85970538b5359c Mon Sep 17 00:00:00 2001 From: Tobias Gruetzmacher Date: Mon, 30 Oct 2023 08:37:49 +0100 Subject: [PATCH 08/12] Fix type annotations for older Python versions --- dosagelib/updater.py | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/dosagelib/updater.py b/dosagelib/updater.py index 3ef4ac512..cc7b84b8b 100644 --- a/dosagelib/updater.py +++ b/dosagelib/updater.py @@ -4,7 +4,7 @@ # SPDX-FileCopyrightText: © 2015 Tobias Gruetzmacher import os import re -from typing import Any +from typing import Any, Dict, Optional, Tuple import dosagelib from . import http @@ -18,7 +18,7 @@ EXTPRIO = { } -def check_update() -> None | tuple[str, None | str]: +def check_update() -> Optional[Tuple[str, Optional[str]]]: """Return the following values: throws exception - online version could not be determined None - user has newest version @@ -36,11 +36,11 @@ def check_update() -> None | tuple[str, None | str]: return (version, None) -def asset_key(asset: dict[str, Any]) -> int: +def asset_key(asset: Dict[str, Any]) -> int: return EXTPRIO.get(os.path.splitext(asset['browser_download_url'])[1], 99) -def get_online_version() -> tuple[str, None | str]: +def get_online_version() -> Tuple[str, Optional[str]]: """Download update info and parse it.""" response = http.default_session.get(UPDATE_URL) response.raise_for_status() @@ -57,7 +57,7 @@ def get_online_version() -> tuple[str, None | str]: return version, url -def version_nums(ver: str) -> tuple[int, ...]: +def version_nums(ver: str) -> Tuple[int, ...]: """Extract all numeric "sub-parts" of a version string. Not very exact, but works for our use case.""" return tuple(int(s) for s in re.split(r'\D+', ver + '0')) From 89afdb37d74c64569e60c11ac2a9fdbb8a1fc0ff Mon Sep 17 00:00:00 2001 From: "D. Moonfire" Date: Sun, 5 Nov 2023 11:06:00 -0600 Subject: [PATCH 09/12] Added Nix build files --- flake.lock | 61 ++++++++++++++++++++++++++++++++++++++++++++++++++++++ flake.nix | 29 ++++++++++++++++++++++++++ 2 files changed, 90 insertions(+) create mode 100644 flake.lock create mode 100644 flake.nix diff --git a/flake.lock b/flake.lock new file mode 100644 index 000000000..db382ba34 --- /dev/null +++ b/flake.lock @@ -0,0 +1,61 @@ +{ + "nodes": { + "flake-utils": { + "inputs": { + "systems": "systems" + }, + "locked": { + "lastModified": 1694529238, + "narHash": "sha256-zsNZZGTGnMOf9YpHKJqMSsa0dXbfmxeoJ7xHlrt+xmY=", + "owner": "numtide", + "repo": "flake-utils", + "rev": "ff7b65b44d01cf9ba6a71320833626af21126384", + "type": "github" + }, + "original": { + "owner": "numtide", + "repo": "flake-utils", + "type": "github" + } + }, + "nixpkgs": { + "locked": { + "lastModified": 1695360818, + "narHash": "sha256-JlkN3R/SSoMTa+CasbxS1gq+GpGxXQlNZRUh9+LIy/0=", + "owner": "nixos", + "repo": "nixpkgs", + "rev": "e35dcc04a3853da485a396bdd332217d0ac9054f", + "type": "github" + }, + "original": { + "owner": "nixos", + "ref": "nixos-unstable", + "repo": "nixpkgs", + "type": "github" + } + }, + "root": { + "inputs": { + "flake-utils": "flake-utils", + "nixpkgs": "nixpkgs" + } + }, + "systems": { + "locked": { + "lastModified": 1681028828, + "narHash": "sha256-Vy1rq5AaRuLzOxct8nz4T6wlgyUR7zLU309k9mBC768=", + "owner": "nix-systems", + "repo": "default", + "rev": "da67096a3b9bf56a91d16901293e51ba5b49a27e", + "type": "github" + }, + "original": { + "owner": "nix-systems", + "repo": "default", + "type": "github" + } + } + }, + "root": "root", + "version": 7 +} diff --git a/flake.nix b/flake.nix new file mode 100644 index 000000000..674bec1f8 --- /dev/null +++ b/flake.nix @@ -0,0 +1,29 @@ +{ + inputs = { + nixpkgs.url = "github:nixos/nixpkgs/nixos-unstable"; + flake-utils.url = "github:numtide/flake-utils"; + }; + + outputs = { + self, + nixpkgs, + flake-utils, + ... + }: let + pythonVersion = "python39"; + in + flake-utils.lib.eachDefaultSystem ( + system: let + pkgs = nixpkgs.legacyPackages.${system}; + in rec + { + devShells.default = pkgs.mkShell { + buildInputs = [ + pkgs.cowsay + pkgs.alejandra + pkgs.python310Full + ]; + }; + } + ); +} From 11cc57c19079e535f629c31aa27610dc83609b83 Mon Sep 17 00:00:00 2001 From: Tobias Gruetzmacher Date: Fri, 27 Oct 2023 01:38:58 +0200 Subject: [PATCH 10/12] Build with Python 3.12 --- .github/workflows/ci.yaml | 4 ++-- .github/workflows/pages.yml | 2 +- Jenkinsfile | 9 +++++---- pyproject.toml | 1 + tests/modules/Jenkinsfile | 2 +- tox.ini | 5 +++-- 6 files changed, 13 insertions(+), 10 deletions(-) diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml index 7a0653a5e..c4b954a61 100644 --- a/.github/workflows/ci.yaml +++ b/.github/workflows/ci.yaml @@ -6,14 +6,14 @@ on: - pull_request env: - DEFAULT_PYTHON: '3.11' + DEFAULT_PYTHON: '3.12' jobs: tests: runs-on: ubuntu-latest strategy: matrix: - python-version: ["3.7", "3.8", "3.9", "3.10", "3.11"] + python-version: ["3.7", "3.8", "3.9", "3.10", "3.11", "3.12"] steps: - uses: actions/checkout@v4 diff --git a/.github/workflows/pages.yml b/.github/workflows/pages.yml index b169ebf9b..ff96512be 100644 --- a/.github/workflows/pages.yml +++ b/.github/workflows/pages.yml @@ -20,7 +20,7 @@ jobs: - name: Set up Python uses: actions/setup-python@v4 with: - python-version: '3.11' + python-version: '3.12' - name: Build run: | diff --git a/Jenkinsfile b/Jenkinsfile index 4a2ee6258..db7d2c10c 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -1,9 +1,10 @@ def pys = [ - [name: 'Python 3.11', docker: '3.11-bookworm', tox:'py311,flake8', main: true], + [name: 'Python 3.12', docker: '3.12-bookworm', tox:'py312,flake8', main: true], + [name: 'Python 3.11', docker: '3.11-bookworm', tox:'py311', main: false], [name: 'Python 3.10', docker: '3.10-bookworm', tox:'py310', main: false], - [name: 'Python 3.9', docker: '3.9-bookworm', tox:'py39', main: false], - [name: 'Python 3.8', docker: '3.8-bookworm', tox:'py38', main: false], - [name: 'Python 3.7', docker: '3.7-bookworm', tox:'py37', main: false], + [name: 'Python 3.9', docker: '3.9-bookworm', tox:'py39', main: false], + [name: 'Python 3.8', docker: '3.8-bookworm', tox:'py38', main: false], + [name: 'Python 3.7', docker: '3.7-bookworm', tox:'py37', main: false], ] properties([ diff --git a/pyproject.toml b/pyproject.toml index af7c92ea0..7b92c58b8 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -22,6 +22,7 @@ classifiers = [ "Programming Language :: Python :: 3.9", "Programming Language :: Python :: 3.10", "Programming Language :: Python :: 3.11", + "Programming Language :: Python :: 3.12", "Topic :: Internet :: WWW/HTTP", "Topic :: Multimedia :: Graphics", ] diff --git a/tests/modules/Jenkinsfile b/tests/modules/Jenkinsfile index 4f157ebc1..8fba07697 100644 --- a/tests/modules/Jenkinsfile +++ b/tests/modules/Jenkinsfile @@ -9,7 +9,7 @@ node { stage ('Run tests') { timeout(time: 12, unit: 'HOURS') { withCredentials([string(credentialsId: 'proxymap', variable: 'PROXYMAP')]) { - sh 'podman run --rm -v $PWD:/work --userns=keep-id docker.io/python:3.11-bookworm /work/tests/modules/testall.sh' + sh 'podman run --rm -v $PWD:/work --userns=keep-id docker.io/python:3.12-bookworm /work/tests/modules/testall.sh' } } } diff --git a/tox.ini b/tox.ini index 161f868bc..27eed7f37 100644 --- a/tox.ini +++ b/tox.ini @@ -1,5 +1,5 @@ [tox] -envlist = py37, py38, py39, py310, py311, flake8 +envlist = py37, py38, py39, py310, py311, py312, flake8 isolated_build = True [gh-actions] @@ -8,7 +8,8 @@ python = 3.8: py38 3.9: py39 3.10: py310 - 3.11: py311, flake8 + 3.11: py311 + 3.12: py312, flake8 [testenv] commands = From bff9a86d81fe46cc2798f8e65c879afd1aa25539 Mon Sep 17 00:00:00 2001 From: Tobias Gruetzmacher Date: Sun, 19 Nov 2023 22:15:24 +0100 Subject: [PATCH 11/12] Improve shell completion support --- dosagelib/cmd.py | 32 ++++++++++++++++++++++---------- 1 file changed, 22 insertions(+), 10 deletions(-) diff --git a/dosagelib/cmd.py b/dosagelib/cmd.py index 134d3749f..86a16d712 100644 --- a/dosagelib/cmd.py +++ b/dosagelib/cmd.py @@ -2,9 +2,15 @@ # SPDX-FileCopyrightText: © 2004 Tristan Seligmann and Jonathan Jacobs # SPDX-FileCopyrightText: © 2012 Bastian Kleineidam # SPDX-FileCopyrightText: © 2015 Tobias Gruetzmacher +# PYTHON_ARGCOMPLETE_OK +from __future__ import annotations + import argparse +import contextlib +import importlib import os import platform +from collections.abc import Iterable from platformdirs import PlatformDirs @@ -18,7 +24,7 @@ from .util import internal_error, strlimit class ArgumentParser(argparse.ArgumentParser): """Custom argument parser.""" - def print_help(self, file=None): + def print_help(self, file=None) -> None: """Paginate help message on TTYs.""" with out.pager(): out.info(self.format_help()) @@ -46,7 +52,7 @@ User plugin directory: {user_plugin_path} """ -def setup_options(): +def setup_options() -> ArgumentParser: """Construct option parser. @return: new option parser @rtype argparse.ArgumentParser @@ -65,8 +71,8 @@ def setup_options(): help='traverse and retrieve all comic strips') parser.add_argument('-c', '--continue', action='store_true', dest='cont', help='traverse and retrieve comic strips until an existing one is found') - parser.add_argument('-b', '--basepath', action='store', default='Comics', - metavar='PATH', + basepath_opt = parser.add_argument('-b', '--basepath', action='store', + default='Comics', metavar='PATH', help='set the path to create invidivual comic directories in, default is Comics') parser.add_argument('--baseurl', action='store', metavar='PATH', help='the base URL of your comics directory (for RSS, HTML, etc.);' @@ -103,16 +109,22 @@ def setup_options(): # are not "real" (moved & removed) parser.add_argument('--list-all', action='store_true', help=argparse.SUPPRESS) - parser.add_argument('comic', nargs='*', + comic_arg = parser.add_argument('comic', nargs='*', help='comic module name (including case insensitive substrings)') - try: - import argcomplete - argcomplete.autocomplete(parser) - except ImportError: - pass + comic_arg.completer = scraper_completion + with contextlib.suppress(ImportError): + completers = importlib.import_module('argcomplete.completers') + basepath_opt.completer = completers.DirectoriesCompleter() + importlib.import_module('argcomplete').autocomplete(parser) return parser +def scraper_completion(**kwargs) -> Iterable[str]: + """Completion helper for argcomplete.""" + scrapercache.adddir(user_plugin_path) + return (comic.name for comic in scrapercache.all()) + + def display_version(verbose): """Display application name, version, copyright and license.""" print(configuration.App) From 98887d246a711f612de77e197d7a7508e31625e3 Mon Sep 17 00:00:00 2001 From: Tobias Gruetzmacher Date: Sun, 19 Nov 2023 22:41:45 +0100 Subject: [PATCH 12/12] Housekeeping: Increase minimum required setuptools version This is probably of little consequence since most users use the newest setuptools version anyways... --- pyproject.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pyproject.toml b/pyproject.toml index 7b92c58b8..c5217a4c0 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,5 +1,5 @@ [build-system] -requires = ["setuptools>=61.2", "setuptools_scm>=6.2"] +requires = ["setuptools>=66.0", "setuptools_scm>=7.1"] build-backend = "setuptools.build_meta" [project]