From 18f8e093a7f50a64ba29563023db4f9a990f21f2 Mon Sep 17 00:00:00 2001
From: Tobias Gruetzmacher <tobias-git@23.gs>
Date: Tue, 3 Dec 2019 23:35:41 +0100
Subject: [PATCH] Tests: Keep comics of the same module in the same process

This allows our host-based throttling to be effective and keeps
cross-process locks to a minimum.
---
 setup.cfg                     |  1 +
 tests/modules/check_comics.py | 19 +++++--------------
 tests/modules/conftest.py     | 18 +++++++++++++++++-
 3 files changed, 23 insertions(+), 15 deletions(-)

diff --git a/setup.cfg b/setup.cfg
index 2841dea4a..1f3ffedbb 100644
--- a/setup.cfg
+++ b/setup.cfg
@@ -55,6 +55,7 @@ bash =
     argcomplete
 test =
     pytest-cov
+    pytest-xdist
     responses
 
 [bdist_wheel]
diff --git a/tests/modules/check_comics.py b/tests/modules/check_comics.py
index 50e79fce9..b13506671 100644
--- a/tests/modules/check_comics.py
+++ b/tests/modules/check_comics.py
@@ -1,7 +1,7 @@
 # -*- coding: utf-8 -*-
 # Copyright (C) 2004-2008 Tristan Seligmann and Jonathan Jacobs
 # Copyright (C) 2012-2014 Bastian Kleineidam
-# Copyright (C) 2015-2018 Tobias Gruetzmacher
+# Copyright (C) 2015-2019 Tobias Gruetzmacher
 
 from __future__ import absolute_import, division, print_function
 
@@ -10,15 +10,10 @@ import multiprocessing
 from six.moves.urllib.parse import urlsplit
 
 
-def get_host(url):
-    """Get host part of URL."""
-    return urlsplit(url)[1].lower()
-
-
 # Dictionary with per-host locks.
 _locks = {}
 # Allowed number of connections per host
-MaxConnections = 4
+MaxConnections = 2
 # Maximum number of strips to get to test a comic
 MaxStrips = 5
 
@@ -30,16 +25,12 @@ def get_lock(host):
     return _locks[host]
 
 
-def test_comicmodule(tmpdir, scraperobj):
+def test_comicmodule(tmpdir, scraperobj, worker_id):
     '''Test a scraper. It must be able to traverse backward for at least 5
     strips from the start, and find strip images on at least 4 pages.'''
     # Limit number of connections to one host.
-    host = get_host(scraperobj.url)
-    try:
-        with get_lock(host):
-            _test_comic(str(tmpdir), scraperobj)
-    except OSError:
-        # interprocess lock not supported
+    host = urlsplit(scraperobj.url).hostname
+    with get_lock(host):
         _test_comic(str(tmpdir), scraperobj)
 
 
diff --git a/tests/modules/conftest.py b/tests/modules/conftest.py
index 9a3df9fbf..213cf99ce 100644
--- a/tests/modules/conftest.py
+++ b/tests/modules/conftest.py
@@ -1,7 +1,7 @@
 # -*- coding: utf-8 -*-
 # Copyright (C) 2004-2008 Tristan Seligmann and Jonathan Jacobs
 # Copyright (C) 2012-2014 Bastian Kleineidam
-# Copyright (C) 2015-2016 Tobias Gruetzmacher
+# Copyright (C) 2015-2019 Tobias Gruetzmacher
 
 from __future__ import absolute_import, division, print_function
 
@@ -9,6 +9,9 @@ import re
 import operator
 import os
 
+import pytest
+from xdist.dsession import LoadScopeScheduling
+
 from dosagelib import scraper
 
 
@@ -45,3 +48,16 @@ def pytest_generate_tests(metafunc):
     if 'scraperobj' in metafunc.fixturenames:
         metafunc.parametrize('scraperobj', get_test_scrapers(),
                              ids=operator.attrgetter('name'))
+
+
+class LoadModScheduling(LoadScopeScheduling):
+    """Implement load scheduling for comic modules. See xdist for details."""
+
+    def _split_scope(self, nodeid):
+        mod, test = nodeid.split("::", 1)
+        return mod + "::" + test.split("/", 1)[0]
+
+
+@pytest.mark.trylast
+def pytest_xdist_make_scheduler(config, log):
+    return LoadModScheduling(config, log)