dmoonfire/dosage
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Activity

Refactor comic module test.

Browse source 
All those create-classes-on-the-fly games make my head hurt ;)
This commit is contained in:
Tobias Gruetzmacher 2015-07-17 23:33:25 +02:00
parent 68d4dd463a
commit f0e61a614f
1 changed files with 82 additions and 114 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

116
tests/test_comics.py
View file

@ -1,16 +1,17 @@
# -*- coding: iso-8859-1 -*-
# -*- coding: utf-8 -*-
# Copyright (C) 2004-2005 Tristan Seligmann and Jonathan Jacobs
# Copyright (C) 2012-2014 Bastian Kleineidam
# Copyright (C) 2015 Tobias Gruetzmacher
import tempfile
import shutil
import re
import os
import multiprocessing
import pytest
try:
from urllib.parse import urlsplit
except ImportError:
from urlparse import urlsplit
from unittest import TestCase
from dosagelib import scraper
@ -30,51 +31,35 @@ def get_lock(host):
_locks[host] = multiprocessing.BoundedSemaphore(MaxConnections)
return _locks[host]
@pytest.yield_fixture
def tmpdir():
tmpdir = tempfile.mkdtemp()
yield tmpdir
shutil.rmtree(tmpdir)
class _ComicTester(TestCase):
"""Basic comic test class."""
scraperclass=None
def setUp(self):
if self.scraperclass is not None:
self.name = self.scraperclass.getName()
self.url = self.scraperclass.starter()
# create a temporary directory for images
self.tmpdir = tempfile.mkdtemp()
else:
self.tmpdir = None
def tearDown(self):
if self.tmpdir is not None:
shutil.rmtree(self.tmpdir)
def get_saved_images(self, filtertxt=False):
def get_saved_images(tmpdir, scraper, filtertxt=False):
"""Get saved images."""
dirs = tuple(self.name.split('/'))
files = os.listdir(os.path.join(self.tmpdir, *dirs))
dirs = tuple(scraper.getName().split('/'))
files = os.listdir(os.path.join(tmpdir, *dirs))
if filtertxt:
files = [x for x in files if not x.endswith(".txt")]
return files
def test_comic(self):
if self.scraperclass is None:
# only run subclasses
import pytest
pytest.skip("base class")
def test_comicmodule(tmpdir, scraperclass):
# Test a scraper. It must be able to traverse backward for
# at least 5 strips from the start, and find strip images
# on at least 4 pages.
scraperobj = self.scraperclass()
scraperobj = scraperclass()
# Limit number of connections to one host.
host = get_host(scraperobj.url)
try:
with get_lock(host):
self._test_comic(scraperobj)
_test_comic(tmpdir, scraperobj)
except OSError:
# interprocess lock not supported
self._test_comic(scraperobj)
_test_comic(tmpdir, scraperobj)
def _test_comic(self, scraperobj):
def _test_comic(tmpdir, scraperobj):
num_strips = 0
max_strips = 5
strip = None
@ -82,81 +67,64 @@ class _ComicTester(TestCase):
images = []
for image in strip.getImages():
images.append(image.url)
self.save(image)
self.check(images, 'failed to find images at %s' % strip.stripUrl)
if not self.scraperclass.multipleImagesPerStrip:
self.check(len(images) == 1, 'found more than 1 image at %s: %s' % (strip.stripUrl, images))
if num_strips > 0 and self.scraperclass.prevUrlMatchesStripUrl:
self.check_stripurl(strip)
image.save(tmpdir)
assert images, 'failed to find images at %s' % strip.stripUrl
if not scraperobj.multipleImagesPerStrip:
assert len(images) == 1, 'found more than 1 image at %s: %s' % (strip.stripUrl, images)
if num_strips > 0 and scraperobj.prevUrlMatchesStripUrl:
check_stripurl(strip, scraperobj)
num_strips += 1
if self.scraperclass.prevSearch and not scraperobj.hitFirstStripUrl:
if scraperobj.prevSearch and not scraperobj.hitFirstStripUrl:
# check strips
num_strips_expected = max_strips - len(scraperobj.skippedUrls)
msg = 'Traversed %d strips instead of %d.' % (num_strips, num_strips_expected)
if strip:
msg += " Check the prevSearch pattern at %s" % strip.stripUrl
self.check(num_strips == num_strips_expected, msg)
assert num_strips == num_strips_expected, msg
# check images
if strip:
self.check_scraperesult(num_strips_expected, strip, scraperobj)
check_scraperesult(tmpdir, num_strips_expected, strip, scraperobj)
def check_scraperesult(self, num_images_expected, strip, scraperobj):
def check_scraperesult(tmpdir, num_images_expected, strip, scraperobj):
# Check that exactly or for multiple pages at least num_strips images are saved.
# This checks saved files, ie. it detects duplicate filenames.
saved_images = self.get_saved_images(filtertxt=bool(scraperobj.textSearch))
saved_images = get_saved_images(tmpdir, scraperobj, filtertxt=bool(scraperobj.textSearch))
num_images = len(saved_images)
# subtract the number of skipped URLs with no image from the expected image number
attrs = (num_images, saved_images, num_images_expected, self.tmpdir)
if self.scraperclass.multipleImagesPerStrip:
self.check(num_images >= num_images_expected, 'saved %d %s instead of at least %d images in %s' % attrs)
attrs = (num_images, saved_images, num_images_expected, tmpdir)
if scraperobj.multipleImagesPerStrip:
assert num_images >= num_images_expected, 'saved %d %s instead of at least %d images in %s' % attrs
else:
self.check(num_images == num_images_expected, 'saved %d %s instead of %d images in %s' % attrs)
assert num_images == num_images_expected, 'saved %d %s instead of %d images in %s' % attrs
def check_stripurl(self, strip):
if not self.scraperclass.stripUrl:
def check_stripurl(strip, scraperobj):
if not scraperobj.stripUrl:
# no indexing support
return
# test that the stripUrl regex matches the retrieved strip URL
urlmatch = re.escape(self.scraperclass.stripUrl)
urlmatch = re.escape(scraperobj.stripUrl)
urlmatch = urlmatch.replace(r"\%s", r".+")
urlmatch = "^%s$" % urlmatch
ro = re.compile(urlmatch)
mo = ro.search(strip.stripUrl)
self.check(mo is not None, 'strip URL %r does not match stripUrl pattern %s' % (strip.stripUrl, urlmatch))
assert mo is not None, 'strip URL %r does not match stripUrl pattern %s' % (strip.stripUrl, urlmatch)
def save(self, image):
try:
image.save(self.tmpdir)
except Exception as msg:
self.check(False, 'could not save %s at %s to %s: %s' % (image.url, image.referrer, self.tmpdir, msg))
def check(self, condition, msg):
self.assertTrue(condition, "%s %s %s" % (self.name, self.url, msg))
def make_comic_tester(name, **kwargs):
"""Create and return a _ComicTester class with given name and attributes."""
return type(name, (_ComicTester,), kwargs)
def generate_comic_testers():
"""For each comic scraper, create a test class."""
g = globals()
def get_test_scraperclasses():
"""Return scrapers that should be tested."""
if "TESTALL" in os.environ:
# test all comics (this will take some time)
scraperclasses = scraper.get_scraperclasses()
else:
# Get limited number of scraper tests on Travis builds to make
# it faster
testscrapernames = ['AbstruseGoose', 'xkcd']
testscrapernames = ['AbstruseGoose', 'GoComics/CalvinandHobbes', 'xkcd']
scraperclasses = [
scraperclass for scraperclass in scraper.get_scraperclasses()
if scraperclass.getName() in testscrapernames
]
scraperclasses.sort(key=lambda scraperclass: scraperclass.__name__)
for scraperclass in scraperclasses:
name = 'Test'+scraperclass.__name__
g[name] = make_comic_tester(name, scraperclass=scraperclass)
return scraperclasses
def pytest_generate_tests(metafunc):
if 'scraperclass' in metafunc.fixturenames:
metafunc.parametrize('scraperclass', get_test_scraperclasses())
generate_comic_testers()