Rename get_scrapers to get_scraperclasses

dosage

@@ -7,6 +7,7 @@ from __future__ import print_function
 import sys
 import os
 import optparse
+from collections import OrderedDict
 
 from dosagelib import events, scraper
 from dosagelib.output import out
@@ -208,7 +209,7 @@ def getScrapers(comics, basepath=None, adult=True):
         # only scrapers whose directory already exists
         if len(comics) > 1:
             out.warn("using '@' as comic name ignores all other specified comics.")
-        for scraperclass in scraper.get_scrapers():
+        for scraperclass in scraper.get_scraperclasses():
             if not adult and scraperclass.adult:
                 warn_adult(scraperclass)
                 continue
@@ -217,15 +218,15 @@ def getScrapers(comics, basepath=None, adult=True):
                 yield scraperclass()
     elif '@@' in comics:
         # all scrapers
-        for scraperclass in scraper.get_scrapers():
+        for scraperclass in scraper.get_scraperclasses():
             if not adult and scraperclass.adult:
                 warn_adult(scraperclass)
                 continue
             yield scraperclass()
     else:
         # get only selected comic scrapers
-        # store them in a list to catch naming errors early
-        scrapers = []
+        # store them in an ordered set to eliminate duplicates
+        scrapers = OrderedDict()
         for comic in comics:
             if ':' in comic:
                 name, index = comic.split(':', 1)
@@ -233,13 +234,16 @@ def getScrapers(comics, basepath=None, adult=True):
             else:
                 name = comic
                 indexes = None
+            # XXX if matchmulti, search for a list of scrapers
             scraperclass = scraper.get_scraper(name)
             if not adult and scraperclass.adult:
                 warn_adult(scraperclass)
                 continue
-            scrapers.append(scraperclass(indexes=indexes))
-        for s in scrapers:
-            yield s
+            scraperobj = scraperclass(indexes=indexes)
+            if scraperobj not in scrapers:
+                scrapers[scraperobj] = True
+        for scraperobj in scrapers:
+            yield scraperobj
 
 
 def warn_adult(scraperclass):
@@ -269,7 +273,7 @@ def main():
 def profile():
     """Profile the loading of all scrapers."""
     import cProfile
-    cProfile.run("scraper.get_scrapers()", "dosage.prof")
+    cProfile.run("scraper.get_scraperclasses()", "dosage.prof")
 
 
 def viewprof():

dosagelib/scraper.py

@@ -166,7 +166,7 @@ def get_scraper(comic):
         raise ValueError("empty comic name")
     candidates = []
     cname = comic.lower()
-    for scraperclass in get_scrapers():
+    for scraperclass in get_scraperclasses():
         lname = scraperclass.get_name().lower()
         if lname == cname:
             # perfect match
@@ -182,29 +182,29 @@ def get_scraper(comic):
         raise ValueError('comic %r not found' % comic)
 
 
-_scrapers = None
-def get_scrapers():
+_scraperclasses = None
+def get_scraperclasses():
     """Find all comic scraper classes in the plugins directory.
     The result is cached.
     @return: list of _BasicScraper classes
     @rtype: list of _BasicScraper
     """
-    global _scrapers
-    if _scrapers is None:
+    global _scraperclasses
+    if _scraperclasses is None:
         out.debug("Loading comic modules...")
         modules = loader.get_modules()
         plugins = loader.get_plugins(modules, _BasicScraper)
-        _scrapers = list(plugins)
-        _scrapers.sort(key=lambda s: s.get_name())
+        _scraperclasses = list(plugins)
+        _scraperclasses.sort(key=lambda s: s.get_name())
         check_scrapers()
-        out.debug("... %d modules loaded." % len(_scrapers))
-    return _scrapers
+        out.debug("... %d modules loaded." % len(_scraperclasses))
+    return _scraperclasses
 
 
 def check_scrapers():
     """Check for duplicate scraper class names."""
     d = {}
-    for scraperclass in _scrapers:
+    for scraperclass in _scraperclasses:
         name = scraperclass.get_name().lower()
         if name in d:
             name1 = scraperclass.get_name()

scripts/arcamax.py

@@ -10,7 +10,7 @@ import os
 import requests
 sys.path.append(os.path.join(os.path.dirname(__file__), ".."))
 from dosagelib.util import getPageContent, asciify, unescape
-from dosagelib.scraper import get_scrapers
+from dosagelib.scraper import get_scraperclasses
 from scriptutil import contains_case_insensitive, capfirst, save_result, load_result, truncate_name
 
 json_file = __file__.replace(".py", ".json")
@@ -64,7 +64,7 @@ def has_comic(name):
         ("KeenSpot/%s" % name).lower(),
         ("SmackJeeves/%s" % name).lower(),
     ]
-    for scraperclass in get_scrapers():
+    for scraperclass in get_scraperclasses():
         lname = scraperclass.get_name().lower()
         if lname in names:
             return True

scripts/gocomics.py

@@ -10,7 +10,7 @@ import os
 import requests
 sys.path.append(os.path.join(os.path.dirname(__file__), ".."))
 from dosagelib.util import tagre, getPageContent, asciify, unescape
-from dosagelib.scraper import get_scrapers
+from dosagelib.scraper import get_scraperclasses
 from scriptutil import contains_case_insensitive, capfirst, save_result, load_result, truncate_name
 
 json_file = __file__.replace(".py", ".json")
@@ -78,7 +78,7 @@ def get_results():
 def has_creators_comic(name):
     """Test if comic name already exists."""
     cname = "Creators/%s" % name
-    for scraperclass in get_scrapers():
+    for scraperclass in get_scraperclasses():
         lname = scraperclass.get_name().lower()
         if lname == cname.lower():
             return True

scripts/keenspot.py

@@ -10,7 +10,7 @@ import os
 import requests
 sys.path.append(os.path.join(os.path.dirname(__file__), ".."))
 from dosagelib.util import getPageContent, asciify, unescape, tagre
-from dosagelib.scraper import get_scrapers
+from dosagelib.scraper import get_scraperclasses
 from scriptutil import contains_case_insensitive, capfirst, save_result, load_result, truncate_name
 
 json_file = __file__.replace(".py", ".json")
@@ -406,7 +406,7 @@ def has_comic(name):
     """Check if comic name already exists."""
     cname = ("Creators/%s" % name).lower()
     gname = ("GoComics/%s" % name).lower()
-    for scraperclass in get_scrapers():
+    for scraperclass in get_scraperclasses():
         lname = scraperclass.get_name().lower()
         if lname == cname or lname == gname:
             return True

scripts/mktestpage.py

@@ -7,8 +7,7 @@ import time
 import cgi
 import codecs
 sys.path.append(os.path.join(os.path.dirname(__file__), ".."))
-from dosagelib.scraper import get_scrapers
-from dosagelib.configuration import Version as DosageVersion
+from dosagelib.scraper import get_scraperclasses
 from scriptutil import load_result, save_result
 
 json_file = __file__.replace(".py", ".json")
@@ -101,9 +100,9 @@ def strdate(t):
 def get_testscraper(line):
     """Get scraper from test output line."""
     classname = line.split('::')[1][4:]
-    for scraper in get_scrapers():
-        if scraper.__name__ == classname:
-            return scraper
+    for scraperclass in get_scraperclasses():
+        if scraperclass.__name__ == classname:
+            return scraperclass
     raise ValueError("Scraper %r not found" % classname)
 
 

tests/test_comicnames.py

@@ -7,7 +7,7 @@ from dosagelib import scraper, util
 class TestComicNames(TestCase):
 
     def test_names(self):
-        for scraperclass in scraper.get_scrapers():
+        for scraperclass in scraper.get_scraperclasses():
             name = scraperclass.get_name()
             self.assertTrue(name.count('/') <= 1, name)
             if '/' in name: