Preserve the order we found images in when removing duplicate images

dosagelib/scraper.py

@@ -27,7 +27,7 @@ except ImportError:
 from . import loader, configuration, languages
 from .util import (get_page, makeSequence, get_system_uid, urlopen,
                    unescape, tagre, normaliseURL, prettyMatcherList,
-                   requests_session)
+                   requests_session, uniq)
 from .comic import ComicStrip
 from .output import out
 from .events import getHandler
@@ -137,7 +137,7 @@ class Scraper(object):
         # map modifier function on image URLs
         imageUrls = [self.imageUrlModifier(x, data) for x in imageUrls]
         # remove duplicate URLs
-        imageUrls = set(imageUrls)
+        imageUrls = uniq(imageUrls)
         if len(imageUrls) > 1 and not self.multipleImagesPerStrip:
             out.warn(
                 u"Found %d images instead of 1 at %s with expressions %s" %

dosagelib/util.py

@@ -522,3 +522,12 @@ def strlimit(s, length=72):
     if length == 0:
         return ""
     return "%s..." % s[:length]
+
+
+def uniq(input):
+    """Remove duplicates from a list while preserving the list order"""
+    output = []
+    for item in input:
+        if item not in output:
+            output.append(item)
+    return output