Feed text content instead of binary to robots.txt parser.

2013-04-07 18:11:29 +02:00 · 2013-04-07 18:11:29 +02:00 · 781bac0ca2
parent cdbeb65ede
commit 781bac0ca2
1 changed files with 1 additions and 1 deletions
--- a/dosagelib/util.py
+++ b/dosagelib/util.py
@ -251,7 +251,7 @@ def get_robotstxt_parser(url, session=None):
        elif req.status_code >= 400:
            rp.allow_all = True
        elif req.status_code == 200:
-            rp.parse(req.content.splitlines())
+            rp.parse(req.text.splitlines())
    return rp