From 781bac0ca2b6fcf7a2e4a81bac2b33fe867fb0c9 Mon Sep 17 00:00:00 2001
From: Bastian Kleineidam <bastian.kleineidam@web.de>
Date: Sun, 7 Apr 2013 18:11:29 +0200
Subject: [PATCH] Feed text content instead of binary to robots.txt parser.

---
 dosagelib/util.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/dosagelib/util.py b/dosagelib/util.py
index e617bbbdc..ba7e475a2 100644
--- a/dosagelib/util.py
+++ b/dosagelib/util.py
@@ -251,7 +251,7 @@ def get_robotstxt_parser(url, session=None):
         elif req.status_code >= 400:
             rp.allow_all = True
         elif req.status_code == 200:
-            rp.parse(req.content.splitlines())
+            rp.parse(req.text.splitlines())
     return rp