dmoonfire/dosage
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Activity

Feed text content instead of binary to robots.txt parser.

Browse source
This commit is contained in:
Bastian Kleineidam 2013-04-07 18:11:29 +02:00
parent cdbeb65ede
commit 781bac0ca2
1 changed files with 1 additions and 1 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

2
dosagelib/util.py
View file

@ -251,7 +251,7 @@ def get_robotstxt_parser(url, session=None):
elif req.status_code >= 400: elif req.status_code >= 400:
rp.allow_all = True rp.allow_all = True
elif req.status_code == 200: elif req.status_code == 200:
rp.parse(req.content.splitlines()) rp.parse(req.text.splitlines())
return rp return rp