dmoonfire/dosage
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Activity

Use str() for robotparser.

Browse source
This commit is contained in:
Bastian Kleineidam 2013-04-09 19:36:00 +02:00
parent becc17029a
commit 190ffcd390
1 changed files with 1 additions and 1 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

2
dosagelib/util.py
View file

@ -289,7 +289,7 @@ def check_robotstxt(url, session):
""" """
roboturl = get_roboturl(url) roboturl = get_roboturl(url)
rp = get_robotstxt_parser(roboturl, session=session) rp = get_robotstxt_parser(roboturl, session=session)
if not rp.can_fetch(UserAgent, url): if not rp.can_fetch(UserAgent, str(url)):
raise IOError("%s is disallowed by robots.txt" % url) raise IOError("%s is disallowed by robots.txt" % url)