diff --git a/dosagelib/util.py b/dosagelib/util.py index 37e90aa5a..9b80468d6 100644 --- a/dosagelib/util.py +++ b/dosagelib/util.py @@ -289,7 +289,7 @@ def check_robotstxt(url, session): """ roboturl = get_roboturl(url) rp = get_robotstxt_parser(roboturl, session=session) - if not rp.can_fetch(UserAgent, url): + if not rp.can_fetch(UserAgent, str(url)): raise IOError("%s is disallowed by robots.txt" % url)