From 190ffcd39099709e10b55c0eca2aa091351c91a8 Mon Sep 17 00:00:00 2001 From: Bastian Kleineidam Date: Tue, 9 Apr 2013 19:36:00 +0200 Subject: [PATCH] Use str() for robotparser. --- dosagelib/util.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/dosagelib/util.py b/dosagelib/util.py index 37e90aa5a..9b80468d6 100644 --- a/dosagelib/util.py +++ b/dosagelib/util.py @@ -289,7 +289,7 @@ def check_robotstxt(url, session): """ roboturl = get_roboturl(url) rp = get_robotstxt_parser(roboturl, session=session) - if not rp.can_fetch(UserAgent, url): + if not rp.can_fetch(UserAgent, str(url)): raise IOError("%s is disallowed by robots.txt" % url)