dmoonfire/dosage
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Activity

Change robotparser import to make PyInstaller happy.

Browse source
This commit is contained in:
Tobias Gruetzmacher 2016-06-06 22:42:01 +02:00
parent 64c8e502ca
commit 4f80016bf0
1 changed files with 2 additions and 2 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

4
dosagelib/util.py
View file

@ -7,7 +7,7 @@ from __future__ import absolute_import, division, print_function
from six.moves.urllib.parse import ( from six.moves.urllib.parse import (
quote as url_quote, unquote as url_unquote, urlparse, urlunparse, urlsplit) quote as url_quote, unquote as url_unquote, urlparse, urlunparse, urlsplit)
from six.moves.urllib import robotparser from six.moves.urllib_robotparser import RobotFileParser
import requests import requests
from requests.adapters import HTTPAdapter from requests.adapters import HTTPAdapter
from requests.packages.urllib3.util.retry import Retry from requests.packages.urllib3.util.retry import Retry
@ -250,7 +250,7 @@ def check_robotstxt(url, session):
@memoized @memoized
def get_robotstxt_parser(url, session=None): def get_robotstxt_parser(url, session=None):
"""Get a RobotFileParser for the given robots.txt URL.""" """Get a RobotFileParser for the given robots.txt URL."""
rp = robotparser.RobotFileParser() rp = RobotFileParser()
try: try:
req = urlopen(url, session, max_content_bytes=MaxContentBytes, req = urlopen(url, session, max_content_bytes=MaxContentBytes,
raise_for_status=False) raise_for_status=False)