dmoonfire/dosage
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Activity

Add POST support to urlopen().

Browse source
This commit is contained in:
Bastian Kleineidam 2013-04-04 18:30:02 +02:00
parent 44c3fb9f16
commit 460c5be689
1 changed files with 11 additions and 2 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

13
dosagelib/util.py
View file

@ -131,6 +131,7 @@ def getPageContent(url, session, max_content_bytes=MaxContentBytes):
tries -= 1 tries -= 1
if not isValidPageContent(data): if not isValidPageContent(data):
raise ValueError("Got invalid page content from %s: %r" % (url, data)) raise ValueError("Got invalid page content from %s: %r" % (url, data))
out.debug("Got page content %r" % data, level=3)
# determine base URL # determine base URL
baseUrl = None baseUrl = None
match = baseSearch.search(data) match = baseSearch.search(data)
@ -265,13 +266,14 @@ def get_robotstxt_parser(url, session=None):
def urlopen(url, session, referrer=None, max_content_bytes=None, def urlopen(url, session, referrer=None, max_content_bytes=None,
timeout=ConnectionTimeoutSecs, raise_for_status=True, timeout=ConnectionTimeoutSecs, raise_for_status=True,
stream=True): stream=True, data=None):
"""Open an URL and return the response object.""" """Open an URL and return the response object."""
out.debug('Open URL %s' % url) out.debug('Open URL %s' % url)
headers = {'User-Agent': UserAgent} headers = {'User-Agent': UserAgent}
if referrer: if referrer:
headers['Referer'] = referrer headers['Referer'] = referrer
out.debug('Sending headers %s' % headers, level=3) out.debug('Sending headers %s' % headers, level=3)
out.debug('Sending cookies %s' % session.cookies)
kwargs = { kwargs = {
"headers": headers, "headers": headers,
"timeout": timeout, "timeout": timeout,
@ -283,8 +285,15 @@ def urlopen(url, session, referrer=None, max_content_bytes=None,
# requests << 1.0 # requests << 1.0
kwargs["prefetch"] = not stream kwargs["prefetch"] = not stream
kwargs["config"] = {"max_retries": MaxRetries} kwargs["config"] = {"max_retries": MaxRetries}
if data:
kwargs['data'] = data
func = session.post
out.debug('Sending POST data %s' % data, level=3)
else:
func = session.get
try: try:
req = session.get(url, **kwargs) req = func(url, **kwargs)
out.debug('Response cookies: %s' % req.cookies)
check_content_size(url, req.headers, max_content_bytes) check_content_size(url, req.headers, max_content_bytes)
if raise_for_status: if raise_for_status:
req.raise_for_status() req.raise_for_status()