659 lines
22 KiB
Python
659 lines
22 KiB
Python
'''
|
|
UrlRequest
|
|
==========
|
|
|
|
.. versionadded:: 1.0.8
|
|
|
|
You can use the :class:`UrlRequest` to make asynchronous requests on the
|
|
web and get the result when the request is completed. The spirit is the
|
|
same as the XHR object in Javascript.
|
|
|
|
The content is also decoded if the Content-Type is
|
|
application/json and the result automatically passed through json.loads.
|
|
|
|
|
|
The syntax to create a request::
|
|
|
|
from kivy.network.urlrequest import UrlRequest
|
|
req = UrlRequest(url, on_success, on_redirect, on_failure, on_error,
|
|
on_progress, req_body, req_headers, chunk_size,
|
|
timeout, method, decode, debug, file_path, ca_file,
|
|
verify)
|
|
|
|
|
|
Only the first argument is mandatory: the rest are optional.
|
|
By default, a "GET" request will be sent. If the :attr:`UrlRequest.req_body` is
|
|
not None, a "POST" request will be sent. It's up to you to adjust
|
|
:attr:`UrlRequest.req_headers` to suit your requirements and the response
|
|
to the request will be accessible as the parameter called "result" on
|
|
the callback function of the on_success event.
|
|
|
|
|
|
Example of fetching JSON::
|
|
|
|
def got_json(req, result):
|
|
for key, value in req.resp_headers.items():
|
|
print('{}: {}'.format(key, value))
|
|
|
|
req = UrlRequest('https://httpbin.org/headers', got_json)
|
|
|
|
Example of Posting data (adapted from httplib example)::
|
|
|
|
import urllib
|
|
|
|
def bug_posted(req, result):
|
|
print('Our bug is posted!')
|
|
print(result)
|
|
|
|
params = urllib.urlencode({'@number': 12524, '@type': 'issue',
|
|
'@action': 'show'})
|
|
headers = {'Content-type': 'application/x-www-form-urlencoded',
|
|
'Accept': 'text/plain'}
|
|
req = UrlRequest('bugs.python.org', on_success=bug_posted, req_body=params,
|
|
req_headers=headers)
|
|
|
|
If you want a synchronous request, you can call the wait() method.
|
|
|
|
'''
|
|
|
|
from base64 import b64encode
|
|
from collections import deque
|
|
from threading import Thread, Event
|
|
from json import loads
|
|
from time import sleep
|
|
from kivy.compat import PY2
|
|
from kivy.config import Config
|
|
|
|
if PY2:
|
|
from httplib import HTTPConnection
|
|
from urlparse import urlparse, urlunparse
|
|
else:
|
|
from http.client import HTTPConnection
|
|
from urllib.parse import urlparse, urlunparse
|
|
|
|
try:
|
|
import ssl
|
|
|
|
HTTPSConnection = None
|
|
if PY2:
|
|
from httplib import HTTPSConnection
|
|
else:
|
|
from http.client import HTTPSConnection
|
|
except ImportError:
|
|
# depending the platform, if openssl support wasn't compiled before python,
|
|
# this class is not available.
|
|
pass
|
|
|
|
from kivy.clock import Clock
|
|
from kivy.weakmethod import WeakMethod
|
|
from kivy.logger import Logger
|
|
from kivy.utils import platform
|
|
|
|
|
|
# list to save UrlRequest and prevent GC on un-referenced objects
|
|
g_requests = []
|
|
|
|
|
|
class UrlRequest(Thread):
|
|
'''A UrlRequest. See module documentation for usage.
|
|
|
|
.. versionchanged:: 1.5.1
|
|
Add `debug` parameter
|
|
|
|
.. versionchanged:: 1.0.10
|
|
Add `method` parameter
|
|
|
|
.. versionchanged:: 1.8.0
|
|
|
|
Parameter `decode` added.
|
|
Parameter `file_path` added.
|
|
Parameter `on_redirect` added.
|
|
Parameter `on_failure` added.
|
|
|
|
.. versionchanged:: 1.9.1
|
|
|
|
Parameter `ca_file` added.
|
|
Parameter `verify` added.
|
|
|
|
.. versionchanged:: 1.10.0
|
|
|
|
Parameters `proxy_host`, `proxy_port` and `proxy_headers` added.
|
|
|
|
.. versionchanged:: 1.11.0
|
|
|
|
Parameters `on_cancel` added.
|
|
|
|
:Parameters:
|
|
`url`: str
|
|
Complete url string to call.
|
|
`on_success`: callback(request, result)
|
|
Callback function to call when the result has been fetched.
|
|
`on_redirect`: callback(request, result)
|
|
Callback function to call if the server returns a Redirect.
|
|
`on_failure`: callback(request, result)
|
|
Callback function to call if the server returns a Client or
|
|
Server Error.
|
|
`on_error`: callback(request, error)
|
|
Callback function to call if an error occurs.
|
|
`on_progress`: callback(request, current_size, total_size)
|
|
Callback function that will be called to report progression of the
|
|
download. `total_size` might be -1 if no Content-Length has been
|
|
reported in the http response.
|
|
This callback will be called after each `chunk_size` is read.
|
|
`on_cancel`: callback(request)
|
|
Callback function to call if user requested to cancel the download
|
|
operation via the .cancel() method.
|
|
`req_body`: str, defaults to None
|
|
Data to sent in the request. If it's not None, a POST will be done
|
|
instead of a GET.
|
|
`req_headers`: dict, defaults to None
|
|
Custom headers to add to the request.
|
|
`chunk_size`: int, defaults to 8192
|
|
Size of each chunk to read, used only when `on_progress` callback
|
|
has been set. If you decrease it too much, a lot of on_progress
|
|
callbacks will be fired and will slow down your download. If you
|
|
want to have the maximum download speed, increase the chunk_size
|
|
or don't use ``on_progress``.
|
|
`timeout`: int, defaults to None
|
|
If set, blocking operations will timeout after this many seconds.
|
|
`method`: str, defaults to 'GET' (or 'POST' if ``body`` is specified)
|
|
The HTTP method to use.
|
|
`decode`: bool, defaults to True
|
|
If False, skip decoding of the response.
|
|
`debug`: bool, defaults to False
|
|
If True, it will use the Logger.debug to print information
|
|
about url access/progression/errors.
|
|
`file_path`: str, defaults to None
|
|
If set, the result of the UrlRequest will be written to this path
|
|
instead of in memory.
|
|
`ca_file`: str, defaults to None
|
|
Indicates a SSL CA certificate file path to validate HTTPS
|
|
certificates against
|
|
`verify`: bool, defaults to True
|
|
If False, disables SSL CA certificate verification
|
|
`proxy_host`: str, defaults to None
|
|
If set, the proxy host to use for this connection.
|
|
`proxy_port`: int, defaults to None
|
|
If set, and `proxy_host` is also set, the port to use for
|
|
connecting to the proxy server.
|
|
`proxy_headers`: dict, defaults to None
|
|
If set, and `proxy_host` is also set, the headers to send to the
|
|
proxy server in the ``CONNECT`` request.
|
|
'''
|
|
|
|
def __init__(self, url, on_success=None, on_redirect=None,
|
|
on_failure=None, on_error=None, on_progress=None,
|
|
req_body=None, req_headers=None, chunk_size=8192,
|
|
timeout=None, method=None, decode=True, debug=False,
|
|
file_path=None, ca_file=None, verify=True, proxy_host=None,
|
|
proxy_port=None, proxy_headers=None, user_agent=None,
|
|
on_cancel=None, cookies=None):
|
|
super(UrlRequest, self).__init__()
|
|
self._queue = deque()
|
|
self._trigger_result = Clock.create_trigger(self._dispatch_result, 0)
|
|
self.daemon = True
|
|
self.on_success = WeakMethod(on_success) if on_success else None
|
|
self.on_redirect = WeakMethod(on_redirect) if on_redirect else None
|
|
self.on_failure = WeakMethod(on_failure) if on_failure else None
|
|
self.on_error = WeakMethod(on_error) if on_error else None
|
|
self.on_progress = WeakMethod(on_progress) if on_progress else None
|
|
self.on_cancel = WeakMethod(on_cancel) if on_cancel else None
|
|
self.decode = decode
|
|
self.file_path = file_path
|
|
self._debug = debug
|
|
self._result = None
|
|
self._error = None
|
|
self._is_finished = False
|
|
self._resp_status = None
|
|
self._resp_headers = None
|
|
self._resp_length = -1
|
|
self._chunk_size = chunk_size
|
|
self._timeout = timeout
|
|
self._method = method
|
|
self.verify = verify
|
|
self._proxy_host = proxy_host
|
|
self._proxy_port = proxy_port
|
|
self._proxy_headers = proxy_headers
|
|
self._cancel_event = Event()
|
|
self._user_agent = user_agent
|
|
self._cookies = cookies
|
|
|
|
if platform in ['android', 'ios']:
|
|
import certifi
|
|
self.ca_file = ca_file or certifi.where()
|
|
else:
|
|
self.ca_file = ca_file
|
|
|
|
#: Url of the request
|
|
self.url = url
|
|
|
|
#: Request body passed in __init__
|
|
self.req_body = req_body
|
|
|
|
#: Request headers passed in __init__
|
|
self.req_headers = req_headers
|
|
|
|
# save our request to prevent GC
|
|
g_requests.append(self)
|
|
|
|
self.start()
|
|
|
|
def run(self):
|
|
q = self._queue.appendleft
|
|
url = self.url
|
|
req_body = self.req_body
|
|
req_headers = self.req_headers or {}
|
|
|
|
user_agent = self._user_agent
|
|
cookies = self._cookies
|
|
|
|
if user_agent:
|
|
req_headers.setdefault('User-Agent', user_agent)
|
|
|
|
elif (
|
|
Config.has_section('network')
|
|
and 'useragent' in Config.items('network')
|
|
):
|
|
useragent = Config.get('network', 'useragent')
|
|
req_headers.setdefault('User-Agent', useragent)
|
|
|
|
if cookies:
|
|
req_headers.setdefault("Cookie", cookies)
|
|
|
|
try:
|
|
result, resp = self._fetch_url(url, req_body, req_headers, q)
|
|
if self.decode:
|
|
result = self.decode_result(result, resp)
|
|
except Exception as e:
|
|
q(('error', None, e))
|
|
else:
|
|
if not self._cancel_event.is_set():
|
|
q(('success', resp, result))
|
|
else:
|
|
q(('killed', None, None))
|
|
|
|
# using trigger can result in a missed on_success event
|
|
self._trigger_result()
|
|
|
|
# clean ourself when the queue is empty
|
|
while len(self._queue):
|
|
sleep(.1)
|
|
self._trigger_result()
|
|
|
|
# ok, authorize the GC to clean us.
|
|
if self in g_requests:
|
|
g_requests.remove(self)
|
|
|
|
def _parse_url(self, url):
|
|
parse = urlparse(url)
|
|
host = parse.hostname
|
|
port = parse.port
|
|
userpass = None
|
|
|
|
# append user + pass to hostname if specified
|
|
if parse.username and parse.password:
|
|
userpass = {
|
|
"Authorization": "Basic {}".format(b64encode(
|
|
"{}:{}".format(
|
|
parse.username,
|
|
parse.password
|
|
).encode('utf-8')
|
|
).decode('utf-8'))
|
|
}
|
|
|
|
return host, port, userpass, parse
|
|
|
|
def _fetch_url(self, url, body, headers, q):
|
|
# Parse and fetch the current url
|
|
trigger = self._trigger_result
|
|
chunk_size = self._chunk_size
|
|
report_progress = self.on_progress is not None
|
|
timeout = self._timeout
|
|
file_path = self.file_path
|
|
ca_file = self.ca_file
|
|
verify = self.verify
|
|
|
|
if self._debug:
|
|
Logger.debug('UrlRequest: {0} Fetch url <{1}>'.format(
|
|
id(self), url))
|
|
Logger.debug('UrlRequest: {0} - body: {1}'.format(
|
|
id(self), body))
|
|
Logger.debug('UrlRequest: {0} - headers: {1}'.format(
|
|
id(self), headers))
|
|
|
|
# parse url
|
|
host, port, userpass, parse = self._parse_url(url)
|
|
if userpass and not headers:
|
|
headers = userpass
|
|
elif userpass and headers:
|
|
key = list(userpass.keys())[0]
|
|
headers[key] = userpass[key]
|
|
|
|
# translate scheme to connection class
|
|
cls = self.get_connection_for_scheme(parse.scheme)
|
|
|
|
# reconstruct path to pass on the request
|
|
path = parse.path
|
|
if parse.params:
|
|
path += ';' + parse.params
|
|
if parse.query:
|
|
path += '?' + parse.query
|
|
if parse.fragment:
|
|
path += '#' + parse.fragment
|
|
|
|
# create connection instance
|
|
args = {}
|
|
if timeout is not None:
|
|
args['timeout'] = timeout
|
|
|
|
if (ca_file is not None and hasattr(ssl, 'create_default_context') and
|
|
parse.scheme == 'https'):
|
|
ctx = ssl.create_default_context(cafile=ca_file)
|
|
ctx.verify_mode = ssl.CERT_REQUIRED
|
|
args['context'] = ctx
|
|
|
|
if not verify and parse.scheme == 'https' and (
|
|
hasattr(ssl, 'create_default_context')):
|
|
ctx = ssl.create_default_context()
|
|
ctx.check_hostname = False
|
|
ctx.verify_mode = ssl.CERT_NONE
|
|
args['context'] = ctx
|
|
|
|
if self._proxy_host:
|
|
Logger.debug('UrlRequest: {0} - proxy via {1}:{2}'.format(
|
|
id(self), self._proxy_host, self._proxy_port
|
|
))
|
|
req = cls(self._proxy_host, self._proxy_port, **args)
|
|
if parse.scheme == 'https':
|
|
req.set_tunnel(host, port, self._proxy_headers)
|
|
else:
|
|
path = urlunparse(parse)
|
|
else:
|
|
req = cls(host, port, **args)
|
|
|
|
# send request
|
|
method = self._method
|
|
if method is None:
|
|
method = 'GET' if body is None else 'POST'
|
|
req.request(method, path, body, headers or {})
|
|
|
|
# read header
|
|
resp = req.getresponse()
|
|
|
|
# read content
|
|
if report_progress or file_path is not None:
|
|
try:
|
|
total_size = int(resp.getheader('content-length'))
|
|
except:
|
|
total_size = -1
|
|
|
|
# before starting the download, send a fake progress to permit the
|
|
# user to initialize his ui
|
|
if report_progress:
|
|
q(('progress', resp, (0, total_size)))
|
|
|
|
def get_chunks(fd=None):
|
|
bytes_so_far = 0
|
|
result = b''
|
|
while 1:
|
|
chunk = resp.read(chunk_size)
|
|
if not chunk:
|
|
break
|
|
|
|
if fd:
|
|
fd.write(chunk)
|
|
else:
|
|
result += chunk
|
|
|
|
bytes_so_far += len(chunk)
|
|
# report progress to user
|
|
if report_progress:
|
|
q(('progress', resp, (bytes_so_far, total_size)))
|
|
trigger()
|
|
if self._cancel_event.is_set():
|
|
break
|
|
return bytes_so_far, result
|
|
|
|
if file_path is not None:
|
|
with open(file_path, 'wb') as fd:
|
|
bytes_so_far, result = get_chunks(fd)
|
|
else:
|
|
bytes_so_far, result = get_chunks()
|
|
|
|
# ensure that results are dispatched for the last chunk,
|
|
# avoid trigger
|
|
if report_progress:
|
|
q(('progress', resp, (bytes_so_far, total_size)))
|
|
trigger()
|
|
else:
|
|
result = resp.read()
|
|
try:
|
|
if isinstance(result, bytes):
|
|
result = result.decode('utf-8')
|
|
except UnicodeDecodeError:
|
|
# if it's an image? decoding would not work
|
|
pass
|
|
req.close()
|
|
|
|
# return everything
|
|
return result, resp
|
|
|
|
def get_connection_for_scheme(self, scheme):
|
|
'''Return the Connection class for a particular scheme.
|
|
This is an internal function that can be expanded to support custom
|
|
schemes.
|
|
|
|
Actual supported schemes: http, https.
|
|
'''
|
|
if scheme == 'http':
|
|
return HTTPConnection
|
|
elif scheme == 'https' and HTTPSConnection is not None:
|
|
return HTTPSConnection
|
|
else:
|
|
raise Exception('No class for scheme %s' % scheme)
|
|
|
|
def decode_result(self, result, resp):
|
|
'''Decode the result fetched from url according to his Content-Type.
|
|
Currently supports only application/json.
|
|
'''
|
|
# Entry to decode url from the content type.
|
|
# For example, if the content type is a json, it will be automatically
|
|
# decoded.
|
|
content_type = resp.getheader('Content-Type', None)
|
|
if content_type is not None:
|
|
ct = content_type.split(';')[0]
|
|
if ct == 'application/json':
|
|
if isinstance(result, bytes):
|
|
result = result.decode('utf-8')
|
|
try:
|
|
return loads(result)
|
|
except:
|
|
return result
|
|
|
|
return result
|
|
|
|
def _dispatch_result(self, dt):
|
|
while True:
|
|
# Read the result pushed on the queue, and dispatch to the client
|
|
try:
|
|
result, resp, data = self._queue.pop()
|
|
except IndexError:
|
|
return
|
|
if resp:
|
|
# Small workaround in order to prevent the situation mentioned
|
|
# in the comment below
|
|
final_cookies = ""
|
|
parsed_headers = []
|
|
for key, value in resp.getheaders():
|
|
if key == "Set-Cookie":
|
|
final_cookies += "{};".format(value)
|
|
else:
|
|
parsed_headers.append((key, value))
|
|
parsed_headers.append(("Set-Cookie", final_cookies[:-1]))
|
|
|
|
# XXX usage of dict can be dangerous if multiple headers
|
|
# are set even if it's invalid. But it look like it's ok
|
|
# ? http://stackoverflow.com/questions/2454494/..
|
|
# ..urllib2-multiple-set-cookie-headers-in-response
|
|
self._resp_headers = dict(parsed_headers)
|
|
self._resp_status = resp.status
|
|
if result == 'success':
|
|
status_class = resp.status // 100
|
|
|
|
if status_class in (1, 2):
|
|
if self._debug:
|
|
Logger.debug('UrlRequest: {0} Download finished with'
|
|
' {1} datalen'.format(id(self),
|
|
len(data)))
|
|
self._is_finished = True
|
|
self._result = data
|
|
if self.on_success:
|
|
func = self.on_success()
|
|
if func:
|
|
func(self, data)
|
|
|
|
elif status_class == 3:
|
|
if self._debug:
|
|
Logger.debug('UrlRequest: {} Download '
|
|
'redirected'.format(id(self)))
|
|
self._is_finished = True
|
|
self._result = data
|
|
if self.on_redirect:
|
|
func = self.on_redirect()
|
|
if func:
|
|
func(self, data)
|
|
|
|
elif status_class in (4, 5):
|
|
if self._debug:
|
|
Logger.debug('UrlRequest: {} Download failed with '
|
|
'http error {}'.format(id(self),
|
|
resp.status))
|
|
self._is_finished = True
|
|
self._result = data
|
|
if self.on_failure:
|
|
func = self.on_failure()
|
|
if func:
|
|
func(self, data)
|
|
|
|
elif result == 'error':
|
|
if self._debug:
|
|
Logger.debug('UrlRequest: {0} Download error '
|
|
'<{1}>'.format(id(self), data))
|
|
self._is_finished = True
|
|
self._error = data
|
|
if self.on_error:
|
|
func = self.on_error()
|
|
if func:
|
|
func(self, data)
|
|
|
|
elif result == 'progress':
|
|
if self._debug:
|
|
Logger.debug('UrlRequest: {0} Download progress '
|
|
'{1}'.format(id(self), data))
|
|
if self.on_progress:
|
|
func = self.on_progress()
|
|
if func:
|
|
func(self, data[0], data[1])
|
|
|
|
elif result == 'killed':
|
|
if self._debug:
|
|
Logger.debug('UrlRequest: Cancelled by user')
|
|
if self.on_cancel:
|
|
func = self.on_cancel()
|
|
if func:
|
|
func(self)
|
|
|
|
else:
|
|
assert(0)
|
|
|
|
@property
|
|
def is_finished(self):
|
|
'''Return True if the request has finished, whether it's a
|
|
success or a failure.
|
|
'''
|
|
return self._is_finished
|
|
|
|
@property
|
|
def result(self):
|
|
'''Return the result of the request.
|
|
This value is not determined until the request is finished.
|
|
'''
|
|
return self._result
|
|
|
|
@property
|
|
def resp_headers(self):
|
|
'''If the request has been completed, return a dictionary containing
|
|
the headers of the response. Otherwise, it will return None.
|
|
'''
|
|
return self._resp_headers
|
|
|
|
@property
|
|
def resp_status(self):
|
|
'''Return the status code of the response if the request is complete,
|
|
otherwise return None.
|
|
'''
|
|
return self._resp_status
|
|
|
|
@property
|
|
def error(self):
|
|
'''Return the error of the request.
|
|
This value is not determined until the request is completed.
|
|
'''
|
|
return self._error
|
|
|
|
@property
|
|
def chunk_size(self):
|
|
'''Return the size of a chunk, used only in "progress" mode (when
|
|
on_progress callback is set.)
|
|
'''
|
|
return self._chunk_size
|
|
|
|
def wait(self, delay=0.5):
|
|
'''Wait for the request to finish (until :attr:`resp_status` is not
|
|
None)
|
|
|
|
.. note::
|
|
This method is intended to be used in the main thread, and the
|
|
callback will be dispatched from the same thread
|
|
from which you're calling.
|
|
|
|
.. versionadded:: 1.1.0
|
|
'''
|
|
while self.resp_status is None:
|
|
self._dispatch_result(delay)
|
|
sleep(delay)
|
|
|
|
def cancel(self):
|
|
'''Cancel the current request. It will be aborted, and the result
|
|
will not be dispatched. Once cancelled, the callback on_cancel will
|
|
be called.
|
|
|
|
.. versionadded:: 1.11.0
|
|
'''
|
|
self._cancel_event.set()
|
|
|
|
|
|
if __name__ == '__main__':
|
|
|
|
from pprint import pprint
|
|
|
|
def on_success(req, result):
|
|
pprint('Got the result:')
|
|
pprint(result)
|
|
|
|
def on_error(req, error):
|
|
pprint('Got an error:')
|
|
pprint(error)
|
|
|
|
Clock.start_clock()
|
|
req = UrlRequest('https://en.wikipedia.org/w/api.php?format'
|
|
'=json&action=query&titles=Kivy&prop=revisions&rvprop=content',
|
|
on_success, on_error)
|
|
while not req.is_finished:
|
|
sleep(1)
|
|
Clock.tick()
|
|
Clock.stop_clock()
|
|
|
|
print('result =', req.result)
|
|
print('error =', req.error)
|