Package restkit :: Package client :: Module request
[hide private]
[frames] | no frames]

Source Code for Module restkit.client.request

  1  # -*- coding: utf-8 - 
  2  # 
  3  # This file is part of restkit released under the MIT license.  
  4  # See the NOTICE for more information. 
  5   
  6  import copy 
  7  import cgi 
  8  import errno 
  9  import logging 
 10  import mimetypes 
 11  import os 
 12  import socket 
 13  import time 
 14  try: 
 15      from cStringIO import StringIO 
 16  except ImportError: 
 17      from StringIO import StringIO 
 18  import types 
 19  import urlparse 
 20  import uuid 
 21   
 22  from restkit import __version__ 
 23  from restkit.client.response import HttpResponse 
 24  from restkit.conn import get_default_manager, \ 
 25  DEFAULT_CONN_NB_CONNECTIONS, DEFAULT_CONN_TIMEOUT 
 26  from restkit.errors import RequestError, InvalidUrl, RedirectLimit, \ 
 27  RequestTimeout 
 28  from restkit.filters import Filters 
 29  from restkit.forms import multipart_form_encode, form_encode 
 30  from restkit.util import sock 
 31  from restkit import util 
 32  from restkit import http 
 33   
 34  MAX_FOLLOW_REDIRECTS = 5 
 35   
 36  USER_AGENT = "restkit/%s" % __version__ 
 37   
 38  log = logging.getLogger(__name__) 
 39   
40 -class HttpRequest(object):
41 """ Http Connection object. """ 42 43 version = (1, 1) 44 response_class = HttpResponse 45
46 - def __init__(self, 47 timeout=DEFAULT_CONN_TIMEOUT, 48 filters=None, 49 follow_redirect=False, 50 force_follow_redirect=False, 51 max_follow_redirect=MAX_FOLLOW_REDIRECTS, 52 decompress=True, 53 pool_instance=None, 54 response_class=None, 55 conn_manager=None, 56 nb_connections=DEFAULT_CONN_NB_CONNECTIONS, 57 **ssl_args):
58 59 """ HttpRequest constructor 60 61 :param timeout: socket timeout 62 :param filters: list, list of http filters. see the doc of http filters 63 for more info 64 :param follow_redirect: boolean, by default is false. If true, 65 if the HTTP status is 301, 302 or 303 the client will follow 66 the location. 67 :param max_follow_redirect: max number of redirection. If max is reached 68 the RedirectLimit exception is raised. 69 :param conn_manager: a connectoin manager instance inherited from 70 `restkit.conn.base.ConnectioManager` 71 :param ssl_args: ssl arguments. See http://docs.python.org/library/ssl.html 72 for more information. 73 """ 74 self._conn = None 75 self.timeout = timeout 76 self.headers = [] 77 self.req_headers = [] 78 self.ua = USER_AGENT 79 self.url = None 80 81 self.follow_redirect = follow_redirect 82 self.nb_redirections = max_follow_redirect 83 self.force_follow_redirect = force_follow_redirect 84 self.decompress = decompress 85 self.method = 'GET' 86 self.body = None 87 self.response_body = StringIO() 88 self.final_url = None 89 90 # build filter lists 91 self.filters = Filters(filters) 92 self.ssl_args = ssl_args or {} 93 94 if pool_instance is not None: 95 self.conn_manager = pool_instance 96 elif conn_manager is not None: 97 self.conn_manager = conn_manager 98 else: 99 self.conn_manager = get_default_manager( 100 timeout=timeout, 101 nb_connections=nb_connections 102 ) 103 104 if response_class is not None: 105 self.response_class = response_class
106
107 - def parse_url(self, url):
108 """ parse url and get host/port""" 109 self.uri = urlparse.urlparse(url) 110 if self.uri.scheme not in ('http', 'https'): 111 raise InvalidUrl("None valid url") 112 113 host, port = util.parse_netloc(self.uri) 114 self.host = host 115 self.port = port
116
117 - def set_body(self, body, headers, chunked=False):
118 """ set HTTP body and manage form if needed """ 119 content_type = headers.get('CONTENT-TYPE') 120 content_length = headers.get('CONTENT-LENGTH') 121 if not body: 122 if content_type is not None: 123 self.headers.append(('Content-Type', content_type)) 124 if self.method in ('POST', 'PUT'): 125 self.headers.append(("Content-Length", "0")) 126 return 127 128 # set content lengh if needed 129 if isinstance(body, dict): 130 if content_type is not None and \ 131 content_type.startswith("multipart/form-data"): 132 type_, opts = cgi.parse_header(content_type) 133 boundary = opts.get('boundary', uuid.uuid4().hex) 134 body, self.headers = multipart_form_encode(body, 135 self.headers, boundary) 136 else: 137 content_type = "application/x-www-form-urlencoded; charset=utf-8" 138 body = form_encode(body) 139 elif hasattr(body, "boundary"): 140 content_type = "multipart/form-data; boundary=%s" % body.boundary 141 content_length = body.get_size() 142 143 if not content_type: 144 content_type = 'application/octet-stream' 145 if hasattr(body, 'name'): 146 content_type = mimetypes.guess_type(body.name)[0] 147 148 if not content_length: 149 if hasattr(body, 'fileno'): 150 try: 151 body.flush() 152 except IOError: 153 pass 154 content_length = str(os.fstat(body.fileno())[6]) 155 elif hasattr(body, 'getvalue'): 156 try: 157 content_length = str(len(body.getvalue())) 158 except AttributeError: 159 pass 160 elif isinstance(body, types.StringTypes): 161 body = util.to_bytestring(body) 162 content_length = len(body) 163 164 if content_length: 165 self.headers.append(("Content-Length", content_length)) 166 if content_type is not None: 167 self.headers.append(('Content-Type', content_type)) 168 169 elif not chunked: 170 raise RequestError("Can't determine content length and " + 171 "Transfer-Encoding header is not chunked") 172 173 self.body = body
174 175
176 - def request(self, url, method='GET', body=None, headers=None):
177 """ make effective request 178 179 :param url: str, url string 180 :param method: str, by default GET. http verbs 181 :param body: the body, could be a string, an iterator or a file-like object 182 :param headers: dict or list of tupple, http headers 183 """ 184 self._conn = None 185 self.url = url 186 self.final_url = url 187 self.parse_url(url) 188 self.method = method.upper() 189 190 self.init_headers = copy.copy(headers or []) 191 self.headers = [] 192 193 # headers are better as list 194 headers = headers or [] 195 if isinstance(headers, dict): 196 headers = headers.items() 197 198 chunked = False 199 200 # normalize headers 201 search_headers = ('USER-AGENT', 'CONTENT-TYPE', 202 'CONTENT-LENGTH', 'ACCEPT-ENCODING', 203 'TRANSFER-ENCODING', 'CONNECTION', 'HOST') 204 found_headers = {} 205 new_headers = copy.copy(headers) 206 for (name, value) in headers: 207 uname = name.upper() 208 if uname in search_headers: 209 if uname == 'TRANSFER-ENCODING': 210 if value.lower() == "chunked": 211 chunked = True 212 else: 213 found_headers[uname] = value 214 new_headers.remove((name, value)) 215 216 self.headers = new_headers 217 self.chunked = chunked 218 219 # set body 220 self.set_body(body, found_headers, chunked=chunked) 221 222 self.found_headers = found_headers 223 224 # Finally do the request 225 return self.do_send()
226
227 - def _req_headers(self):
228 # by default all connections are HTTP/1.1 229 if self.version == (1,1): 230 httpver = "HTTP/1.1" 231 else: 232 httpver = "HTTP/1.0" 233 234 # request path 235 path = self.uri.path or "/" 236 req_path = urlparse.urlunparse(('','', path, self.uri.params, 237 self.uri.query, self.uri.fragment)) 238 239 240 ua = self.found_headers.get('USER-AGENT') 241 accept_encoding = self.found_headers.get('ACCEPT-ENCODING') 242 connection = self.found_headers.get('CONNECTION') 243 244 # default host header 245 try: 246 host = self.uri.netloc.encode('ascii') 247 except UnicodeEncodeError: 248 host = self.uri.netloc.encode('idna') 249 host = self.found_headers.get('HOST') or host 250 251 # build final request headers 252 req_headers = [ 253 "%s %s %s\r\n" % (self.method, req_path, httpver), 254 "Host: %s\r\n" % host, 255 "User-Agent: %s\r\n" % ua or USER_AGENT, 256 "Accept-Encoding: %s\r\n" % accept_encoding or 'identity' 257 ] 258 259 if connection is not None: 260 req_headers.append("Connection: %s\r\n" % connection) 261 262 req_headers.extend(["%s: %s\r\n" % (k, v) for k, v in self.headers]) 263 req_headers.append('\r\n') 264 return req_headers
265
266 - def shutdown_connection(self):
267 self._pool.shutdown() 268 if not self._conn: 269 return 270 self._conn.close() 271 self._conn = None
272
273 - def do_send(self):
274 addr = (self.host, self.port) 275 is_ssl = (self.uri.scheme == "https") 276 route = (addr, is_ssl, self.filters, self.ssl_args) 277 self._pool = self.conn_manager.get_pool(route) 278 tries = 2 279 while True: 280 try: 281 if not self._conn: 282 # get new connection 283 self._conn = self._pool.request() 284 # socket 285 s = self._conn.socket() 286 self.headers.extend(self._conn.headers) 287 288 # apply on_request filters 289 self.filters.apply("on_request", self, tries) 290 291 # build request headers 292 self.req_headers = req_headers = self._req_headers() 293 294 # send request 295 log.info('Start request: %s %s', self.method, self.url) 296 log.debug("Request headers: [%s]", req_headers) 297 298 s.sendall("".join(req_headers)) 299 300 if self.body is not None: 301 if hasattr(self.body, 'read'): 302 if hasattr(self.body, 'seek'): self.body.seek(0) 303 sock.sendfile(s, self.body, self.chunked) 304 elif isinstance(self.body, types.StringTypes): 305 sock.send(s, self.body, self.chunked) 306 else: 307 sock.sendlines(s, self.body, self.chunked) 308 309 if self.chunked: # final chunk 310 sock.send_chunk(s, "") 311 312 return self.start_response() 313 except socket.gaierror, e: 314 self.shutdown_connection() 315 raise RequestError(str(e)) 316 except socket.timeout, e: 317 self.shutdown_connection() 318 raise RequestTimeout(str(e)) 319 except socket.error, e: 320 if e[0] not in (errno.EAGAIN, errno.ECONNABORTED, 321 errno.EPIPE, errno.ECONNREFUSED, 322 errno.ECONNRESET) or tries <= 0: 323 self.shutdown_connection() 324 raise RequestError(str(e)) 325 if e[0] in (errno.EPIPE, errno.ECONNRESET): 326 self.shutdown_connection() 327 except (KeyboardInterrupt, SystemExit): 328 break 329 except: 330 if tries < 0: 331 raise 332 # we don't know what happend. 333 self.shutdown_connection() 334 time.sleep(0.2) 335 tries -= 1
336
337 - def do_redirect(self, response, location):
338 """ follow redirections if needed""" 339 if self.nb_redirections <= 0: 340 raise RedirectLimit("Redirection limit is reached") 341 342 if not location: 343 raise RequestError('no Location header') 344 345 new_uri = urlparse.urlparse(location) 346 if not new_uri.netloc: # we got a relative url 347 absolute_uri = "%s://%s" % (self.uri.scheme, self.uri.netloc) 348 location = urlparse.urljoin(absolute_uri, location) 349 350 log.debug("Redirect to %s" % location) 351 352 self.final_url = location 353 response.body.read() 354 self.nb_redirections -= 1 355 if response.should_close: 356 self._conn.close() 357 self._conn = None 358 return self.request(location, self.method, self.body, self.init_headers)
359
360 - def start_response(self):
361 """ 362 Get headers, set Body object and return HttpResponse 363 """ 364 # read headers 365 while True: 366 parser = http.ResponseParser(self._conn, 367 release_source=self._pool.release, 368 decompress=self.decompress) 369 resp = parser.next() 370 if resp.status_int != 100: 371 break 372 373 log.debug("Start response: %s", resp.status) 374 log.debug("Response headers: [%s]", resp.headers) 375 376 location = None 377 for hdr_name, hdr_value in resp.headers: 378 if hdr_name.lower() == "location": 379 location = hdr_value 380 break 381 382 if self.follow_redirect: 383 if resp.status_int in (301, 302, 307): 384 if self.method in ('GET', 'HEAD') or \ 385 self.force_follow_redirect: 386 if self.method not in ('GET', 'HEAD') and \ 387 hasattr(self.body, 'seek'): 388 self.body.seek(0) 389 return self.do_redirect(resp, location) 390 elif resp.status_int == 303 and self.method in ('GET', 391 'HEAD'): 392 # only 'GET' is possible with this status 393 # according the rfc 394 return self.do_redirect(resp, location) 395 396 397 # apply on response filters 398 self.filters.apply("on_response", self) 399 400 self.final_url = location or self.final_url 401 log.debug("Return response: %s" % self.final_url) 402 if self.method == "HEAD": 403 resp.body = StringIO() 404 self._conn.close() 405 self._conn = None 406 return self.response_class(resp, self.final_url)
407