Package restkit :: Package client :: Module request
[hide private]
[frames] | no frames]

Source Code for Module restkit.client.request

  1  # -*- coding: utf-8 - 
  2  # 
  3  # This file is part of restkit released under the MIT license.  
  4  # See the NOTICE for more information. 
  5   
  6  import copy 
  7  import cgi 
  8  import errno 
  9  import logging 
 10  import mimetypes 
 11  import os 
 12  import socket 
 13  import time 
 14  try: 
 15      from cStringIO import StringIO 
 16  except ImportError: 
 17      from StringIO import StringIO 
 18  import types 
 19  import urlparse 
 20  import uuid 
 21   
 22  from restkit import __version__ 
 23  from restkit.client.response import HttpResponse 
 24  from restkit.errors import RequestError, InvalidUrl, RedirectLimit 
 25  from restkit.filters import Filters 
 26  from restkit.forms import multipart_form_encode, form_encode 
 27  from restkit.util import sock 
 28  from restkit import util 
 29  from restkit import http 
 30   
 31  MAX_FOLLOW_REDIRECTS = 5 
 32   
 33  USER_AGENT = "restkit/%s" % __version__ 
 34   
 35  log = logging.getLogger(__name__) 
 36   
37 -class HttpRequest(object):
38 """ Http Connection object. """ 39 40 version = (1, 1) 41 response_class = HttpResponse 42
43 - def __init__(self, timeout=sock._GLOBAL_DEFAULT_TIMEOUT, 44 filters=None, follow_redirect=False, force_follow_redirect=False, 45 max_follow_redirect=MAX_FOLLOW_REDIRECTS, 46 decompress=True, 47 pool_instance=None, response_class=None, 48 **ssl_args):
49 50 """ HttpRequest constructor 51 52 :param timeout: socket timeout 53 :param filters: list, list of http filters. see the doc of http filters 54 for more info 55 :param follow_redirect: boolean, by default is false. If true, 56 if the HTTP status is 301, 302 or 303 the client will follow 57 the location. 58 :param max_follow_redirect: max number of redirection. If max is reached 59 the RedirectLimit exception is raised. 60 :param pool_instance: a pool instance inherited from 61 `restkit.pool.PoolInterface` 62 :param ssl_args: ssl arguments. See http://docs.python.org/library/ssl.html 63 for more information. 64 """ 65 self._sock = None 66 self.timeout = timeout 67 self.headers = [] 68 self.req_headers = [] 69 self.ua = USER_AGENT 70 self.url = None 71 72 self.follow_redirect = follow_redirect 73 self.nb_redirections = max_follow_redirect 74 self.force_follow_redirect = force_follow_redirect 75 self.decompress = decompress 76 self.method = 'GET' 77 self.body = None 78 self.response_body = StringIO() 79 self.final_url = None 80 81 # build filter lists 82 self.filters = Filters(filters) 83 self.ssl_args = ssl_args or {} 84 85 if not pool_instance: 86 self.should_close = True 87 self.pool = None 88 else: 89 self.pool = pool_instance 90 self.should_close = False 91 92 if response_class is not None: 93 self.response_class = response_class
94
95 - def make_connection(self):
96 """ initate a connection if needed or reuse a socket""" 97 98 # apply on connect filters 99 self.filters.apply("on_connect", self) 100 if self._sock is not None: 101 return self._sock 102 103 addr = (self.host, self.port) 104 s = None 105 # if we defined a pool use it 106 if self.pool is not None: 107 s = self.pool.get(addr) 108 109 if not s: 110 # pool is empty or we don't use a pool 111 if self.uri.scheme == "https": 112 s = sock.connect(addr, True, self.timeout, **self.ssl_args) 113 else: 114 s = sock.connect(addr, False, self.timeout) 115 return s
116
117 - def clean_connections(self):
118 sock.close(self._sock) 119 self._sock = None 120 if hasattr(self.pool,'clear'): 121 self.pool.clear_host((self.host, self.port))
122
123 - def release_connection(self, address, socket):
124 if self.should_close: 125 sock.close(self._sock) 126 else: 127 self.pool.put(address, self._sock) 128 self._sock = None
129
130 - def parse_url(self, url):
131 """ parse url and get host/port""" 132 self.uri = urlparse.urlparse(url) 133 if self.uri.scheme not in ('http', 'https'): 134 raise InvalidUrl("None valid url") 135 136 host, port = util.parse_netloc(self.uri) 137 self.host = host 138 self.port = port
139
140 - def set_body(self, body, headers, chunked=False):
141 """ set HTTP body and manage form if needed """ 142 content_type = headers.get('CONTENT-TYPE') 143 content_length = headers.get('CONTENT-LENGTH') 144 if not body: 145 if content_type is not None: 146 self.headers.append(('Content-Type', content_type)) 147 if self.method in ('POST', 'PUT'): 148 self.headers.append(("Content-Length", "0")) 149 return 150 151 # set content lengh if needed 152 if isinstance(body, dict): 153 if content_type is not None and \ 154 content_type.startswith("multipart/form-data"): 155 type_, opts = cgi.parse_header(content_type) 156 boundary = opts.get('boundary', uuid.uuid4().hex) 157 body, self.headers = multipart_form_encode(body, 158 self.headers, boundary) 159 else: 160 content_type = "application/x-www-form-urlencoded; charset=utf-8" 161 body = form_encode(body) 162 elif hasattr(body, "boundary"): 163 content_type = "multipart/form-data; boundary=%s" % body.boundary 164 content_length = body.get_size() 165 166 if not content_type: 167 content_type = 'application/octet-stream' 168 if hasattr(body, 'name'): 169 content_type = mimetypes.guess_type(body.name)[0] 170 171 if not content_length: 172 if hasattr(body, 'fileno'): 173 try: 174 body.flush() 175 except IOError: 176 pass 177 content_length = str(os.fstat(body.fileno())[6]) 178 elif hasattr(body, 'getvalue'): 179 try: 180 content_length = str(len(body.getvalue())) 181 except AttributeError: 182 pass 183 elif isinstance(body, types.StringTypes): 184 body = util.to_bytestring(body) 185 content_length = len(body) 186 187 if content_length: 188 self.headers.append(("Content-Length", content_length)) 189 if content_type is not None: 190 self.headers.append(('Content-Type', content_type)) 191 192 elif not chunked: 193 raise RequestError("Can't determine content length and " + 194 "Transfer-Encoding header is not chunked") 195 196 self.body = body
197 198
199 - def request(self, url, method='GET', body=None, headers=None):
200 """ make effective request 201 202 :param url: str, url string 203 :param method: str, by default GET. http verbs 204 :param body: the body, could be a string, an iterator or a file-like object 205 :param headers: dict or list of tupple, http headers 206 """ 207 self._sock = None 208 self.url = url 209 self.final_url = url 210 self.parse_url(url) 211 self.method = method.upper() 212 213 self.init_headers = copy.copy(headers or []) 214 self.headers = [] 215 216 # headers are better as list 217 headers = headers or [] 218 if isinstance(headers, dict): 219 headers = headers.items() 220 221 chunked = False 222 223 # normalize headers 224 search_headers = ('USER-AGENT', 'CONTENT-TYPE', 225 'CONTENT-LENGTH', 'ACCEPT-ENCODING', 226 'TRANSFER-ENCODING', 'CONNECTION', 'HOST') 227 found_headers = {} 228 new_headers = copy.copy(headers) 229 for (name, value) in headers: 230 uname = name.upper() 231 if uname in search_headers: 232 if uname == 'TRANSFER-ENCODING': 233 if value.lower() == "chunked": 234 chunked = True 235 else: 236 found_headers[uname] = value 237 new_headers.remove((name, value)) 238 239 self.headers = new_headers 240 self.chunked = chunked 241 242 # set body 243 self.set_body(body, found_headers, chunked=chunked) 244 245 # force connection close if needed 246 if found_headers.get('CONNECTION') == "close": 247 self.should_close = True 248 elif self.pool is None: 249 found_headers['CONNECTION'] = "close" 250 251 self.found_headers = found_headers 252 253 # Finally do the request 254 return self.do_send()
255
256 - def _req_headers(self):
257 # by default all connections are HTTP/1.1 258 if self.version == (1,1): 259 httpver = "HTTP/1.1" 260 else: 261 httpver = "HTTP/1.0" 262 263 # request path 264 path = self.uri.path or "/" 265 req_path = urlparse.urlunparse(('','', path, self.uri.params, 266 self.uri.query, self.uri.fragment)) 267 268 269 ua = self.found_headers.get('USER-AGENT') 270 accept_encoding = self.found_headers.get('ACCEPT-ENCODING') 271 connection = self.found_headers.get('CONNECTION') 272 273 # default host header 274 try: 275 host = self.uri.netloc.encode('ascii') 276 except UnicodeEncodeError: 277 host = self.uri.netloc.encode('idna') 278 host = self.found_headers.get('HOST') or host 279 280 # build final request headers 281 req_headers = [ 282 "%s %s %s\r\n" % (self.method, req_path, httpver), 283 "Host: %s\r\n" % host, 284 "User-Agent: %s\r\n" % ua or USER_AGENT, 285 "Accept-Encoding: %s\r\n" % accept_encoding or 'identity' 286 ] 287 288 if connection is not None: 289 req_headers.append("Connection: %s\r\n" % connection) 290 291 req_headers.extend(["%s: %s\r\n" % (k, v) for k, v in self.headers]) 292 req_headers.append('\r\n') 293 return req_headers
294
295 - def do_send(self):
296 tries = 2 297 while True: 298 try: 299 # get socket 300 self._sock = self.make_connection() 301 302 # apply on request filters 303 self.filters.apply("on_request", self, tries) 304 305 # build request headers 306 self.req_headers = req_headers = self._req_headers() 307 308 # send request 309 log.info('Start request: %s %s', self.method, self.url) 310 log.debug("Request headers: [%s]", req_headers) 311 312 self._sock.sendall("".join(req_headers)) 313 314 if self.body is not None: 315 if hasattr(self.body, 'read'): 316 if hasattr(self.body, 'seek'): self.body.seek(0) 317 sock.sendfile(self._sock, self.body, self.chunked) 318 elif isinstance(self.body, types.StringTypes): 319 sock.send(self._sock, self.body, self.chunked) 320 else: 321 sock.sendlines(self._sock, self.body, self.chunked) 322 323 if self.chunked: # final chunk 324 sock.send_chunk(self._sock, "") 325 326 return self.start_response() 327 except socket.gaierror, e: 328 self.clean_connections() 329 raise 330 except socket.error, e: 331 if e[0] not in (errno.EAGAIN, errno.ECONNABORTED, errno.EPIPE, 332 errno.ECONNREFUSED, errno.ECONNRESET) or tries <= 0: 333 self.clean_connections() 334 raise 335 if e[0] in (errno.EPIPE, errno.ECONNRESET): 336 self.clean_connections() 337 except (KeyboardInterrupt, SystemExit): 338 break 339 except: 340 if tries <= 0: 341 raise 342 # we don't know what happend. 343 self.clean_connections() 344 time.sleep(0.2) 345 tries -= 1
346 347
348 - def do_redirect(self, response, location):
349 """ follow redirections if needed""" 350 if self.nb_redirections <= 0: 351 raise RedirectLimit("Redirection limit is reached") 352 353 if not location: 354 raise RequestError('no Location header') 355 356 new_uri = urlparse.urlparse(location) 357 if not new_uri.netloc: # we got a relative url 358 absolute_uri = "%s://%s" % (self.uri.scheme, self.uri.netloc) 359 location = urlparse.urljoin(absolute_uri, location) 360 361 log.debug("Redirect to %s" % location) 362 363 self.final_url = location 364 response.body.read() 365 self.nb_redirections -= 1 366 367 self.release_connection((self.host, self.port), self._sock) 368 return self.request(location, self.method, self.body, self.init_headers)
369
370 - def start_response(self):
371 """ 372 Get headers, set Body object and return HttpResponse 373 """ 374 # read headers 375 release_fun = lambda:self.release_connection( 376 (self.host, self.port), self._sock) 377 while True: 378 parser = http.ResponseParser(self._sock, 379 release_source=release_fun, 380 decompress=self.decompress) 381 resp = parser.next() 382 if resp.status_int != 100: 383 break 384 385 log.debug("Start response: %s", resp.status) 386 log.debug("Response headers: [%s]", resp.headers) 387 388 location = None 389 for hdr_name, hdr_value in resp.headers: 390 if hdr_name.lower() == "location": 391 location = hdr_value 392 break 393 394 if self.follow_redirect: 395 if resp.status_int in (301, 302, 307): 396 if self.method in ('GET', 'HEAD') or \ 397 self.force_follow_redirect: 398 if self.method not in ('GET', 'HEAD') and \ 399 hasattr(self.body, 'seek'): 400 self.body.seek(0) 401 return self.do_redirect(resp, location) 402 elif resp.status_int == 303 and self.method in ('GET', 403 'HEAD'): 404 # only 'GET' is possible with this status 405 # according the rfc 406 return self.do_redirect(resp, location) 407 408 409 # apply on response filters 410 self.filters.apply("on_response", self) 411 412 self.final_url = location or self.final_url 413 log.debug("Return response: %s" % self.final_url) 414 if self.method == "HEAD": 415 resp.body = StringIO() 416 417 return self.response_class(resp, self.final_url)
418