Package restkit :: Module client
[hide private]
[frames] | no frames]

Source Code for Module restkit.client

  1  # -*- coding: utf-8 - 
  2  # 
  3  # This file is part of restkit released under the MIT license.  
  4  # See the NOTICE for more information. 
  5   
  6  import cgi 
  7  import errno 
  8  import logging 
  9  import mimetypes 
 10  import os 
 11  import socket 
 12  import time 
 13  try: 
 14      from cStringIO import StringIO 
 15  except ImportError: 
 16      from StringIO import StringIO 
 17  import types 
 18  import urlparse 
 19  import uuid 
 20   
 21  from restkit import __version__ 
 22  from restkit.errors import RequestError, InvalidUrl, RedirectLimit, \ 
 23  AlreadyRead 
 24  from restkit.filters import Filters 
 25  from restkit.forms import multipart_form_encode, form_encode 
 26  from restkit.util import sock 
 27  from restkit import tee 
 28  from restkit import util 
 29  from restkit.util.misc import deprecated_property 
 30  from restkit import http 
 31   
 32  MAX_FOLLOW_REDIRECTS = 5 
 33   
 34  USER_AGENT = "restkit/%s" % __version__ 
 35   
 36  log = logging.getLogger(__name__) 
37 38 -class HttpResponse(object):
39 """ Http Response object returned by HttpConnction""" 40 41 charset = "utf8" 42 unicode_errors = 'strict' 43
44 - def __init__(self, response, final_url):
45 self.response = response 46 self.status = response.status 47 self.status_int = response.status_int 48 self.version = response.version 49 self.headerslist = response.headers 50 self.final_url = final_url 51 52 headers = {} 53 for key, value in response.headers: 54 headers[key.lower()] = value 55 self.headers = headers 56 self.closed = False
57
58 - def __getitem__(self, key):
59 try: 60 return getattr(self, key) 61 except AttributeError: 62 pass 63 return self.headers[key.lower()]
64
65 - def __contains__(self, key):
66 return (key.lower() in self.headers)
67
68 - def __iter__(self):
69 for item in list(self.headers.items()): 70 yield item
71
72 - def body_string(self, charset=None, unicode_errors="strict"):
73 """ return body string, by default in bytestring """ 74 if self.closed or self.response.body.closed: 75 raise AlreadyRead("The response have already been read") 76 body = self.response.body.read() 77 if charset is not None: 78 try: 79 body = body.decode(charset, unicode_errors) 80 except UnicodeDecodeError: 81 pass 82 self.close() 83 return body
84
85 - def body_stream(self):
86 """ return full body stream """ 87 if self.closed or self.response.body.closed: 88 raise AlreadyRead("The response have already been read") 89 return self.response.body
90
91 - def close(self):
92 """ release the socket """ 93 self.closed = True 94 self.response.body.close()
95 96 @property
97 - def body(self):
98 """ body in bytestring """ 99 return self.body_string()
100 101 body = deprecated_property( 102 body, 'body', 'use body_string() instead', 103 warning=True) 104 105 @property
106 - def body_file(self):
107 """ return body as a file like object""" 108 return self.body_stream()
109 110 body_file = deprecated_property( 111 body_file, 'body_file', 'use body_stream() instead', 112 warning=True) 113 114 @property
115 - def unicode_body(self):
116 """ like body but converted to unicode""" 117 if not self.charset: 118 raise AttributeError( 119 "You cannot access HttpResponse.unicode_body unless charset is set") 120 body = self.body_string() 121 return body.decode(self.charset, self.unicode_errors)
122 123 unicode_body = deprecated_property( 124 unicode_body, 'unicode_body', 'replaced by body_string()', 125 warning=True)
126
127 -class HttpConnection(object):
128 """ Http Connection object. """ 129 130 version = (1, 1) 131 response_class = HttpResponse 132
133 - def __init__(self, timeout=sock._GLOBAL_DEFAULT_TIMEOUT, 134 filters=None, follow_redirect=False, force_follow_redirect=False, 135 max_follow_redirect=MAX_FOLLOW_REDIRECTS, 136 pool_instance=None, response_class=None, 137 **ssl_args):
138 139 """ HttpConnection constructor 140 141 :param timeout: socket timeout 142 :param filters: list, list of http filters. see the doc of http filters 143 for more info 144 :param follow_redirect: boolean, by default is false. If true, 145 if the HTTP status is 301, 302 or 303 the client will follow 146 the location. 147 :param max_follow_redirect: max number of redirection. If max is reached 148 the RedirectLimit exception is raised. 149 :param pool_instance: a pool instance inherited from 150 `restkit.pool.PoolInterface` 151 :param ssl_args: ssl arguments. See http://docs.python.org/library/ssl.html 152 for more information. 153 """ 154 self._sock = None 155 self.timeout = timeout 156 self.headers = [] 157 self.req_headers = [] 158 self.ua = USER_AGENT 159 self.url = None 160 161 self.follow_redirect = follow_redirect 162 self.nb_redirections = max_follow_redirect 163 self.force_follow_redirect = force_follow_redirect 164 self.method = 'GET' 165 self.body = None 166 self.response_body = StringIO() 167 self.final_url = None 168 169 # build filter lists 170 self.filters = Filters(filters) 171 self.ssl_args = ssl_args or {} 172 173 if not pool_instance: 174 self.should_close = True 175 self.pool = None 176 else: 177 self.pool = pool_instance 178 self.should_close = False 179 180 if response_class is not None: 181 self.response_class = response_class
182
183 - def make_connection(self):
184 """ initate a connection if needed or reuse a socket""" 185 186 # apply on connect filters 187 self.filters.apply("on_connect", self) 188 if self._sock is not None: 189 return self._sock 190 191 addr = (self.host, self.port) 192 s = None 193 # if we defined a pool use it 194 if self.pool is not None: 195 s = self.pool.get(addr) 196 197 if not s: 198 # pool is empty or we don't use a pool 199 if self.uri.scheme == "https": 200 s = sock.connect(addr, True, self.timeout, **self.ssl_args) 201 else: 202 s = sock.connect(addr, False, self.timeout) 203 return s
204
205 - def clean_connections(self):
206 sock.close(self._sock) 207 self._sock = None 208 if hasattr(self.pool,'clear'): 209 self.pool.clear_host((self.host, self.port))
210
211 - def release_connection(self, address, socket):
212 if self.should_close: 213 sock.close(self._sock) 214 else: 215 self.pool.put(address, self._sock) 216 self._sock = None
217
218 - def parse_url(self, url):
219 """ parse url and get host/port""" 220 self.uri = urlparse.urlparse(url) 221 if self.uri.scheme not in ('http', 'https'): 222 raise InvalidUrl("None valid url") 223 224 host, port = util.parse_netloc(self.uri) 225 self.host = host 226 self.port = port
227
228 - def set_body(self, body, content_type=None, content_length=None, 229 chunked=False):
230 """ set HTTP body and manage form if needed """ 231 if not body: 232 if content_type is not None: 233 self.headers.append(('Content-Type', content_type)) 234 if self.method in ('POST', 'PUT'): 235 self.headers.append(("Content-Length", "0")) 236 return 237 238 # set content lengh if needed 239 if isinstance(body, dict): 240 if content_type is not None and \ 241 content_type.startswith("multipart/form-data"): 242 type_, opts = cgi.parse_header(content_type) 243 boundary = opts.get('boundary', uuid.uuid4().hex) 244 body, self.headers = multipart_form_encode(body, 245 self.headers, boundary) 246 else: 247 content_type = "application/x-www-form-urlencoded; charset=utf-8" 248 body = form_encode(body) 249 elif hasattr(body, "boundary"): 250 content_type = "multipart/form-data; boundary=%s" % body.boundary 251 content_length = body.get_size() 252 253 if not content_type: 254 content_type = 'application/octet-stream' 255 if hasattr(body, 'name'): 256 content_type = mimetypes.guess_type(body.name)[0] 257 258 if not content_length: 259 if hasattr(body, 'fileno'): 260 try: 261 body.flush() 262 except IOError: 263 pass 264 content_length = str(os.fstat(body.fileno())[6]) 265 elif hasattr(body, 'getvalue'): 266 try: 267 content_length = str(len(body.getvalue())) 268 except AttributeError: 269 pass 270 elif isinstance(body, types.StringTypes): 271 body = util.to_bytestring(body) 272 content_length = len(body) 273 274 if content_length: 275 self.headers.append(("Content-Length", content_length)) 276 if content_type is not None: 277 self.headers.append(('Content-Type', content_type)) 278 279 elif not chunked: 280 raise RequestError("Can't determine content length and" + 281 "Transfer-Encoding header is not chunked") 282 283 self.body = body
284 285
286 - def request(self, url, method='GET', body=None, headers=None):
287 """ make effective request 288 289 :param url: str, url string 290 :param method: str, by default GET. http verbs 291 :param body: the body, could be a string, an iterator or a file-like object 292 :param headers: dict or list of tupple, http headers 293 """ 294 self._sock = None 295 self.url = url 296 self.final_url = url 297 self.parse_url(url) 298 self.method = method.upper() 299 self.headers = [] 300 301 # headers are better as list 302 headers = headers or [] 303 if isinstance(headers, dict): 304 headers = list(headers.items()) 305 306 ua = USER_AGENT 307 content_length = None 308 accept_encoding = 'identity' 309 chunked = False 310 content_type = None 311 connection = None 312 313 314 # default host 315 try: 316 host = self.uri.netloc.encode('ascii') 317 except UnicodeEncodeError: 318 host = self.uri.netloc.encode('idna') 319 320 # normalize headers 321 for name, value in headers: 322 name = name.title() 323 if name == "User-Agent": 324 ua = value 325 elif name == "Content-Type": 326 content_type = value 327 elif name == "Content-Length": 328 content_length = str(value) 329 elif name == "Accept-Encoding": 330 accept_encoding = value 331 elif name == "Host": 332 host = value 333 elif name == "Transfer-Encoding": 334 if value.lower() == "chunked": 335 chunked = True 336 self.headers.append((name, value)) 337 elif name == "Connection": 338 connection = value 339 else: 340 if not isinstance(value, types.StringTypes): 341 value = str(value) 342 self.headers.append((name, value)) 343 344 self.set_body(body, content_type=content_type, 345 content_length=content_length, chunked=chunked) 346 347 self.ua = ua 348 self.chunked = chunked 349 self.host_hdr = host 350 self.accept_encoding = accept_encoding 351 if connection == "close": 352 self.should_close = True 353 elif self.pool is not None: 354 self.should_close = False 355 356 # Finally do the request 357 return self.do_send()
358
359 - def _req_headers(self):
360 # by default all connections are HTTP/1.1 361 if self.version == (1,1): 362 httpver = "HTTP/1.1" 363 else: 364 httpver = "HTTP/1.0" 365 366 # request path 367 path = self.uri.path or "/" 368 req_path = urlparse.urlunparse(('','', path, '', 369 self.uri.query, self.uri.fragment)) 370 371 # build final request headers 372 req_headers = [ 373 "%s %s %s\r\n" % (self.method, req_path, httpver), 374 "Host: %s\r\n" % self.host_hdr, 375 "User-Agent: %s\r\n" % self.ua, 376 "Accept-Encoding: %s\r\n" % self.accept_encoding 377 ] 378 req_headers.extend(["%s: %s\r\n" % (k, v) for k, v in self.headers]) 379 req_headers.append('\r\n') 380 return req_headers
381
382 - def do_send(self):
383 tries = 2 384 while True: 385 try: 386 # get socket 387 self._sock = self.make_connection() 388 389 # apply on request filters 390 self.filters.apply("on_request", self, tries) 391 392 # build request headers 393 self.req_headers = req_headers = self._req_headers() 394 395 # send request 396 log.info('Start request: %s %s', self.method, self.url) 397 log.debug("Request headers: [%s]", req_headers) 398 399 self._sock.sendall("".join(req_headers)) 400 401 if self.body is not None: 402 if hasattr(self.body, 'read'): 403 if hasattr(self.body, 'seek'): self.body.seek(0) 404 sock.sendfile(self._sock, self.body, self.chunked) 405 elif isinstance(self.body, types.StringTypes): 406 sock.send(self._sock, self.body, self.chunked) 407 else: 408 sock.sendlines(self._sock, self.body, self.chunked) 409 410 if self.chunked: # final chunk 411 sock.send_chunk(self._sock, "") 412 413 return self.start_response() 414 except socket.gaierror, e: 415 self.clean_connections() 416 raise 417 except socket.error, e: 418 if e[0] not in (errno.EAGAIN, errno.ECONNABORTED, errno.EPIPE, 419 errno.ECONNREFUSED, errno.ECONNRESET) or tries <= 0: 420 self.clean_connections() 421 raise 422 if e[0] in (errno.EPIPE, errno.ECONNRESET): 423 self.clean_connections() 424 except Exception, e: 425 if isinstance(e, (KeyboardInterrupt, SystemExit)): 426 raise 427 428 if tries <= 0: 429 raise 430 # we don't know what happend. 431 self.clean_connections() 432 time.sleep(0.2) 433 tries -= 1
434 435
436 - def do_redirect(self, response, location):
437 """ follow redirections if needed""" 438 if self.nb_redirections <= 0: 439 raise RedirectLimit("Redirection limit is reached") 440 441 if not location: 442 raise RequestError('no Location header') 443 444 new_uri = urlparse.urlparse(location) 445 if not new_uri.netloc: # we got a relative url 446 absolute_uri = "%s://%s" % (self.uri.scheme, self.uri.netloc) 447 location = urlparse.urljoin(absolute_uri, location) 448 449 log.debug("Redirect to %s" % location) 450 451 self.final_url = location 452 response.body.read() 453 self.nb_redirections -= 1 454 sock.close(self._sock) 455 return self.request(location, self.method, self.body, self.headers)
456
457 - def start_response(self):
458 """ 459 Get headers, set Body object and return HttpResponse 460 """ 461 # read headers 462 parser = http.ResponseParser(self._sock, 463 release_source = lambda:self.release_connection( 464 (self.host, self.port), self._sock)) 465 resp = parser.next() 466 467 log.debug("Start response: %s", resp.status) 468 log.debug("Response headers: [%s]", resp.headers) 469 470 location = None 471 for hdr_name, hdr_value in resp.headers: 472 if hdr_name.lower() == "location": 473 location = hdr_value 474 break 475 476 if self.follow_redirect: 477 if resp.status_int in (301, 302, 307): 478 if self.method in ('GET', 'HEAD') or \ 479 self.force_follow_redirect: 480 if self.method not in ('GET', 'HEAD') and \ 481 hasattr(self.body, 'seek'): 482 self.body.seek(0) 483 return self.do_redirect(resp, location) 484 elif resp.status_int == 303 and self.method in ('GET', 485 'HEAD'): 486 # only 'GET' is possible with this status 487 # according the rfc 488 return self.do_redirect(resp, location) 489 490 491 # apply on response filters 492 self.filters.apply("on_response", self) 493 494 self.final_url = location or self.final_url 495 log.debug("Return response: %s" % self.final_url) 496 if self.method == "HEAD": 497 resp.body = StringIO() 498 499 return self.response_class(resp, self.final_url)
500