Package restkit :: Module client
[hide private]
[frames] | no frames]

Source Code for Module restkit.client

  1  # -*- coding: utf-8 - 
  2  # 
  3  # This file is part of restkit released under the MIT license.  
  4  # See the NOTICE for more information. 
  5   
  6  import errno 
  7  import gzip 
  8  import logging 
  9  import os 
 10  import socket 
 11  import time 
 12  from StringIO import StringIO 
 13  import types 
 14  import urlparse 
 15   
 16  from restkit import __version__ 
 17  from restkit.errors import RequestError, InvalidUrl, RedirectLimit 
 18  from restkit.parser import Parser 
 19  from restkit import sock 
 20  from restkit import tee 
 21  from restkit import util 
 22   
 23  MAX_FOLLOW_REDIRECTS = 5 
 24   
 25  USER_AGENT = "restkit/%s" % __version__ 
 26   
 27  log = logging.getLogger(__name__) 
28 29 -class HttpResponse(object):
30 """ Http Response object returned by HttpConnction""" 31 32 charset = "utf8" 33 unicode_errors = 'strict' 34
35 - def __init__(self, http_client):
36 self.http_client = http_client 37 self.status = self.http_client.parser.status 38 self.status_int = self.http_client.parser.status_int 39 self.version = self.http_client.parser.version 40 self.headerslist = self.http_client.parser.headers 41 self.final_url = self.http_client.final_url 42 43 headers = {} 44 for key, value in self.http_client.parser.headers: 45 headers[key.lower()] = value 46 self.headers = headers 47 48 encoding = headers.get('content-encoding', None) 49 if encoding in ('gzip', 'deflate'): 50 self._body = gzip.GzipFile(fileobj=self.http_client.response_body) 51 else: 52 self._body = self.http_client.response_body 53 self._body_eof = False
54
55 - def __getitem__(self, key):
56 try: 57 return getattr(self, key) 58 except AttributeError: 59 pass 60 return self.headers[key]
61
62 - def __contains__(self, key):
63 return (key in self.headers)
64
65 - def __iter__(self):
66 for item in list(self.headers.items()): 67 yield item
68 69 @property
70 - def body(self):
71 """ body in bytestring """ 72 if self._body_eof: 73 self._body.seek(0) 74 self._body_eof = True 75 ret = self._body.read() 76 self._body.seek(0) 77 return ret
78 79 @property
80 - def body_file(self):
81 """ return body as a file like object""" 82 return self._body
83 84 @property
85 - def unicode_body(self):
86 """ like body but converted to unicode""" 87 if not self.charset: 88 raise AttributeError( 89 "You cannot access HttpResponse.unicode_body unless charset is set") 90 return self.body.decode(self.charset, self.unicode_errors)
91
92 -class HttpConnection(object):
93 """ Http Connection object. """ 94 95 version = (1, 1) 96 response_class = HttpResponse 97 98
99 - def __init__(self, timeout=sock._GLOBAL_DEFAULT_TIMEOUT, 100 filters=None, follow_redirect=False, force_follow_redirect=False, 101 max_follow_redirect=MAX_FOLLOW_REDIRECTS, key_file=None, 102 cert_file=None, pool_instance=None, response_class=None):
103 104 """ HttpConnection constructor 105 106 :param timeout: socket timeout 107 :param filters: list, list of http filters. see the doc of http filters 108 for more info 109 :param follow_redirect: boolean, by default is false. If true, 110 if the HTTP status is 301, 302 or 303 the client will follow 111 the location. 112 :param max_follow_redirect: max number of redirection. If max is reached 113 the RedirectLimit exception is raised. 114 :param key_file: the key fle to use with ssl 115 :param cert_file: the cert file to use with ssl 116 :param pool_instance: a pool instance inherited from 117 `restkit.pool.PoolInterface` 118 """ 119 self._sock = None 120 self.timeout = timeout 121 self.headers = [] 122 self.req_headers = [] 123 self.ua = USER_AGENT 124 self.url = None 125 126 self.follow_redirect = follow_redirect 127 self.nb_redirections = max_follow_redirect 128 self.force_follow_redirect = force_follow_redirect 129 self.method = 'GET' 130 self.body = None 131 self.response_body = StringIO() 132 self.final_url = None 133 134 # build filter lists 135 self.filters = filters or [] 136 self.request_filters = [] 137 self.response_filters = [] 138 self.cert_file = cert_file 139 self.key_file = key_file 140 141 for f in self.filters: 142 self._add_filter(f) 143 144 if not pool_instance: 145 self.should_close = True 146 self.connections = None 147 else: 148 self.connections = pool_instance 149 self.should_close = False 150 151 if response_class is not None: 152 self.response_class = response_class
153
154 - def add_filter(self, f):
155 self.filters.append(f) 156 self._add_filter(f)
157
158 - def _add_filter(self, f):
159 if hasattr(f, 'on_request'): 160 self.request_filters.append(f) 161 162 if hasattr(f, 'on_response'): 163 self.response_filters.append(f)
164
165 - def remove_filter(self, f):
166 for i, f1 in enumerate(self.filters): 167 if f == f1: del self.filters[i] 168 169 if hasattr(f, 'on_request'): 170 for i, f1 in enumerate(self.request_filters): 171 if f == f1: del self.request_filters[i] 172 173 if hasattr(f, 'on_response'): 174 for i, f1 in enumerate(self.response_filters): 175 if f == f1: del self.response_filters[i]
176
177 - def make_connection(self):
178 """ initate a connection if needed or reuse a socket""" 179 addr = (self.host, self.port) 180 s = None 181 # if we defined a pool use it 182 if self.connections is not None: 183 s = self.connections.get(addr) 184 185 if not s: 186 # pool is empty or we don't use a pool 187 if self.uri.scheme == "https": 188 s = sock.connect(addr, self.timeout, True, 189 self.key_file, self.cert_file) 190 else: 191 s = sock.connect(addr, self.timeout) 192 return s
193
194 - def clean_connections(self):
195 sock.close(self._sock) 196 if hasattr(self.connections,'clean'): 197 self.connections.clean((self.host, self.port))
198
199 - def release_connection(self, address, socket):
200 if not self.connections: 201 sock.close(socket) 202 else: 203 self.connections.put(address, self._sock)
204
205 - def parse_url(self, url):
206 """ parse url and get host/port""" 207 self.uri = urlparse.urlparse(url) 208 if self.uri.scheme not in ('http', 'https'): 209 raise InvalidUrl("None valid url") 210 211 host = self.uri.netloc 212 i = host.rfind(':') 213 j = host.rfind(']') # ipv6 addresses have [...] 214 if i > j: 215 try: 216 port = int(host[i+1:]) 217 except ValueError: 218 raise InvalidUrl("nonnumeric port: '%s'" % host[i+1:]) 219 host = host[:i] 220 else: 221 # default port 222 if self.uri.scheme == "https": 223 port = 443 224 else: 225 port = 80 226 227 if host and host[0] == '[' and host[-1] == ']': 228 host = host[1:-1] 229 230 self.host = host 231 self.port = port
232
233 - def request(self, url, method='GET', body=None, headers=None):
234 """ make effective request 235 236 :param url: str, url string 237 :param method: str, by default GET. http verbs 238 :param body: the body, could be a string, an iterator or a file-like object 239 :param headers: dict or list of tupple, http headers 240 """ 241 self.parser = Parser.parse_response(should_close=self.should_close) 242 self._sock = None 243 self.url = url 244 self.final_url = url 245 self.parse_url(url) 246 self.method = method.upper() 247 248 249 # headers are better as list 250 headers = headers or [] 251 if isinstance(headers, dict): 252 headers = list(headers.items()) 253 254 ua = USER_AGENT 255 normalized_headers = [] 256 content_len = None 257 accept_encoding = 'identity' 258 chunked = False 259 260 # default host 261 try: 262 host = self.uri.netloc.encode('ascii') 263 except UnicodeEncodeError: 264 host = self.uri.netloc.encode('idna') 265 266 # normalize headers 267 for name, value in headers: 268 name = name.title() 269 if name == "User-Agent": 270 ua = value 271 elif name == "Content-Length": 272 content_len = str(value) 273 elif name == "Accept-Encoding": 274 accept_encoding = 'identity' 275 elif name == "Host": 276 host = value 277 elif name == "Transfer-Encoding": 278 if value.lower() == "chunked": 279 chunked = True 280 normalized_headers.append((name, value)) 281 else: 282 if not isinstance(value, types.StringTypes): 283 value = str(value) 284 normalized_headers.append((name, value)) 285 286 # set content lengh if needed 287 if body and body is not None: 288 if not content_len: 289 if hasattr(body, 'fileno'): 290 try: 291 body.flush() 292 except IOError: 293 pass 294 content_len = str(os.fstat(body.fileno())[6]) 295 elif hasattr(body, 'len'): 296 try: 297 content_len = str(body.len) 298 except AttributeError: 299 pass 300 elif isinstance(body, types.StringTypes): 301 body = util.to_bytestring(body) 302 content_len = len(body) 303 304 if content_len: 305 normalized_headers.append(("Content-Length", content_len)) 306 elif not chunked: 307 raise RequestError("Can't determine content length and" + 308 "Transfer-Encoding header is not chunked") 309 310 if self.method in ('POST', 'PUT') and not body: 311 normalized_headers.append(("Content-Length", "0")) 312 313 self.body = body 314 self.headers = normalized_headers 315 self.ua = ua 316 self.chunked = chunked 317 self.host_hdr = host 318 self.accept_encoding = accept_encoding 319 320 # Finally do the request 321 return self.do_send()
322
323 - def _req_headers(self):
324 # by default all connections are HTTP/1.1 325 if self.version == (1,1): 326 httpver = "HTTP/1.1" 327 else: 328 httpver = "HTTP/1.0" 329 330 # request path 331 path = self.uri.path or "/" 332 req_path = urlparse.urlunparse(('','', path, '', 333 self.uri.query, self.uri.fragment)) 334 335 # build final request headers 336 req_headers = [ 337 "%s %s %s\r\n" % (self.method, req_path, httpver), 338 "Host: %s\r\n" % self.host_hdr, 339 "User-Agent: %s\r\n" % self.ua, 340 "Accept-Encoding: %s\r\n" % self.accept_encoding 341 ] 342 req_headers.extend(["%s:%s\r\n" % (k, v) for k, v in self.headers]) 343 req_headers.append('\r\n') 344 return req_headers
345
346 - def do_send(self):
347 tries = 2 348 while True: 349 try: 350 # get socket 351 self._sock = self.make_connection() 352 353 # apply on request filters 354 for bf in self.request_filters: 355 bf.on_request(self) 356 357 # build request headers 358 self.req_headers = req_headers = self._req_headers() 359 360 # send request 361 log.info('Start request: %s %s' % (self.method, self.url)) 362 log.debug("Request headers: [%s]" % str(req_headers)) 363 364 self._sock.sendall("".join(req_headers)) 365 366 if self.body is not None: 367 if hasattr(self.body, 'read'): 368 if hasattr(self.body, 'seek'): self.body.seek(0) 369 sock.sendfile(self._sock, self.body, self.chunked) 370 elif isinstance(self.body, types.StringTypes): 371 sock.send(self._sock, self.body, self.chunked) 372 else: 373 sock.sendlines(self._sock, self.body, self.chunked) 374 375 if self.chunked: # final chunk 376 sock.send_chunk(self._sock, "") 377 378 return self.start_response() 379 except socket.gaierror, e: 380 self.clean_connections() 381 raise 382 except socket.error, e: 383 if e[0] not in (errno.EAGAIN, errno.ECONNABORTED, errno.EPIPE, 384 errno.ECONNREFUSED) or tries <= 0: 385 self.clean_connections() 386 raise 387 if e[0] == errno.EPIPE: 388 log.debug("Got EPIPE") 389 self.clean_connections() 390 except: 391 if tries <= 0: 392 raise 393 # we don't know what happend. 394 self.clean_connections() 395 time.sleep(0.2) 396 tries -= 1
397 398
399 - def do_redirect(self):
400 """ follow redirections if needed""" 401 if self.nb_redirections <= 0: 402 raise RedirectLimit("Redirection limit is reached") 403 404 location = self.parser.headers_dict.get('Location') 405 if not location: 406 raise RequestError('no Location header') 407 408 new_uri = urlparse.urlparse(location) 409 if not new_uri.netloc: # we got a relative url 410 absolute_uri = "%s://%s" % (self.uri.scheme, self.uri.netloc) 411 location = urlparse.urljoin(absolute_uri, location) 412 413 log.debug("Redirect to %s" % location) 414 415 self.final_url = location 416 self.response_body.read() 417 self.nb_redirections -= 1 418 sock.close(self._sock) 419 return self.request(location, self.method, self.body, self.headers)
420
421 - def start_response(self):
422 """ 423 Get headers, set Body object and return HttpResponse 424 """ 425 # read headers 426 headers = [] 427 buf = StringIO() 428 data = self._sock.recv(sock.CHUNK_SIZE) 429 buf.write(data) 430 buf2 = self.parser.filter_headers(headers, buf) 431 if not buf2: 432 while True: 433 data = self._sock.recv(sock.CHUNK_SIZE) 434 if not data: 435 break 436 buf.write(data) 437 buf2 = self.parser.filter_headers(headers, buf) 438 if buf2: 439 break 440 441 log.debug("Start response: %s" % str(self.parser.status_line)) 442 log.debug("Response headers: [%s]" % str(self.parser.headers)) 443 444 if (not self.parser.content_len and not self.parser.is_chunked): 445 if self.parser.should_close: 446 # http 1.0 or something like it. 447 # we try to get missing body 448 log.debug("No content len an not chunked transfer, get body") 449 while True: 450 try: 451 chunk = self._sock.recv(sock.CHUNK_SIZE) 452 except socket.error: 453 break 454 if not chunk: 455 break 456 buf2.write(chunk) 457 sock.close(self._sock) 458 buf2.seek(0) 459 self.response_body = buf2 460 461 elif self.method == "HEAD": 462 self.response_body = StringIO() 463 sock.close(self._sock) 464 else: 465 self.response_body = tee.TeeInput(self._sock, self.parser, buf2, 466 maybe_close=lambda: self.release_connection( 467 self.uri.netloc, self._sock)) 468 469 # apply on response filters 470 for af in self.response_filters: 471 af.on_response(self) 472 473 if self.follow_redirect: 474 if self.parser.status_int in (301, 302, 307): 475 if self.method in ('GET', 'HEAD') or \ 476 self.force_follow_redirect: 477 if self.method not in ('GET', 'HEAD') and \ 478 hasattr(self.body, 'seek'): 479 self.body.seek(0) 480 return self.do_redirect() 481 elif self.parser.status_int == 303 and self.method in ('GET', 482 'HEAD'): 483 # only 'GET' is possible with this status 484 # according the rfc 485 return self.do_redirect() 486 487 488 self.final_url = self.parser.headers_dict.get('Location', 489 self.final_url) 490 log.debug("Return response: %s" % self.final_url) 491 return self.response_class(self)
492