Package restkit :: Module client
[hide private]
[frames] | no frames]

Source Code for Module restkit.client

  1  # -*- coding: utf-8 - 
  2  # 
  3  # This file is part of restkit released under the MIT license.  
  4  # See the NOTICE for more information. 
  5   
  6  import copy 
  7  import errno 
  8  import logging 
  9  import mimetypes 
 10  import os 
 11  import time 
 12  import socket 
 13  import types 
 14  import urlparse 
 15  import uuid 
 16   
 17  try: 
 18      from cStringIO import StringIO 
 19  except ImportError: 
 20      from StringIO import StringIO 
 21   
 22  try: 
 23      import ssl # python 2.6 
 24      have_ssl = True 
 25  except ImportError: 
 26      have_ssl = False 
 27   
 28  from . import __version__  
 29  from .datastructures import MultiDict 
 30  from .errors import * 
 31  from .filters import Filters 
 32  from .forms import multipart_form_encode, form_encode 
 33  from .globals import get_manager  
 34  from . import http 
 35   
 36  from .sock import close, send, sendfile, sendlines, send_chunk 
 37  from .tee import TeeInput 
 38  from .util import parse_netloc, to_bytestring, rewrite_location 
 39   
 40  MAX_CLIENT_TIMEOUT=300 
 41  MAX_CLIENT_CONNECTIONS = 5 
 42  MAX_CLIENT_TRIES = 5 
 43  CLIENT_WAIT_TRIES = 0.3 
 44  MAX_FOLLOW_REDIRECTS = 5 
 45  USER_AGENT = "restkit/%s" % __version__ 
 46   
 47  log = logging.getLogger(__name__) 
 48   
49 -class BodyWrapper(object):
50
51 - def __init__(self, resp, client):
52 self.resp = resp 53 self.body = resp._body 54 self.client = client 55 self._sock = client._sock 56 self._sock_key = copy.copy(client._sock_key)
57
58 - def __enter__(self):
59 return self
60
61 - def __exit__(self, exc_type, exc_val, traceback):
62 self.close()
63
64 - def close(self):
65 """ release connection """ 66 self.client.release_connection(self._sock_key, 67 self._sock, self.resp.should_close)
68
69 - def __iter__(self):
70 return self
71
72 - def next(self):
73 try: 74 return self.body.next() 75 except StopIteration: 76 self.close() 77 raise
78
79 - def read(self, size=None):
80 data = self.body.read(size=size) 81 if not data: 82 self.close() 83 return data
84
85 - def readline(self, size=None):
86 line = self.body.readline(size=size) 87 if not line: 88 self.close() 89 return line
90
91 - def readlines(self, size=None):
92 lines = self.body.readlines(size=size) 93 if self.body.close: 94 self.close() 95 return line
96 97
98 -class ClientResponse(object):
99 100 charset = "utf8" 101 unicode_errors = 'strict' 102
103 - def __init__(self, client, resp):
104 self.client = client 105 self._sock = client._sock 106 self._sock_key = copy.copy(client._sock_key) 107 self._body = resp.body 108 109 # response infos 110 self.headers = resp.headers 111 self.status = resp.status 112 self.status_int = resp.status_int 113 self.version = resp.version 114 self.headerslist = resp.headers.items() 115 self.location = resp.headers.iget('location') 116 self.final_url = client.url 117 self.should_close = resp.should_close() 118 119 120 self._closed = False 121 self._already_read = False 122 123 if client.method == "HEAD": 124 """ no body on HEAD, release the connection now """ 125 self._body = StringIO()
126
127 - def __getitem__(self, key):
128 try: 129 return getattr(self, key) 130 except AttributeError: 131 pass 132 return self.headers.iget(key)
133
134 - def __contains__(self, key):
135 return (self.headers.iget(key) is not None)
136
137 - def __iter__(self):
138 return self.headers.iteritems()
139
140 - def release_connection(self):
141 """ release the connection in the client or pool """ 142 self.client.release_connection(self._sock_key, 143 self._sock, self.should_close) 144 self._closed = True
145
146 - def can_read(self):
147 return not self._closed and not self._already_read
148
149 - def body_string(self, charset=None, unicode_errors="strict"):
150 """ return body string, by default in bytestring """ 151 152 if not self.can_read(): 153 raise AlreadyRead() 154 155 body = self._body.read() 156 self._already_read = True 157 158 # release connection 159 self.release_connection() 160 161 if charset is not None: 162 try: 163 body = body.decode(charset, unicode_errors) 164 except UnicodeDecodeError: 165 pass 166 return body
167
168 - def body_stream(self):
169 """ stream body """ 170 if not self.can_read(): 171 raise AlreadyRead() 172 173 self._already_read = True 174 175 return BodyWrapper(self, self.client)
176
177 - def tee(self):
178 """ copy response input to standard output or a file if length > 179 sock.MAX_BODY. This make possible to reuse it in your 180 appplication. When all the input has been read, connection is 181 released """ 182 if not hasattr(self._body, "reader"): 183 # head case 184 return self._body 185 186 return TeeInput(self, self.client)
187 188
189 -class Client(object):
190 191 """ A client handle a connection at a time. A client is threadsafe, 192 but an handled shouldn't be shared between threads. All connections 193 are shared between threads via a pool. 194 195 >>> from restkit import * 196 >>> c = Client() 197 >>> c.url = "http://google.com" 198 >>> r = c.perform() 199 r>>> r.status 200 '301 Moved Permanently' 201 >>> r.body_string() 202 '<HTML><HEAD><meta http-equiv="content-type" content="text/html;charset=utf-8">\n<TITLE>301 Moved</TITLE></HEAD><BODY>\n<H1>301 Moved</H1>\nThe document has moved\n<A HREF="http://www.google.com/">here</A>.\r\n</BODY></HTML>\r\n' 203 >>> c.follow_redirect = True 204 >>> r = c.perform() 205 >>> r.status 206 '200 OK' 207 208 """ 209 210 version = (1, 1) 211 response_class=ClientResponse 212
213 - def __init__(self, 214 follow_redirect=False, 215 force_follow_redirect=False, 216 max_follow_redirect=MAX_FOLLOW_REDIRECTS, 217 filters=None, 218 decompress=True, 219 manager=None, 220 response_class=None, 221 timeout=MAX_CLIENT_TIMEOUT, 222 force_dns=False, 223 max_tries=5, 224 wait_tries=1.0, 225 **ssl_args):
226 227 self.follow_redirect = follow_redirect 228 self.force_follow_redirect = force_follow_redirect 229 self.max_follow_redirect = max_follow_redirect 230 self.filters = Filters(filters) 231 self.decompress = decompress 232 233 # set manager 234 if manager is None: 235 manager = get_manager() 236 self._manager = manager 237 238 # change default response class 239 if response_class is not None: 240 self.response_class = response_class 241 242 self.max_tries = max_tries 243 self.wait_tries = wait_tries 244 self.timeout = timeout 245 246 self._nb_redirections = self.max_follow_redirect 247 self._url = None 248 self._initial_url = None 249 self._write_cb = None 250 self._headers = None 251 self._sock_key = None 252 self._sock = None 253 self._original = None 254 255 self.method = 'GET' 256 self.body = None 257 self.ssl_args = ssl_args or {}
258 259
260 - def _headers__get(self):
261 if not isinstance(self._headers, MultiDict): 262 self._headers = MultiDict(self._headers or []) 263 return self._headers
264 - def _headers__set(self, value):
265 self._headers = MultiDict(value)
266 headers = property(_headers__get, _headers__set, doc=_headers__get.__doc__) 267 268
269 - def write_callback(self, cb):
270 if not callable(cb): 271 raise ValueError("%s isn't a callable" % str(cb)) 272 self._write_cb = cb
273
274 - def _url__get(self):
275 if self._url is None: 276 raise ValueError("url isn't set") 277 return urlparse.urlunparse(self._url)
278 - def _url__set(self, string):
279 self._url = urlparse.urlparse(string)
280 url = property(_url__get, _url__set, doc="current url to request") 281
282 - def _parsed_url__get(self):
283 if self._url is None: 284 raise ValueError("url isn't set") 285 return self._url
286 parsed_url = property(_parsed_url__get) 287
288 - def _host__get(self):
289 try: 290 h = self.parsed_url.netloc.encode('ascii') 291 except UnicodeEncodeError: 292 h = self.parsed_url.netloc.encode('idna') 293 294 hdr_host = self.headers.iget("host") 295 if not hdr_host: 296 return h 297 return hdr_host
298 host = property(_host__get) 299
300 - def _path__get(self):
301 path = self.parsed_url.path or '/' 302 303 return urlparse.urlunparse(('','', path, self._url.params, 304 self._url.query, self._url.fragment))
305 path = property(_path__get, doc="request path") 306
307 - def req_is_chunked(self):
308 te = self.headers.iget("transfer-encoding") 309 return (te is not None and te.lower() == "chunked")
310
311 - def req_is_ssl(self):
312 return self.parsed_url.scheme == "ssl"
313
314 - def connect(self, addr, ssl):
315 """ create a socket """ 316 log.debug("create new connection") 317 for res in socket.getaddrinfo(addr[0], addr[1], 0, 318 socket.SOCK_STREAM): 319 af, socktype, proto, canonname, sa = res 320 321 try: 322 sck = socket.socket(af, socktype, proto) 323 324 sck.settimeout(self.timeout) 325 sck.connect(sa) 326 327 if ssl: 328 if not have_ssl: 329 raise ValueError("https isn't supported. On python 2.5x," 330 + " https support requires ssl module " 331 + "(http://pypi.python.org/pypi/ssl) " 332 + "to be intalled.") 333 validate_ssl_args(self.ssl_args) 334 sock = ssl.wrap_socket(sck, **self.ssl_args) 335 336 # apply connect filters 337 self.filters.apply("on_connect", self, sck, ssl) 338 339 return sck 340 except socket.error: 341 close(sck) 342 raise socket.error, "getaddrinfo returns an empty list"
343
344 - def get_connection(self):
345 """ get a connection from the pool or create new one. """ 346 addr = parse_netloc(self.parsed_url) 347 ssl = self.req_is_ssl() 348 self._sock_key = (addr, ssl) 349 350 sock = self._manager.find_socket(addr, ssl) 351 if sock is None: 352 sock = self.connect(addr, ssl) 353 return sock
354
355 - def release_connection(self, key, sck, should_close=False):
356 """ release a connection to the pool """ 357 358 if should_close: 359 log.debug("close connection") 360 close(sck) 361 return 362 363 log.debug("release connection") 364 self._manager.store_socket(sck, key[0], key[1])
365
366 - def close_connection(self):
367 """ close a connection """ 368 log.debug("close connection") 369 close(self._sock) 370 self._sock = None
371
372 - def parse_body(self):
373 """ transform a body if needed and set appropriate headers """ 374 if not self.body: 375 if self.method in ('POST', 'PUT',): 376 self.headers['Content-Length'] = 0 377 return 378 379 ctype = self.headers.iget('content-type') 380 clen = self.headers.iget('content-length') 381 382 if isinstance(self.body, dict): 383 if ctype is not None and \ 384 ctype.startswith("multipart/form-data"): 385 type_, opts = cgi.parse_header(ctype) 386 boundary = opts.get('boundary', uuid.uuid4().hex) 387 self.body, self.headers = multipart_form_encode(body, 388 self.headers, boundary) 389 else: 390 ctype = "application/x-www-form-urlencoded; charset=utf-8" 391 self.body = form_encode(self.body) 392 elif hasattr(self.body, "boundary"): 393 ctype = "multipart/form-data; boundary=%s" % self.body.boundary 394 clen = self.body.get_size() 395 396 if not ctype: 397 ctype = 'application/octet-stream' 398 if hasattr(self.body, 'name'): 399 ctype = mimetypes.guess_type(self.body.name)[0] 400 401 if not clen: 402 if hasattr(self.body, 'fileno'): 403 try: 404 self.body.flush() 405 except IOError: 406 pass 407 try: 408 fno = self.body.fileno() 409 clen = str(os.fstat(fno)[6]) 410 except IOError: 411 if not self.req_is_chunked(): 412 clen = len(self.body.read()) 413 elif hasattr(self.body, 'getvalue') and not \ 414 self.req_is_chunked(): 415 clen = len(self.body.getvalue()) 416 elif isinstance(self.body, types.StringTypes): 417 self.body = to_bytestring(self.body) 418 clen = len(self.body) 419 420 if clen is not None: 421 self.headers['Content-Length'] = clen 422 elif not self.req_is_chunked(): 423 raise RequestError("Can't determine content length and " + 424 "Transfer-Encoding header is not chunked") 425 426 if ctype is not None: 427 self.headers['Content-Type'] = ctype
428
429 - def make_headers_string(self):
430 """ create final header string """ 431 if self.version == (1,1): 432 httpver = "HTTP/1.1" 433 else: 434 httpver = "HTTP/1.0" 435 436 ua = self.headers.iget('user_agent') 437 host = self.host 438 accept_encoding = self.headers.iget('accept-encoding') 439 440 headers = [ 441 "%s %s %s\r\n" % (self.method, self.path, httpver), 442 "Host: %s\r\n" % host, 443 "User-Agent: %s\r\n" % ua or USER_AGENT, 444 "Accept-Encoding: %s\r\n" % accept_encoding or 'identity' 445 ] 446 447 headers.extend(["%s: %s\r\n" % (k, str(v)) for k, v in \ 448 self.headers.items() if k.lower() not in \ 449 ('user-agent', 'host', 'accept-encoding',)]) 450 451 log.debug("Send headers: %s" % headers) 452 return "%s\r\n" % "".join(headers)
453
454 - def reset_request(self):
455 """ reset a client handle to its intial state before performing. 456 It doesn't handle case where body has already been consumed """ 457 if self._original is None: 458 return 459 460 self.url = self._original["url"] 461 self.method = self._original["method"] 462 self.body = self._original["body"] 463 self.headers = self._original["headers"] 464 self._nb_redirections = self.max_follow_redirect
465
466 - def perform(self):
467 """ perform the request. If an error happen it will first try to 468 restart it """ 469 if not self.url: 470 raise RequestError("url isn't set") 471 472 log.debug("Start to perform request: %s %s %s" % (self.method, 473 self.host, self.path)) 474 475 self._original = dict( 476 url = self.url, 477 method = self.method, 478 body = self.body, 479 headers = self.headers 480 ) 481 482 tries = self.max_tries 483 wait = self.wait_tries 484 while tries > 0: 485 try: 486 # generate final body 487 self.parse_body() 488 489 # get or create a connection to the remote host 490 self._sock = self.get_connection() 491 492 # set socket timeout in case default has changed 493 self._sock.settimeout(self.timeout) 494 495 # apply on_request filters 496 self.filters.apply("on_request", self) 497 498 # send headers 499 headers_str = self.make_headers_string() 500 self._sock.sendall(headers_str) 501 502 # send body 503 if self.body is not None: 504 chunked = self.req_is_chunked() 505 log.debug("send body (chunked: %s) %s" % (chunked, 506 type(self.body))) 507 508 if hasattr(self.body, 'read'): 509 if hasattr(self.body, 'seek'): self.body.seek(0) 510 sendfile(self._sock, self.body, chunked) 511 elif isinstance(self.body, types.StringTypes): 512 send(self._sock, self.body, chunked) 513 else: 514 sendlines(self._sock, self.body, chunked) 515 if chunked: 516 send_chunk(self._sock, "") 517 518 return self.get_response() 519 except socket.gaierror, e: 520 self.close_connection() 521 raise RequestError(str(e)) 522 except socket.timeout, e: 523 self.close_connection() 524 wait = wait * 2 525 if tries <= 0: 526 raise RequestTimeout(str(e)) 527 except socket.error, e: 528 self.close_connection() 529 log.debug("socket error: %s" % str(e)) 530 if e[0] not in (errno.EAGAIN, errno.ECONNABORTED, 531 errno.EPIPE, errno.ECONNREFUSED, 532 errno.ECONNRESET, errno.EBADF) or tries <= 0: 533 raise RequestError(str(e)) 534 except (KeyboardInterrupt, SystemExit): 535 break 536 except Exception, e: 537 # unkown error 538 self.close_connection() 539 raise 540 541 # time until we retry. 542 time.sleep(wait) 543 544 tries = tries - 1 545 546 # reset request 547 self.reset_request()
548
549 - def request(self, url, method='GET', body=None, headers=None):
550 """ perform immediatly a new request """ 551 self.url = url 552 self.method = method 553 self.body = body 554 self.headers = copy.copy(headers) or [] 555 self._nb_redirections = self.max_follow_redirect 556 return self.perform()
557
558 - def redirect(self, resp, location, method=None):
559 """ reset request, set new url of request and perform it """ 560 if self._nb_redirections <= 0: 561 raise RedirectLimit("Redirection limit is reached") 562 563 if self._initial_url is None: 564 self._initial_url = self.url 565 566 # discard response body and reset request informations 567 if hasattr(resp, "_body"): 568 resp._body.discard() 569 else: 570 resp.body.discard() 571 self.reset_request() 572 573 # make sure location follow rfc2616 574 location = rewrite_location(self.url, location) 575 576 log.debug("Redirect to %s" % location) 577 578 # change request url and method if needed 579 self.url = location 580 if method is not None: 581 self.method = "GET" 582 583 self._nb_redirections -= 1 584 return self.perform()
585
586 - def get_response(self):
587 """ return final respons, it is only accessible via peform 588 method """ 589 unreader = http.Unreader(self._sock) 590 591 log.info("Start to parse response") 592 while True: 593 resp = http.Request(unreader) 594 if resp.status_int != 100: 595 break 596 resp.body.discard() 597 log.debug("Go 100-Continue header") 598 599 log.info("Got response: %s" % resp.status) 600 log.info("headers: [%s]" % resp.headers) 601 602 location = resp.headers.iget('location') 603 604 if self.follow_redirect: 605 if resp.status_int in (301, 302, 307,): 606 if self.method in ('GET', 'HEAD',) or \ 607 self.force_follow_redirect: 608 if hasattr(self.body, 'read'): 609 try: 610 self.body.seek(0) 611 except AttributeError: 612 raise RequestError("Can't redirect %s to %s " 613 "because body has already been read" 614 % (self.url, location)) 615 return self.redirect(resp, location) 616 617 elif resp.status_int == 303 and self.method == "POST": 618 return self.redirect(resp, location, method="GET") 619 620 # apply final response 621 self.filters.apply("on_response", self, resp) 622 623 # reset request 624 self.reset_request() 625 626 log.debug("return response class") 627 return self.response_class(self, resp)
628