Package restkit :: Module client
[hide private]
[frames] | no frames]

Source Code for Module restkit.client

  1  # -*- coding: utf-8 - 
  2  # 
  3  # This file is part of restkit released under the MIT license. 
  4  # See the NOTICE for more information. 
  5  import base64 
  6  import errno 
  7  import logging 
  8  import os 
  9  import time 
 10  import socket 
 11  import traceback 
 12  import types 
 13  import urlparse 
 14   
 15  try: 
 16      import ssl # python 2.6 
 17      _ssl_wrapper = ssl.wrap_socket 
 18      have_ssl = True 
 19  except ImportError: 
 20      if hasattr(socket, "ssl"): 
 21          from httplib import FakeSocket 
 22          from restkit.sock import trust_all_certificates 
 23   
 24          @trust_all_certificates 
25 - def _ssl_wrapper(sck, **kwargs):
26 ssl_sck = socket.ssl(sck, **kwargs) 27 return FakeSocket(sck, ssl_sck)
28 have_ssl = True 29 else: 30 have_ssl = False 31 32 try: 33 from http_parser.http import HttpStream 34 from http_parser.reader import SocketReader 35 except ImportError: 36 raise ImportError("""http-parser isn't installed. 37 38 pip install http-parser""") 39 40 from restkit import __version__ 41 42 from restkit.conn import Connection 43 from restkit.errors import RequestError, RequestTimeout, RedirectLimit, \ 44 NoMoreData, ProxyError 45 from restkit.globals import get_manager 46 from restkit.sock import close, send, sendfile, sendlines, send_chunk, \ 47 validate_ssl_args 48 from restkit.util import parse_netloc, rewrite_location 49 from restkit.wrappers import Request, Response 50 51 MAX_CLIENT_TIMEOUT=300 52 MAX_CLIENT_CONNECTIONS = 5 53 MAX_CLIENT_TRIES = 5 54 CLIENT_WAIT_TRIES = 0.3 55 MAX_FOLLOW_REDIRECTS = 5 56 USER_AGENT = "restkit/%s" % __version__ 57 58 log = logging.getLogger(__name__)
59 60 -class Client(object):
61 62 """ A client handle a connection at a time. A client is threadsafe, 63 but an handled shouldn't be shared between threads. All connections 64 are shared between threads via a pool. 65 66 >>> from restkit import * 67 >>> c = Client() 68 >>> r = c.request("http://google.com") 69 r>>> r.status 70 '301 Moved Permanently' 71 >>> r.body_string() 72 '<HTML><HEAD><meta http-equiv="content-type" content="text/html;charset=utf-8">\n<TITLE>301 Moved</TITLE></HEAD><BODY>\n<H1>301 Moved</H1>\nThe document has moved\n<A HREF="http://www.google.com/">here</A>.\r\n</BODY></HTML>\r\n' 73 >>> c.follow_redirect = True 74 >>> r = c.request("http://google.com") 75 >>> r.status 76 '200 OK' 77 78 """ 79 80 version = (1, 1) 81 response_class=Response 82
83 - def __init__(self, 84 follow_redirect=False, 85 force_follow_redirect=False, 86 max_follow_redirect=MAX_FOLLOW_REDIRECTS, 87 filters=None, 88 decompress=True, 89 max_status_line_garbage=None, 90 max_header_count=0, 91 manager=None, 92 response_class=None, 93 timeout=None, 94 use_proxy=False, 95 max_tries=5, 96 wait_tries=1.0, 97 **ssl_args):
98 """ 99 Client parameters 100 ~~~~~~~~~~~~~~~~~ 101 102 :param follow_redirect: follow redirection, by default False 103 :param max_ollow_redirect: number of redirections available 104 :filters: http filters to pass 105 :param decompress: allows the client to decompress the response 106 body 107 :param max_status_line_garbage: defines the maximum number of ignorable 108 lines before we expect a HTTP response's status line. With 109 HTTP/1.1 persistent connections, the problem arises that broken 110 scripts could return a wrong Content-Length (there are more 111 bytes sent than specified). Unfortunately, in some cases, this 112 cannot be detected after the bad response, but only before the 113 next one. So the client is abble to skip bad lines using this 114 limit. 0 disable garbage collection, None means unlimited number 115 of tries. 116 :param max_header_count: determines the maximum HTTP header count 117 allowed. by default no limit. 118 :param manager: the manager to use. By default we use the global 119 one. 120 :parama response_class: the response class to use 121 :param timeout: the default timeout of the connection 122 (SO_TIMEOUT) 123 124 :param max_tries: the number of tries before we give up a 125 connection 126 :param wait_tries: number of time we wait between each tries. 127 :param ssl_args: named argument, see ssl module for more 128 informations 129 """ 130 self.follow_redirect = follow_redirect 131 self.force_follow_redirect = force_follow_redirect 132 self.max_follow_redirect = max_follow_redirect 133 self.decompress = decompress 134 self.filters = filters or [] 135 self.max_status_line_garbage = max_status_line_garbage 136 self.max_header_count = max_header_count 137 self.use_proxy = use_proxy 138 139 self.request_filters = [] 140 self.response_filters = [] 141 self.load_filters() 142 143 144 # set manager 145 if manager is None: 146 manager = get_manager() 147 self._manager = manager 148 149 # change default response class 150 if response_class is not None: 151 self.response_class = response_class 152 153 self.max_tries = max_tries 154 self.wait_tries = wait_tries 155 self.timeout = timeout 156 157 self._nb_redirections = self.max_follow_redirect 158 self._url = None 159 self._initial_url = None 160 self._write_cb = None 161 self._headers = None 162 self._sock_key = None 163 self._sock = None 164 self._original = None 165 166 self.method = 'GET' 167 self.body = None 168 self.ssl_args = ssl_args or {}
169
170 - def load_filters(self):
171 """ Populate filters from self.filters. 172 Must be called each time self.filters is updated. 173 """ 174 for f in self.filters: 175 if hasattr(f, "on_request"): 176 self.request_filters.append(f) 177 if hasattr(f, "on_response"): 178 self.response_filters.append(f)
179
180 - def connect(self, addr, is_ssl):
181 """ create a socket """ 182 if log.isEnabledFor(logging.DEBUG): 183 log.debug("create new connection") 184 for res in socket.getaddrinfo(addr[0], addr[1], 0, 185 socket.SOCK_STREAM): 186 af, socktype, proto, canonname, sa = res 187 188 try: 189 sck = socket.socket(af, socktype, proto) 190 191 if self.timeout is not None: 192 sck.settimeout(self.timeout) 193 194 sck.connect(sa) 195 196 if is_ssl: 197 if not have_ssl: 198 raise ValueError("https isn't supported. On python 2.5x," 199 + " https support requires ssl module " 200 + "(http://pypi.python.org/pypi/ssl) " 201 + "to be intalled.") 202 validate_ssl_args(self.ssl_args) 203 sck = _ssl_wrapper(sck, **self.ssl_args) 204 205 return sck 206 except socket.error, ex: 207 if ex == 24: # too many open files 208 raise 209 else: 210 close(sck) 211 raise socket.error, "Can't connect to %s" % str(addr)
212
213 - def get_connection(self, request):
214 """ get a connection from the pool or create new one. """ 215 216 addr = parse_netloc(request.parsed_url) 217 is_ssl = request.is_ssl() 218 219 extra_headers = [] 220 sck = None 221 if self.use_proxy: 222 sck, addr, extra_headers = self.proxy_connection(request, 223 addr, is_ssl) 224 if not sck: 225 sck = self._manager.find_socket(addr, is_ssl) 226 if sck is None: 227 sck = self.connect(addr, is_ssl) 228 229 # set socket timeout in case default has changed 230 if self.timeout is not None: 231 sck.settimeout(self.timeout) 232 233 connection = Connection(sck, self._manager, addr, 234 ssl=is_ssl, extra_headers=extra_headers) 235 return connection
236
237 - def proxy_connection(self, request, req_addr, is_ssl):
238 """ do the proxy connection """ 239 proxy_settings = os.environ.get('%s_proxy' % 240 request.parsed_url.scheme) 241 242 if proxy_settings and proxy_settings is not None: 243 proxy_settings, proxy_auth = _get_proxy_auth(proxy_settings) 244 addr = parse_netloc(urlparse.urlparse(proxy_settings)) 245 246 if is_ssl: 247 if proxy_auth: 248 proxy_auth = 'Proxy-authorization: %s' % proxy_auth 249 proxy_connect = 'CONNECT %s:%s HTTP/1.0\r\n' % req_addr 250 251 user_agent = request.headers.iget('user_agent') 252 if not user_agent: 253 user_agent = "User-Agent: restkit/%s\r\n" % __version__ 254 255 proxy_pieces = '%s%s%s\r\n' % (proxy_connect, proxy_auth, 256 user_agent) 257 258 sck = self._manager.find_socket(addr, ssl) 259 if sck is None: 260 self = self.connect(addr, ssl) 261 262 send(sck, proxy_pieces) 263 unreader = http.Unreader(sck) 264 resp = http.Request(unreader) 265 body = resp.body.read() 266 if resp.status_int != 200: 267 raise ProxyError("Tunnel connection failed: %d %s" % 268 (resp.status_int, body)) 269 270 return sck, addr, [] 271 else: 272 headers = [] 273 if proxy_auth: 274 headers = [('Proxy-authorization', proxy_auth)] 275 276 sck = self._manager.find_socket(addr, ssl) 277 if sck is None: 278 sck = self.connect(addr, ssl) 279 return sck, addr, headers 280 return None, req_addr, []
281
282 - def make_headers_string(self, request, extra_headers=None):
283 """ create final header string """ 284 headers = request.headers.copy() 285 if extra_headers is not None: 286 for k, v in extra_headers: 287 headers[k] = v 288 289 if not request.body and request.method in ('POST', 'PUT',): 290 headers['Content-Length'] = 0 291 292 if self.version == (1,1): 293 httpver = "HTTP/1.1" 294 else: 295 httpver = "HTTP/1.0" 296 297 ua = headers.iget('user_agent') 298 if not ua: 299 ua = USER_AGENT 300 host = request.host 301 302 accept_encoding = headers.iget('accept-encoding') 303 if not accept_encoding: 304 accept_encoding = 'identity' 305 306 lheaders = [ 307 "%s %s %s\r\n" % (request.method, request.path, httpver), 308 "Host: %s\r\n" % host, 309 "User-Agent: %s\r\n" % ua, 310 "Accept-Encoding: %s\r\n" % accept_encoding 311 ] 312 313 lheaders.extend(["%s: %s\r\n" % (k, str(v)) for k, v in \ 314 headers.items() if k.lower() not in \ 315 ('user-agent', 'host', 'accept-encoding',)]) 316 if log.isEnabledFor(logging.DEBUG): 317 log.debug("Send headers: %s" % lheaders) 318 return "%s\r\n" % "".join(lheaders)
319
320 - def perform(self, request):
321 """ perform the request. If an error happen it will first try to 322 restart it """ 323 324 if log.isEnabledFor(logging.DEBUG): 325 log.debug("Start to perform request: %s %s %s" % 326 (request.host, request.method, request.path)) 327 328 tries = self.max_tries 329 wait = self.wait_tries 330 while tries > 0: 331 try: 332 # get or create a connection to the remote host 333 connection = self.get_connection(request) 334 sck = connection.socket() 335 336 # send headers 337 msg = self.make_headers_string(request, 338 connection.extra_headers) 339 340 # send body 341 if request.body is not None: 342 chunked = request.is_chunked() 343 if request.headers.iget('content-length') is None and \ 344 not chunked: 345 raise RequestError( 346 "Can't determine content length and " + 347 "Transfer-Encoding header is not chunked") 348 349 350 # handle 100-Continue status 351 # http://www.w3.org/Protocols/rfc2616/rfc2616-sec8.html#sec8.2.3 352 hdr_expect = request.headers.iget("expect") 353 if hdr_expect is not None and \ 354 hdr_expect.lower() == "100-continue": 355 sck.sendall(msg) 356 msg = None 357 resp = http.Request(http.Unreader(self._sock)) 358 if resp.status_int != 100: 359 self.reset_request() 360 if log.isEnabledFor(logging.DEBUG): 361 log.debug("return response class") 362 return self.response_class(connection, 363 request, resp) 364 365 chunked = request.is_chunked() 366 if log.isEnabledFor(logging.DEBUG): 367 log.debug("send body (chunked: %s)" % chunked) 368 369 370 if isinstance(request.body, types.StringTypes): 371 if msg is not None: 372 send(sck, msg + request.body, chunked) 373 else: 374 send(sck, request.body, chunked) 375 else: 376 if msg is not None: 377 sck.sendall(msg) 378 379 if hasattr(request.body, 'read'): 380 if hasattr(request.body, 'seek'): request.body.seek(0) 381 sendfile(sck, request.body, chunked) 382 else: 383 sendlines(sck, request.body, chunked) 384 if chunked: 385 send_chunk(sck, "") 386 else: 387 sck.sendall(msg) 388 389 return self.get_response(request, connection) 390 except socket.gaierror, e: 391 try: 392 connection.close() 393 except: 394 pass 395 396 raise RequestError(str(e)) 397 except socket.timeout, e: 398 try: 399 connection.close() 400 except: 401 pass 402 403 if tries <= 0: 404 raise RequestTimeout(str(e)) 405 except socket.error, e: 406 if log.isEnabledFor(logging.DEBUG): 407 log.debug("socket error: %s" % str(e)) 408 try: 409 connection.close() 410 except: 411 pass 412 413 if e[0] not in (errno.EAGAIN, errno.ECONNABORTED, 414 errno.EPIPE, errno.ECONNREFUSED, 415 errno.ECONNRESET, errno.EBADF) or tries <= 0: 416 raise RequestError("socket.error: %s" % str(e)) 417 except (StopIteration, NoMoreData): 418 connection.close() 419 if tries <= 0: 420 raise 421 else: 422 if request.body is not None: 423 if not hasattr(request.body, 'read') and \ 424 not isinstance(request.body, types.StringTypes): 425 raise RequestError("connection closed and can't" 426 + "be resent") 427 except Exception: 428 # unkown error 429 log.debug("unhandled exception %s" % 430 traceback.format_exc()) 431 raise 432 433 # time until we retry. 434 time.sleep(wait) 435 wait = wait * 2 436 tries = tries - 1
437 438
439 - def request(self, url, method='GET', body=None, headers=None):
440 """ perform immediatly a new request """ 441 442 request = Request(url, method=method, body=body, 443 headers=headers) 444 445 # apply request filters 446 # They are applied only once time. 447 for f in self.request_filters: 448 ret = f.on_request(request) 449 if isinstance(ret, Response): 450 # a response instance has been provided. 451 # just return it. Useful for cache filters 452 return ret 453 454 # no response has been provided, do the request 455 self._nb_redirections = self.max_follow_redirect 456 return self.perform(request)
457
458 - def redirect(self, resp, location, request):
459 """ reset request, set new url of request and perform it """ 460 if self._nb_redirections <= 0: 461 raise RedirectLimit("Redirection limit is reached") 462 463 if request.initial_url is None: 464 request.initial_url = self.url 465 466 # make sure location follow rfc2616 467 location = rewrite_location(request.url, location) 468 469 if log.isEnabledFor(logging.DEBUG): 470 log.debug("Redirect to %s" % location) 471 472 # change request url and method if needed 473 request.url = location 474 475 self._nb_redirections -= 1 476 477 #perform a new request 478 return self.perform(request)
479
480 - def get_response(self, request, connection):
481 """ return final respons, it is only accessible via peform 482 method """ 483 if log.isEnabledFor(logging.DEBUG): 484 log.debug("Start to parse response") 485 486 p = HttpStream(SocketReader(connection.socket()), kind=1, 487 decompress=self.decompress) 488 489 if log.isEnabledFor(logging.DEBUG): 490 log.debug("Got response: %s" % p.status()) 491 log.debug("headers: [%s]" % p.headers()) 492 493 location = p.headers().get('location') 494 495 if self.follow_redirect: 496 if p.status_code() in (301, 302, 307,): 497 if request.method in ('GET', 'HEAD',) or \ 498 self.force_follow_redirect: 499 if hasattr(self.body, 'read'): 500 try: 501 self.body.seek(0) 502 except AttributeError: 503 connection.release() 504 raise RequestError("Can't redirect %s to %s " 505 "because body has already been read" 506 % (self.url, location)) 507 connection.release() 508 return self.redirect(p, location, request) 509 510 elif p.status_code() == 303 and self.method == "POST": 511 connection.release() 512 request.method = "GET" 513 request.body = None 514 return self.redirect(p, location, request) 515 516 # create response object 517 resp = self.response_class(connection, request, p) 518 519 # apply response filters 520 for f in self.response_filters: 521 f.on_response(resp, request) 522 523 if log.isEnabledFor(logging.DEBUG): 524 log.debug("return response class") 525 526 # return final response 527 return resp
528
529 530 -def _get_proxy_auth(proxy_settings):
531 proxy_username = os.environ.get('proxy-username') 532 if not proxy_username: 533 proxy_username = os.environ.get('proxy_username') 534 proxy_password = os.environ.get('proxy-password') 535 if not proxy_password: 536 proxy_password = os.environ.get('proxy_password') 537 538 proxy_password = proxy_password or "" 539 540 if not proxy_username: 541 u = urlparse.urlparse(proxy_settings) 542 if u.username: 543 proxy_password = u.password or proxy_password 544 proxy_settings = urlparse.urlunparse((u.scheme, 545 u.netloc.split("@")[-1], u.path, u.params, u.query, 546 u.fragment)) 547 548 if proxy_username: 549 user_auth = base64.encodestring('%s:%s' % (proxy_username, 550 proxy_password)) 551 return proxy_settings, 'Basic %s\r\n' % (user_auth.strip()) 552 else: 553 return proxy_settings, ''
554