Package restkit :: Module client
[hide private]
[frames] | no frames]

Source Code for Module restkit.client

  1  # -*- coding: utf-8 - 
  2  # 
  3  # This file is part of restkit released under the MIT license.  
  4  # See the NOTICE for more information. 
  5   
  6  import ctypes 
  7  import errno 
  8  import gzip 
  9  import logging 
 10  import os 
 11  import socket 
 12  import StringIO 
 13  import urlparse 
 14   
 15  from restkit import __version__ 
 16  from restkit.errors import RequestError, InvalidUrl, RedirectLimit 
 17  from restkit.parser import Parser 
 18  from restkit import sock 
 19  from restkit import tee 
 20  from restkit import util 
 21   
 22  MAX_FOLLOW_REDIRECTS = 5 
 23   
 24  USER_AGENT = "restkit/%s" % __version__ 
 25   
 26  log = logging.getLogger(__name__) 
27 28 -class HttpResponse(object):
29 """ Http Response object returned by HttpConnction""" 30 31 charset = "utf8" 32 unicode_errors = 'strict' 33
34 - def __init__(self, http_client):
35 self.http_client = http_client 36 self.status = self.http_client.parser.status 37 self.status_int = self.http_client.parser.status_int 38 self.version = self.http_client.parser.version 39 self.headerslist = self.http_client.parser.headers 40 self.final_url = self.http_client.final_url 41 42 headers = {} 43 for key, value in self.http_client.parser.headers: 44 headers[key.lower()] = value 45 self.headers = headers 46 47 encoding = headers.get('content-encoding', None) 48 if encoding in ('gzip', 'deflate'): 49 self._body = gzip.GzipFile(fileobj=self.http_client.response_body) 50 else: 51 self._body = self.http_client.response_body 52 self._body_eof = False
53
54 - def __getitem__(self, key):
55 try: 56 return getattr(self, key) 57 except AttributeError: 58 pass 59 return self.headers[key]
60
61 - def __contains__(self, key):
62 return (key in self.headers)
63
64 - def __iter__(self):
65 for item in list(self.headers.items()): 66 yield item
67 68 @property
69 - def body(self):
70 """ body in bytestring """ 71 if self._body_eof: 72 self._body.seek(0) 73 self._body_eof = True 74 ret = self._body.read() 75 self._body.seek(0) 76 return ret
77 78 @property
79 - def body_file(self):
80 """ return body as a file like object""" 81 return self._body
82 83 @property
84 - def unicode_body(self):
85 """ like body but converted to unicode""" 86 if not self.charset: 87 raise AttributeError( 88 "You cannot access HttpResponse.unicode_body unless charset is set") 89 return self.body.decode(self.charset, self.unicode_errors)
90
91 -class HttpConnection(object):
92 """ Http Connection object. """ 93 94 version = (1, 1) 95 response_class = HttpResponse 96 97
98 - def __init__(self, timeout=sock._GLOBAL_DEFAULT_TIMEOUT, 99 filters=None, follow_redirect=False, force_follow_redirect=False, 100 max_follow_redirect=MAX_FOLLOW_REDIRECTS, key_file=None, 101 cert_file=None, pool_instance=None, socket=None, 102 response_class=None):
103 104 """ HttpConnection constructor 105 106 :param timeout: socket timeout 107 :param filters: list, list of http filters. see the doc of http filters 108 for more info 109 :param follow_redirect: boolean, by default is false. If true, 110 if the HTTP status is 301, 302 or 303 the client will follow 111 the location. 112 :param max_follow_redirect: max number of redirection. If max is reached 113 the RedirectLimit exception is raised. 114 :param key_file: the key fle to use with ssl 115 :param cert_file: the cert file to use with ssl 116 :param pool_instance: a pool instance inherited from 117 `restkit.pool.PoolInterface` 118 :param socket: eventually you can pass your own socket object to 119 the client. 120 """ 121 self.socket = socket 122 self.timeout = timeout 123 self.headers = [] 124 self.req_headers = [] 125 self.ua = USER_AGENT 126 self.url = None 127 128 self.follow_redirect = follow_redirect 129 self.nb_redirections = max_follow_redirect 130 self.force_follow_redirect = force_follow_redirect 131 self.method = 'GET' 132 self.body = None 133 self.response_body = StringIO.StringIO() 134 self.final_url = None 135 136 # build filter lists 137 self.filters = filters or [] 138 self.request_filters = [] 139 self.response_filters = [] 140 141 for f in self.filters: 142 self._add_filter(f) 143 144 if not pool_instance: 145 self.should_close = True 146 self.connections = None 147 else: 148 self.connections = pool_instance 149 self.should_close = False 150 151 if response_class is not None: 152 self.response_class = response_class
153
154 - def add_filter(self, f):
155 self.filters.append(f) 156 self._add_filter(f)
157
158 - def _add_filter(self, f):
159 if hasattr(f, 'on_request'): 160 self.request_filters.append(f) 161 162 if hasattr(f, 'on_response'): 163 self.response_filters.append(f)
164
165 - def remove_filter(self, f):
166 for i, f1 in enumerate(self.filters): 167 if f == f1: del self.filters[i] 168 169 if hasattr(f, 'on_request'): 170 for i, f1 in enumerate(self.request_filters): 171 if f == f1: del self.request_filters[i] 172 173 if hasattr(f, 'on_response'): 174 for i, f1 in enumerate(self.response_filters): 175 if f == f1: del self.response_filters[i]
176
177 - def make_connection(self):
178 """ initate a connection if needed or reuse a socket""" 179 addr = (self.host, self.port) 180 s = self.socket or None 181 182 # if we defined a pool use it 183 if self.connections is not None: 184 s = self.connections.get(addr) 185 186 if not s: 187 # pool is empty or we don't use a pool 188 if self.uri.scheme == "https": 189 s = sock.connect(addr, self.timeout, True, 190 self.key_file, self.cert_file) 191 else: 192 s = sock.connect(addr, self.timeout) 193 194 self.socket = s 195 return s
196
197 - def clean_connections(self):
198 self.socket = None 199 if hasattr(self.connections,'clean'): 200 self.connections.clean((self.host, self.port))
201
202 - def maybe_close(self):
203 if not self.socket: return 204 if self.parser.should_close: 205 sock.close(self.socket) 206 else: # release the socket in the pool 207 self.connections.put((self.host, self.port), self.socket) 208 self.socket = None
209
210 - def parse_url(self, url):
211 """ parse url and get host/port""" 212 self.uri = urlparse.urlparse(url) 213 if self.uri.scheme not in ('http', 'https'): 214 raise InvalidUrl("None valid url") 215 216 host = self.uri.netloc 217 i = host.rfind(':') 218 j = host.rfind(']') # ipv6 addresses have [...] 219 if i > j: 220 try: 221 port = int(host[i+1:]) 222 except ValueError: 223 raise InvalidUrl("nonnumeric port: '%s'" % host[i+1:]) 224 host = host[:i] 225 else: 226 # default por 227 if self.uri.scheme == "https": 228 port = 443 229 else: 230 port = 80 231 232 if host and host[0] == '[' and host[-1] == ']': 233 host = host[1:-1] 234 235 self.host = host 236 self.port = port
237
238 - def request(self, url, method='GET', body=None, headers=None):
239 """ make effective request 240 241 :param url: str, url string 242 :param method: str, by default GET. http verbs 243 :param body: the body, could be a string, an iterator or a file-like object 244 :param headers: dict or list of tupple, http headers 245 """ 246 self.parser = Parser.parse_response(should_close=self.should_close) 247 self.url = url 248 self.final_url = url 249 self.parse_url(url) 250 self.method = method.upper() 251 252 253 # headers are better as list 254 headers = headers or [] 255 if isinstance(headers, dict): 256 headers = list(headers.items()) 257 258 ua = USER_AGENT 259 normalized_headers = [] 260 content_len = None 261 accept_encoding = 'identity' 262 chunked = False 263 264 # default host 265 try: 266 host = self.uri.netloc.encode('ascii') 267 except UnicodeEncodeError: 268 host = self.uri.netloc.encode('idna') 269 270 # normalize headers 271 for name, value in headers: 272 name = util.normalize_name(name) 273 if name == "User-Agenr": 274 ua = value 275 elif name == "Content-Length": 276 content_len = str(value) 277 elif name == "Accept-Encoding": 278 accept_encoding = 'identity' 279 elif name == "Host": 280 host = value 281 elif name == "Transfer-Encoding": 282 if value.lower() == "chunked": 283 chunked = True 284 normalized_headers.append((name, value)) 285 else: 286 if not isinstance(value, basestring): 287 value = str(value) 288 normalized_headers.append((name, value)) 289 290 # set content lengh if needed 291 if body and body is not None: 292 if not content_len: 293 if hasattr(body, 'fileno'): 294 try: 295 body.flush() 296 except IOError: 297 pass 298 content_len = str(os.fstat(body.fileno())[6]) 299 elif hasattr(body, 'len'): 300 try: 301 content_len = str(body.len) 302 except AttributeError: 303 pass 304 elif isinstance(body, basestring): 305 body = util.to_bytestring(body) 306 content_len = len(body) 307 308 if content_len: 309 normalized_headers.append(("Content-Length", content_len)) 310 elif not chunked: 311 raise RequestError("Can't determine content length and" + 312 "Transfer-Encoding header is not chunked") 313 314 if self.method in ('POST', 'PUT') and not body: 315 normalized_headers.append(("Content-Length", "0")) 316 317 self.body = body 318 self.headers = normalized_headers 319 self.ua = ua 320 321 # apply on request filters 322 for bf in self.request_filters: 323 bf.on_request(self) 324 325 # by default all connections are HTTP/1.1 326 if self.version == (1,1): 327 httpver = "HTTP/1.1" 328 else: 329 httpver = "HTTP/1.0" 330 331 # request path 332 path = self.uri.path or "/" 333 req_path = urlparse.urlunparse(('','', path, '', 334 self.uri.query, self.uri.fragment)) 335 336 # build final request headers 337 req_headers = [] 338 req_headers.append("%s %s %s\r\n" % (method, req_path, httpver)) 339 req_headers.append("Host: %s\r\n" % host) 340 req_headers.append("User-Agent: %s\r\n" % self.ua) 341 req_headers.append("Accept-Encoding: %s\r\n" % accept_encoding) 342 for name, value in self.headers: 343 req_headers.append("%s: %s\r\n" % (name, value)) 344 req_headers.append("\r\n") 345 self.req_headers = req_headers 346 347 # Finally do the request 348 return self.do_send(req_headers, self.body, chunked)
349
350 - def do_send(self, req_headers, body=None, chunked=False):
351 tries = 2 352 while True: 353 try: 354 s = self.make_connection() 355 # send request 356 sock.sendlines(s, req_headers) 357 358 log.info('Start request: %s %s' % (self.method, self.url)) 359 log.debug("Request headers: [%s]" % str(req_headers)) 360 361 if body is not None: 362 if hasattr(body, 'read'): 363 if hasattr(body, 'seek'): body.seek(0) 364 sock.sendfile(s, body, chunked) 365 elif isinstance(body, basestring): 366 sock.sendfile(s, StringIO.StringIO( 367 util.to_bytestring(body)), chunked) 368 else: 369 sock.sendlines(s, body, chunked) 370 371 if chunked: # final chunk 372 sock.send_chunk(s, "") 373 374 return self.start_response() 375 except socket.gaierror, e: 376 self.clean_connections() 377 raise 378 except socket.error, e: 379 if e[0] not in (errno.EAGAIN, errno.ECONNABORTED, errno.EPIPE, 380 errno.ECONNREFUSED) or tries <= 0: 381 self.clean_connections() 382 raise 383 except: 384 if tries <= 0: 385 raise 386 tries -= 1
387 388
389 - def do_redirect(self):
390 """ follow redirections if needed""" 391 if self.nb_redirections <= 0: 392 raise RedirectLimit("Redirection limit is reached") 393 394 location = self.parser.headers_dict.get('Location') 395 if not location: 396 raise RequestError('no Location header') 397 398 new_uri = urlparse.urlparse(location) 399 if not new_uri.netloc: # we got a relative url 400 absolute_uri = "%s://%s" % (self.uri.scheme, self.uri.netloc) 401 location = urlparse.urljoin(absolute_uri, location) 402 403 log.debug("Redirect to %s" % location) 404 405 self.final_url = location 406 self.response_body.read() 407 self.nb_redirections -= 1 408 self.maybe_close() 409 return self.request(location, self.method, self.body, 410 self.headers)
411
412 - def start_response(self):
413 """ 414 Get headers, set Body object and return HttpResponse 415 """ 416 # read headers 417 headers = [] 418 buf = sock.recv(self.socket, sock.CHUNK_SIZE) 419 i = self.parser.filter_headers(headers, buf) 420 if i == -1 and buf: 421 while True: 422 data = sock.recv(self.socket, sock.CHUNK_SIZE) 423 if not data: break 424 buf += data 425 i = self.parser.filter_headers(headers, buf) 426 if i != -1: 427 break 428 429 log.debug("Start response: %s" % str(self.parser.status_line)) 430 log.debug("Response headers: [%s]" % str(headers)) 431 432 if (not self.parser.content_len and not self.parser.is_chunked): 433 response_body = StringIO.StringIO("".join(buf[i:])) 434 if self.parser.should_close: 435 # http 1.0 or something like it. 436 # we try to get missing body 437 log.debug("No content len an not chunked transfer, get body") 438 while True: 439 try: 440 chunk = sock.recv(self.socket, sock.CHUNK_SIZE) 441 except socket.error: 442 break 443 if not chunk: break 444 response_body.write("".join(chunk)) 445 self.maybe_close() 446 447 response_body.seek(0) 448 self.response_body = response_body 449 elif self.method == "HEAD": 450 self.response_body = StringIO.StringIO() 451 self.maybe_close() 452 else: 453 self.response_body = tee.TeeInput(self.socket, self.parser, 454 buf[i:], maybe_close=self.maybe_close) 455 456 # apply on response filters 457 for af in self.response_filters: 458 af.on_response(self) 459 460 if self.follow_redirect: 461 if self.parser.status_int in (301, 302, 307): 462 if self.method in ('GET', 'HEAD') or \ 463 self.force_follow_redirect: 464 if self.method not in ('GET', 'HEAD') and \ 465 hasattr(self.body, 'seek'): 466 self.body.seek(0) 467 return self.do_redirect() 468 elif self.parser.status_int == 303 and self.method in ('GET', 469 'HEAD'): 470 # only 'GET' is possible with this status 471 # according the rfc 472 return self.do_redirect() 473 474 475 self.final_url = self.parser.headers_dict.get('Location', 476 self.final_url) 477 log.debug("Return response: %s" % self.final_url) 478 return self.response_class(self)
479