Package restkit :: Module client
[hide private]
[frames] | no frames]

Source Code for Module restkit.client

  1  # -*- coding: utf-8 - 
  2  # 
  3  # This file is part of restkit released under the MIT license.  
  4  # See the NOTICE for more information. 
  5   
  6  import ctypes 
  7  import errno 
  8  import gzip 
  9  import logging 
 10  import os 
 11  import socket 
 12  import StringIO 
 13  import urlparse 
 14   
 15  from restkit import __version__ 
 16  from restkit.errors import RequestError, InvalidUrl, RedirectLimit 
 17  from restkit.parser import Parser 
 18  from restkit import sock 
 19  from restkit import tee 
 20  from restkit import util 
 21   
 22  MAX_FOLLOW_REDIRECTS = 5 
 23   
 24  USER_AGENT = "restkit/%s" % __version__ 
 25   
 26  log = logging.getLogger(__name__) 
27 28 -class HttpResponse(object):
29 """ Http Response object returned by HttpConnction""" 30 31 charset = "utf8" 32 unicode_errors = 'strict' 33
34 - def __init__(self, http_client):
35 self.http_client = http_client 36 self.status = self.http_client.parser.status 37 self.status_int = self.http_client.parser.status_int 38 self.version = self.http_client.parser.version 39 self.headerslist = self.http_client.parser.headers 40 self.final_url = self.http_client.final_url 41 42 headers = {} 43 for key, value in self.http_client.parser.headers: 44 headers[key.lower()] = value 45 self.headers = headers 46 47 encoding = headers.get('content-encoding', None) 48 if encoding in ('gzip', 'deflate'): 49 self._body = gzip.GzipFile(fileobj=self.http_client.response_body) 50 else: 51 self._body = self.http_client.response_body 52 self._body_eof = False
53
54 - def __getitem__(self, key):
55 try: 56 return getattr(self, key) 57 except AttributeError: 58 pass 59 return self.headers[key]
60
61 - def __contains__(self, key):
62 return (key in self.headers)
63
64 - def __iter__(self):
65 for item in list(self.headers.items()): 66 yield item
67 68 @property
69 - def body(self):
70 """ body in bytestring """ 71 if self._body_eof: 72 self._body.seek(0) 73 self._body_eof = True 74 ret = self._body.read() 75 self._body.seek(0) 76 return ret
77 78 @property
79 - def body_file(self):
80 """ return body as a file like object""" 81 return self._body
82 83 @property
84 - def unicode_body(self):
85 """ like body but converted to unicode""" 86 if not self.charset: 87 raise AttributeError( 88 "You cannot access HttpResponse.unicode_body unless charset is set") 89 return self.body.decode(self.charset, self.unicode_errors)
90
91 -class HttpConnection(object):
92 """ Http Connection object. """ 93 94 version = (1, 1) 95 response_class = HttpResponse 96 97
98 - def __init__(self, timeout=sock._GLOBAL_DEFAULT_TIMEOUT, 99 filters=None, follow_redirect=False, force_follow_redirect=False, 100 max_follow_redirect=MAX_FOLLOW_REDIRECTS, key_file=None, 101 cert_file=None, pool_instance=None, socket=None, 102 response_class=None):
103 104 """ HttpConnection constructor 105 106 :param timeout: socket timeout 107 :param filters: list, list of http filters. see the doc of http filters 108 for more info 109 :param follow_redirect: boolean, by default is false. If true, 110 if the HTTP status is 301, 302 or 303 the client will follow 111 the location. 112 :param max_follow_redirect: max number of redirection. If max is reached 113 the RedirectLimit exception is raised. 114 :param key_file: the key fle to use with ssl 115 :param cert_file: the cert file to use with ssl 116 :param pool_instance: a pool instance inherited from 117 `restkit.pool.PoolInterface` 118 :param socket: eventually you can pass your own socket object to 119 the client. 120 """ 121 self.socket = socket 122 self.timeout = timeout 123 self.headers = [] 124 self.req_headers = [] 125 self.ua = USER_AGENT 126 self.url = None 127 128 self.follow_redirect = follow_redirect 129 self.nb_redirections = max_follow_redirect 130 self.force_follow_redirect = force_follow_redirect 131 self.method = 'GET' 132 self.body = None 133 self.response_body = StringIO.StringIO() 134 self.final_url = None 135 136 # build filter lists 137 self.filters = filters or [] 138 self.request_filters = [] 139 self.response_filters = [] 140 141 for f in self.filters: 142 self._add_filter(f) 143 144 if not pool_instance: 145 self.should_close = True 146 self.connections = None 147 else: 148 self.connections = pool_instance 149 self.should_close = False 150 151 if response_class is not None: 152 self.response_class = response_class
153
154 - def add_filter(self, f):
155 self.filters.append(f) 156 self._add_filter(f)
157
158 - def _add_filter(self, f):
159 if hasattr(f, 'on_request'): 160 self.request_filters.append(f) 161 162 if hasattr(f, 'on_response'): 163 self.response_filters.append(f)
164
165 - def remove_filter(self, f):
166 for i, f1 in enumerate(self.filters): 167 if f == f1: del self.filters[i] 168 169 if hasattr(f, 'on_request'): 170 for i, f1 in enumerate(self.request_filters): 171 if f == f1: del self.request_filters[i] 172 173 if hasattr(f, 'on_response'): 174 for i, f1 in enumerate(self.response_filters): 175 if f == f1: del self.response_filters[i]
176
177 - def make_connection(self):
178 """ initate a connection if needed or reuse a socket""" 179 addr = (self.host, self.port) 180 s = self.socket or None 181 182 # if we defined a pool use it 183 if self.connections is not None: 184 s = self.connections.get(addr) 185 186 if not s: 187 # pool is empty or we don't use a pool 188 if self.uri.scheme == "https": 189 s = sock.connect(addr, self.timeout, True, 190 self.key_file, self.cert_file) 191 else: 192 s = sock.connect(addr, self.timeout) 193 194 self.socket = s 195 return s
196
197 - def clean_connections(self):
198 self.socket = None 199 if hasattr(self.connections,'clean'): 200 self.connections.clean((self.host, self.port))
201
202 - def maybe_close(self):
203 if not self.socket: return 204 if not hasattr(self.connections,'clear') or self.parser.should_close: 205 sock.close(self.socket) 206 else: # release the socket in the pool 207 self.connections.put((self.host, self.port), self.socket) 208 self.socket = None
209
210 - def parse_url(self, url):
211 """ parse url and get host/port""" 212 self.uri = urlparse.urlparse(url) 213 214 host = self.uri.netloc 215 i = host.rfind(':') 216 j = host.rfind(']') # ipv6 addresses have [...] 217 if i > j: 218 try: 219 port = int(host[i+1:]) 220 except ValueError: 221 raise InvalidUrl("nonnumeric port: '%s'" % host[i+1:]) 222 host = host[:i] 223 else: 224 # default por 225 if self.uri.scheme == "https": 226 port = 443 227 else: 228 port = 80 229 230 if host and host[0] == '[' and host[-1] == ']': 231 host = host[1:-1] 232 233 self.host = host 234 self.port = port
235
236 - def request(self, url, method='GET', body=None, headers=None):
237 """ make effective request 238 239 :param url: str, url string 240 :param method: str, by default GET. http verbs 241 :param body: the body, could be a string, an iterator or a file-like object 242 :param headers: dict or list of tupple, http headers 243 """ 244 self.parser = Parser.parse_response(should_close=self.should_close) 245 self.url = url 246 self.final_url = url 247 self.parse_url(url) 248 self.method = method.upper() 249 250 251 # headers are better as list 252 headers = headers or [] 253 if isinstance(headers, dict): 254 headers = list(headers.items()) 255 256 ua = USER_AGENT 257 normalized_headers = [] 258 content_len = None 259 accept_encoding = 'identity' 260 chunked = False 261 262 # default host 263 try: 264 host = self.uri.netloc.encode('ascii') 265 except UnicodeEncodeError: 266 host = self.uri.netloc.encode('idna') 267 268 # normalize headers 269 for name, value in headers: 270 name = util.normalize_name(name) 271 if name == "User-Agenr": 272 ua = value 273 elif name == "Content-Length": 274 content_len = str(value) 275 elif name == "Accept-Encoding": 276 accept_encoding = 'identity' 277 elif name == "Host": 278 host = value 279 elif name == "Transfer-Encoding": 280 if value.lower() == "chunked": 281 chunked = True 282 normalized_headers.append((name, value)) 283 else: 284 if not isinstance(value, basestring): 285 value = str(value) 286 normalized_headers.append((name, value)) 287 288 # set content lengh if needed 289 if body and body is not None: 290 if not content_len: 291 if hasattr(body, 'fileno'): 292 try: 293 body.flush() 294 except IOError: 295 pass 296 content_len = str(os.fstat(body.fileno())[6]) 297 elif hasattr(body, 'len'): 298 try: 299 content_len = str(body.len) 300 except AttributeError: 301 pass 302 elif isinstance(body, basestring): 303 body = util.to_bytestring(body) 304 content_len = len(body) 305 306 if content_len: 307 normalized_headers.append(("Content-Length", content_len)) 308 elif not chunked: 309 raise RequestError("Can't determine content length and" + 310 "Transfer-Encoding header is not chunked") 311 312 if self.method in ('POST', 'PUT') and not body: 313 normalized_headers.append(("Content-Length", "0")) 314 315 self.body = body 316 self.headers = normalized_headers 317 self.ua = ua 318 319 # apply on request filters 320 for bf in self.request_filters: 321 bf.on_request(self) 322 323 # by default all connections are HTTP/1.1 324 if self.version == (1,1): 325 httpver = "HTTP/1.1" 326 else: 327 httpver = "HTTP/1.0" 328 329 # request path 330 path = self.uri.path or "/" 331 req_path = urlparse.urlunparse(('','', path, '', 332 self.uri.query, self.uri.fragment)) 333 334 # build final request headers 335 req_headers = [] 336 req_headers.append("%s %s %s\r\n" % (method, req_path, httpver)) 337 req_headers.append("Host: %s\r\n" % host) 338 req_headers.append("User-Agent: %s\r\n" % self.ua) 339 req_headers.append("Accept-Encoding: %s\r\n" % accept_encoding) 340 for name, value in self.headers: 341 req_headers.append("%s: %s\r\n" % (name, value)) 342 req_headers.append("\r\n") 343 self.req_headers = req_headers 344 345 # Finally do the request 346 return self.do_send(req_headers, self.body, chunked)
347
348 - def do_send(self, req_headers, body=None, chunked=False):
349 for i in range(2): 350 try: 351 s = self.make_connection() 352 # send request 353 sock.sendlines(s, req_headers) 354 355 log.info('%s %s\n\n' % (self.method, self.url)) 356 log.debug("Headers: [%s]" % "".join(req_headers)) 357 358 if body is not None: 359 if hasattr(body, 'read'): 360 sock.sendfile(s, body, chunked) 361 elif isinstance(body, basestring): 362 sock.sendfile(s, StringIO.StringIO( 363 util.to_bytestring(body)), chunked) 364 else: 365 sock.sendlines(s, body, chunked) 366 367 if chunked: # final chunk 368 sock.send_chunk(s, "") 369 370 return self.start_response() 371 except socket.gaierror, e: 372 self.clean_connections() 373 raise 374 except socket.error, e: 375 if e[0] not in (errno.EAGAIN, errno.ECONNABORTED, errno.EPIPE, 376 errno.ECONNREFUSED): 377 self.clean_connections() 378 raise
379
380 - def do_redirect(self):
381 """ follow redirections if needed""" 382 if self.nb_redirections <= 0: 383 raise RedirectLimit("Redirection limit is reached") 384 385 location = self.parser.headers_dict.get('Location') 386 if not location: 387 raise RequestError('no Location header') 388 389 new_uri = urlparse.urlparse(location) 390 if not new_uri.netloc: # we got a relative url 391 absolute_uri = "%s://%s" % (self.uri.scheme, self.uri.netloc) 392 location = urlparse.urljoin(absolute_uri, location) 393 394 self.final_url = location 395 self.response_body.read() 396 self.nb_redirections -= 1 397 self.maybe_close() 398 return self.request(location, self.method, self.body, 399 self.headers)
400
401 - def start_response(self):
402 """ 403 Get headers, set Body object and return HttpResponse 404 """ 405 # read headers 406 headers = [] 407 buf = sock.recv(self.socket, sock.CHUNK_SIZE) 408 i = self.parser.filter_headers(headers, buf) 409 if i == -1 and buf: 410 while True: 411 data = sock.recv(self.socket, sock.CHUNK_SIZE) 412 if not data: break 413 buf += data 414 i = self.parser.filter_headers(headers, buf) 415 if i != -1: 416 break 417 log.debug("Start response") 418 log.debug("Response headers: [%s]" % str(headers)) 419 420 if (not self.parser.content_len and not self.parser.is_chunked): 421 response_body = StringIO.StringIO("".join(buf[i:])) 422 423 if self.parser.should_close: 424 # http 1.0 or something like it. 425 # we try to get missing body 426 while True: 427 try: 428 chunk = sock.recv(self.socket, sock.CHUNK_SIZE) 429 except socket.error: 430 break 431 response_body.write("".join(chunk)) 432 if not chunk: break 433 self.maybe_close() 434 435 response_body.seek(0) 436 self.response_body = response_body 437 elif self.method == "HEAD": 438 self.response_body = StringIO.StringIO() 439 else: 440 self.response_body = tee.TeeInput(self.socket, self.parser, 441 buf[i:], maybe_close=self.maybe_close) 442 443 # apply on response filters 444 for af in self.response_filters: 445 af.on_response(self) 446 447 if self.follow_redirect: 448 if self.parser.status_int in (301, 302, 307): 449 if self.method in ('GET', 'HEAD') or \ 450 self.force_follow_redirect: 451 if self.method not in ('GET', 'HEAD') and \ 452 hasattr(self.body, 'seek'): 453 self.body.seek(0) 454 return self.do_redirect() 455 elif self.parser.status_int == 303 and self.method in ('GET', 456 'HEAD'): 457 # only 'GET' is possible with this status 458 # according the rfc 459 return self.do_redirect() 460 461 self.final_url = self.parser.headers_dict.get('Location', 462 self.final_url) 463 return self.response_class(self)
464