Package restkit :: Package client :: Module request
[hide private]
[frames] | no frames]

Source Code for Module restkit.client.request

  1  # -*- coding: utf-8 - 
  2  # 
  3  # This file is part of restkit released under the MIT license.  
  4  # See the NOTICE for more information. 
  5   
  6  import copy 
  7  import cgi 
  8  import errno 
  9  import logging 
 10  import mimetypes 
 11  import os 
 12  import socket 
 13  import time 
 14  try: 
 15      from cStringIO import StringIO 
 16  except ImportError: 
 17      from StringIO import StringIO 
 18  import types 
 19  import urlparse 
 20  import uuid 
 21   
 22  from restkit import __version__ 
 23  from restkit.client.response import HttpResponse 
 24  from restkit.errors import RequestError, InvalidUrl, RedirectLimit 
 25  from restkit.filters import Filters 
 26  from restkit.forms import multipart_form_encode, form_encode 
 27  from restkit.util import sock 
 28  from restkit import util 
 29  from restkit import http 
 30   
 31  MAX_FOLLOW_REDIRECTS = 5 
 32   
 33  USER_AGENT = "restkit/%s" % __version__ 
 34   
 35  log = logging.getLogger(__name__) 
 36   
37 -class HttpRequest(object):
38 """ Http Connection object. """ 39 40 version = (1, 1) 41 response_class = HttpResponse 42
43 - def __init__(self, timeout=sock._GLOBAL_DEFAULT_TIMEOUT, 44 filters=None, follow_redirect=False, force_follow_redirect=False, 45 max_follow_redirect=MAX_FOLLOW_REDIRECTS, 46 pool_instance=None, response_class=None, 47 **ssl_args):
48 49 """ HttpRequest constructor 50 51 :param timeout: socket timeout 52 :param filters: list, list of http filters. see the doc of http filters 53 for more info 54 :param follow_redirect: boolean, by default is false. If true, 55 if the HTTP status is 301, 302 or 303 the client will follow 56 the location. 57 :param max_follow_redirect: max number of redirection. If max is reached 58 the RedirectLimit exception is raised. 59 :param pool_instance: a pool instance inherited from 60 `restkit.pool.PoolInterface` 61 :param ssl_args: ssl arguments. See http://docs.python.org/library/ssl.html 62 for more information. 63 """ 64 self._sock = None 65 self.timeout = timeout 66 self.headers = [] 67 self.req_headers = [] 68 self.ua = USER_AGENT 69 self.url = None 70 71 self.follow_redirect = follow_redirect 72 self.nb_redirections = max_follow_redirect 73 self.force_follow_redirect = force_follow_redirect 74 self.method = 'GET' 75 self.body = None 76 self.response_body = StringIO() 77 self.final_url = None 78 79 # build filter lists 80 self.filters = Filters(filters) 81 self.ssl_args = ssl_args or {} 82 83 if not pool_instance: 84 self.should_close = True 85 self.pool = None 86 else: 87 self.pool = pool_instance 88 self.should_close = False 89 90 if response_class is not None: 91 self.response_class = response_class
92
93 - def make_connection(self):
94 """ initate a connection if needed or reuse a socket""" 95 96 # apply on connect filters 97 self.filters.apply("on_connect", self) 98 if self._sock is not None: 99 return self._sock 100 101 addr = (self.host, self.port) 102 s = None 103 # if we defined a pool use it 104 if self.pool is not None: 105 s = self.pool.get(addr) 106 107 if not s: 108 # pool is empty or we don't use a pool 109 if self.uri.scheme == "https": 110 s = sock.connect(addr, True, self.timeout, **self.ssl_args) 111 else: 112 s = sock.connect(addr, False, self.timeout) 113 return s
114
115 - def clean_connections(self):
116 sock.close(self._sock) 117 self._sock = None 118 if hasattr(self.pool,'clear'): 119 self.pool.clear_host((self.host, self.port))
120
121 - def release_connection(self, address, socket):
122 if self.should_close: 123 sock.close(self._sock) 124 else: 125 self.pool.put(address, self._sock) 126 self._sock = None
127
128 - def parse_url(self, url):
129 """ parse url and get host/port""" 130 self.uri = urlparse.urlparse(url) 131 if self.uri.scheme not in ('http', 'https'): 132 raise InvalidUrl("None valid url") 133 134 host, port = util.parse_netloc(self.uri) 135 self.host = host 136 self.port = port
137
138 - def set_body(self, body, headers, chunked=False):
139 """ set HTTP body and manage form if needed """ 140 content_type = headers.get('CONTENT-TYPE') 141 content_length = headers.get('CONTENT-LENGTH') 142 if not body: 143 if content_type is not None: 144 self.headers.append(('Content-Type', content_type)) 145 if self.method in ('POST', 'PUT'): 146 self.headers.append(("Content-Length", "0")) 147 return 148 149 # set content lengh if needed 150 if isinstance(body, dict): 151 if content_type is not None and \ 152 content_type.startswith("multipart/form-data"): 153 type_, opts = cgi.parse_header(content_type) 154 boundary = opts.get('boundary', uuid.uuid4().hex) 155 body, self.headers = multipart_form_encode(body, 156 self.headers, boundary) 157 else: 158 content_type = "application/x-www-form-urlencoded; charset=utf-8" 159 body = form_encode(body) 160 elif hasattr(body, "boundary"): 161 content_type = "multipart/form-data; boundary=%s" % body.boundary 162 content_length = body.get_size() 163 164 if not content_type: 165 content_type = 'application/octet-stream' 166 if hasattr(body, 'name'): 167 content_type = mimetypes.guess_type(body.name)[0] 168 169 if not content_length: 170 if hasattr(body, 'fileno'): 171 try: 172 body.flush() 173 except IOError: 174 pass 175 content_length = str(os.fstat(body.fileno())[6]) 176 elif hasattr(body, 'getvalue'): 177 try: 178 content_length = str(len(body.getvalue())) 179 except AttributeError: 180 pass 181 elif isinstance(body, types.StringTypes): 182 body = util.to_bytestring(body) 183 content_length = len(body) 184 185 if content_length: 186 self.headers.append(("Content-Length", content_length)) 187 if content_type is not None: 188 self.headers.append(('Content-Type', content_type)) 189 190 elif not chunked: 191 raise RequestError("Can't determine content length and " + 192 "Transfer-Encoding header is not chunked") 193 194 self.body = body
195 196
197 - def request(self, url, method='GET', body=None, headers=None):
198 """ make effective request 199 200 :param url: str, url string 201 :param method: str, by default GET. http verbs 202 :param body: the body, could be a string, an iterator or a file-like object 203 :param headers: dict or list of tupple, http headers 204 """ 205 self._sock = None 206 self.url = url 207 self.final_url = url 208 self.parse_url(url) 209 self.method = method.upper() 210 211 self.init_headers = copy.copy(headers or []) 212 self.headers = [] 213 214 # headers are better as list 215 headers = headers or [] 216 if isinstance(headers, dict): 217 headers = headers.items() 218 219 chunked = False 220 221 # normalize headers 222 search_headers = ('USER-AGENT', 'CONTENT-TYPE', 223 'CONTENT-LENGTH', 'ACCEPT-ENCODING', 224 'TRANSFER-ENCODING', 'CONNECTION', 'HOST') 225 found_headers = {} 226 new_headers = copy.copy(headers) 227 for (name, value) in headers: 228 uname = name.upper() 229 if uname in search_headers: 230 if uname == 'TRANSFER-ENCODING': 231 if value.lower() == "chunked": 232 chunked = True 233 else: 234 found_headers[uname] = value 235 new_headers.remove((name, value)) 236 237 self.headers = new_headers 238 self.chunked = chunked 239 240 # set body 241 self.set_body(body, found_headers, chunked=chunked) 242 243 # force connection close if needed 244 if found_headers.get('CONNECTION') == "close": 245 self.should_close = True 246 elif self.pool is None: 247 found_headers['CONNECTION'] = "close" 248 249 self.found_headers = found_headers 250 251 # Finally do the request 252 return self.do_send()
253
254 - def _req_headers(self):
255 # by default all connections are HTTP/1.1 256 if self.version == (1,1): 257 httpver = "HTTP/1.1" 258 else: 259 httpver = "HTTP/1.0" 260 261 # request path 262 path = self.uri.path or "/" 263 req_path = urlparse.urlunparse(('','', path, '', 264 self.uri.query, self.uri.fragment)) 265 266 267 ua = self.found_headers.get('USER-AGENT') 268 accept_encoding = self.found_headers.get('ACCEPT-ENCODING') 269 connection = self.found_headers.get('CONNECTION') 270 271 # default host header 272 try: 273 host = self.uri.netloc.encode('ascii') 274 except UnicodeEncodeError: 275 host = self.uri.netloc.encode('idna') 276 host = self.found_headers.get('HOST') or host 277 278 # build final request headers 279 req_headers = [ 280 "%s %s %s\r\n" % (self.method, req_path, httpver), 281 "Host: %s\r\n" % host, 282 "User-Agent: %s\r\n" % ua or USER_AGENT, 283 "Accept-Encoding: %s\r\n" % accept_encoding or 'identity' 284 ] 285 286 if connection is not None: 287 req_headers.append("Connection: %s\r\n" % connection) 288 289 req_headers.extend(["%s: %s\r\n" % (k, v) for k, v in self.headers]) 290 req_headers.append('\r\n') 291 return req_headers
292
293 - def do_send(self):
294 tries = 2 295 while True: 296 try: 297 # get socket 298 self._sock = self.make_connection() 299 300 # apply on request filters 301 self.filters.apply("on_request", self, tries) 302 303 # build request headers 304 self.req_headers = req_headers = self._req_headers() 305 306 # send request 307 log.info('Start request: %s %s', self.method, self.url) 308 log.debug("Request headers: [%s]", req_headers) 309 310 self._sock.sendall("".join(req_headers)) 311 312 if self.body is not None: 313 if hasattr(self.body, 'read'): 314 if hasattr(self.body, 'seek'): self.body.seek(0) 315 sock.sendfile(self._sock, self.body, self.chunked) 316 elif isinstance(self.body, types.StringTypes): 317 sock.send(self._sock, self.body, self.chunked) 318 else: 319 sock.sendlines(self._sock, self.body, self.chunked) 320 321 if self.chunked: # final chunk 322 sock.send_chunk(self._sock, "") 323 324 return self.start_response() 325 except socket.gaierror, e: 326 self.clean_connections() 327 raise 328 except socket.error, e: 329 if e[0] not in (errno.EAGAIN, errno.ECONNABORTED, errno.EPIPE, 330 errno.ECONNREFUSED, errno.ECONNRESET) or tries <= 0: 331 self.clean_connections() 332 raise 333 if e[0] in (errno.EPIPE, errno.ECONNRESET): 334 self.clean_connections() 335 except (KeyboardInterrupt, SystemExit): 336 break 337 except: 338 if tries <= 0: 339 raise 340 # we don't know what happend. 341 self.clean_connections() 342 time.sleep(0.2) 343 tries -= 1
344 345
346 - def do_redirect(self, response, location):
347 """ follow redirections if needed""" 348 if self.nb_redirections <= 0: 349 raise RedirectLimit("Redirection limit is reached") 350 351 if not location: 352 raise RequestError('no Location header') 353 354 new_uri = urlparse.urlparse(location) 355 if not new_uri.netloc: # we got a relative url 356 absolute_uri = "%s://%s" % (self.uri.scheme, self.uri.netloc) 357 location = urlparse.urljoin(absolute_uri, location) 358 359 log.debug("Redirect to %s" % location) 360 361 self.final_url = location 362 response.body.read() 363 self.nb_redirections -= 1 364 365 self.release_connection((self.host, self.port), self._sock) 366 return self.request(location, self.method, self.body, self.init_headers)
367
368 - def start_response(self):
369 """ 370 Get headers, set Body object and return HttpResponse 371 """ 372 # read headers 373 parser = http.ResponseParser(self._sock, 374 release_source = lambda:self.release_connection( 375 (self.host, self.port), self._sock)) 376 resp = parser.next() 377 378 log.debug("Start response: %s", resp.status) 379 log.debug("Response headers: [%s]", resp.headers) 380 381 location = None 382 for hdr_name, hdr_value in resp.headers: 383 if hdr_name.lower() == "location": 384 location = hdr_value 385 break 386 387 if self.follow_redirect: 388 if resp.status_int in (301, 302, 307): 389 if self.method in ('GET', 'HEAD') or \ 390 self.force_follow_redirect: 391 if self.method not in ('GET', 'HEAD') and \ 392 hasattr(self.body, 'seek'): 393 self.body.seek(0) 394 return self.do_redirect(resp, location) 395 elif resp.status_int == 303 and self.method in ('GET', 396 'HEAD'): 397 # only 'GET' is possible with this status 398 # according the rfc 399 return self.do_redirect(resp, location) 400 401 402 # apply on response filters 403 self.filters.apply("on_response", self) 404 405 self.final_url = location or self.final_url 406 log.debug("Return response: %s" % self.final_url) 407 if self.method == "HEAD": 408 resp.body = StringIO() 409 410 return self.response_class(resp, self.final_url)
411