1
2
3
4
5
6 import cgi
7 import errno
8 import logging
9 import mimetypes
10 import os
11 import socket
12 import time
13 try:
14 from cStringIO import StringIO
15 except ImportError:
16 from StringIO import StringIO
17 import types
18 import urlparse
19 import uuid
20
21 from restkit import __version__
22 from restkit.errors import RequestError, InvalidUrl, RedirectLimit, \
23 AlreadyRead
24 from restkit.filters import Filters
25 from restkit.forms import multipart_form_encode, form_encode
26 from restkit.util import sock
27 from restkit import tee
28 from restkit import util
29 from restkit.util.misc import deprecated_property
30 from restkit import http
31
32 MAX_FOLLOW_REDIRECTS = 5
33
34 USER_AGENT = "restkit/%s" % __version__
35
36 log = logging.getLogger(__name__)
39 """ Http Response object returned by HttpConnction"""
40
41 charset = "utf8"
42 unicode_errors = 'strict'
43
44 - def __init__(self, response, final_url):
45 self.response = response
46 self.status = response.status
47 self.status_int = response.status_int
48 self.version = response.version
49 self.headerslist = response.headers
50 self.final_url = final_url
51
52 headers = {}
53 for key, value in response.headers:
54 headers[key.lower()] = value
55 self.headers = headers
56 self.closed = False
57
59 try:
60 return getattr(self, key)
61 except AttributeError:
62 pass
63 return self.headers[key.lower()]
64
66 return (key.lower() in self.headers)
67
69 for item in list(self.headers.items()):
70 yield item
71
72 - def body_string(self, charset=None, unicode_errors="strict"):
73 """ return body string, by default in bytestring """
74 if self.closed or self.response.body.closed:
75 raise AlreadyRead("The response have already been read")
76 body = self.response.body.read()
77 if charset is not None:
78 try:
79 body = body.decode(charset, unicode_errors)
80 except UnicodeDecodeError:
81 pass
82 self.close()
83 return body
84
85 - def body_stream(self):
86 """ return full body stream """
87 if self.closed or self.response.body.closed:
88 raise AlreadyRead("The response have already been read")
89 return self.response.body
90
92 """ release the socket """
93 self.closed = True
94 self.response.body.close()
95
96 @property
98 """ body in bytestring """
99 return self.body_string()
100
101 body = deprecated_property(
102 body, 'body', 'use body_string() instead',
103 warning=True)
104
105 @property
106 - def body_file(self):
107 """ return body as a file like object"""
108 return self.body_stream()
109
110 body_file = deprecated_property(
111 body_file, 'body_file', 'use body_stream() instead',
112 warning=True)
113
114 @property
115 - def unicode_body(self):
116 """ like body but converted to unicode"""
117 if not self.charset:
118 raise AttributeError(
119 "You cannot access HttpResponse.unicode_body unless charset is set")
120 body = self.body_string()
121 return body.decode(self.charset, self.unicode_errors)
122
123 unicode_body = deprecated_property(
124 unicode_body, 'unicode_body', 'replaced by body_string()',
125 warning=True)
126
128 """ Http Connection object. """
129
130 version = (1, 1)
131 response_class = HttpResponse
132
138
139 """ HttpConnection constructor
140
141 :param timeout: socket timeout
142 :param filters: list, list of http filters. see the doc of http filters
143 for more info
144 :param follow_redirect: boolean, by default is false. If true,
145 if the HTTP status is 301, 302 or 303 the client will follow
146 the location.
147 :param max_follow_redirect: max number of redirection. If max is reached
148 the RedirectLimit exception is raised.
149 :param pool_instance: a pool instance inherited from
150 `restkit.pool.PoolInterface`
151 :param ssl_args: ssl arguments. See http://docs.python.org/library/ssl.html
152 for more information.
153 """
154 self._sock = None
155 self.timeout = timeout
156 self.headers = []
157 self.req_headers = []
158 self.ua = USER_AGENT
159 self.url = None
160
161 self.follow_redirect = follow_redirect
162 self.nb_redirections = max_follow_redirect
163 self.force_follow_redirect = force_follow_redirect
164 self.method = 'GET'
165 self.body = None
166 self.response_body = StringIO()
167 self.final_url = None
168
169
170 self.filters = Filters(filters)
171 self.ssl_args = ssl_args or {}
172
173 if not pool_instance:
174 self.should_close = True
175 self.pool = None
176 else:
177 self.pool = pool_instance
178 self.should_close = False
179
180 if response_class is not None:
181 self.response_class = response_class
182
184 """ initate a connection if needed or reuse a socket"""
185
186
187 self.filters.apply("on_connect", self)
188 if self._sock is not None:
189 return self._sock
190
191 addr = (self.host, self.port)
192 s = None
193
194 if self.pool is not None:
195 s = self.pool.get(addr)
196
197 if not s:
198
199 if self.uri.scheme == "https":
200 s = sock.connect(addr, True, self.timeout, **self.ssl_args)
201 else:
202 s = sock.connect(addr, False, self.timeout)
203 return s
204
210
217
219 """ parse url and get host/port"""
220 self.uri = urlparse.urlparse(url)
221 if self.uri.scheme not in ('http', 'https'):
222 raise InvalidUrl("None valid url")
223
224 host, port = util.parse_netloc(self.uri)
225 self.host = host
226 self.port = port
227
228 - def set_body(self, body, content_type=None, content_length=None,
229 chunked=False):
230 """ set HTTP body and manage form if needed """
231 if not body:
232 if content_type is not None:
233 self.headers.append(('Content-Type', content_type))
234 if self.method in ('POST', 'PUT'):
235 self.headers.append(("Content-Length", "0"))
236 return
237
238
239 if isinstance(body, dict):
240 if content_type is not None and \
241 content_type.startswith("multipart/form-data"):
242 type_, opts = cgi.parse_header(content_type)
243 boundary = opts.get('boundary', uuid.uuid4().hex)
244 body, self.headers = multipart_form_encode(body,
245 self.headers, boundary)
246 else:
247 content_type = "application/x-www-form-urlencoded; charset=utf-8"
248 body = form_encode(body)
249 elif hasattr(body, "boundary"):
250 content_type = "multipart/form-data; boundary=%s" % body.boundary
251 content_length = body.get_size()
252
253 if not content_type:
254 content_type = 'application/octet-stream'
255 if hasattr(body, 'name'):
256 content_type = mimetypes.guess_type(body.name)[0]
257
258 if not content_length:
259 if hasattr(body, 'fileno'):
260 try:
261 body.flush()
262 except IOError:
263 pass
264 content_length = str(os.fstat(body.fileno())[6])
265 elif hasattr(body, 'getvalue'):
266 try:
267 content_length = str(len(body.getvalue()))
268 except AttributeError:
269 pass
270 elif isinstance(body, types.StringTypes):
271 body = util.to_bytestring(body)
272 content_length = len(body)
273
274 if content_length:
275 self.headers.append(("Content-Length", content_length))
276 if content_type is not None:
277 self.headers.append(('Content-Type', content_type))
278
279 elif not chunked:
280 raise RequestError("Can't determine content length and" +
281 "Transfer-Encoding header is not chunked")
282
283 self.body = body
284
285
286 - def request(self, url, method='GET', body=None, headers=None):
287 """ make effective request
288
289 :param url: str, url string
290 :param method: str, by default GET. http verbs
291 :param body: the body, could be a string, an iterator or a file-like object
292 :param headers: dict or list of tupple, http headers
293 """
294 self._sock = None
295 self.url = url
296 self.final_url = url
297 self.parse_url(url)
298 self.method = method.upper()
299 self.headers = []
300
301
302 headers = headers or []
303 if isinstance(headers, dict):
304 headers = list(headers.items())
305
306 ua = USER_AGENT
307 content_length = None
308 accept_encoding = 'identity'
309 chunked = False
310 content_type = None
311 connection = None
312
313
314
315 try:
316 host = self.uri.netloc.encode('ascii')
317 except UnicodeEncodeError:
318 host = self.uri.netloc.encode('idna')
319
320
321 for name, value in headers:
322 name = name.title()
323 if name == "User-Agent":
324 ua = value
325 elif name == "Content-Type":
326 content_type = value
327 elif name == "Content-Length":
328 content_length = str(value)
329 elif name == "Accept-Encoding":
330 accept_encoding = value
331 elif name == "Host":
332 host = value
333 elif name == "Transfer-Encoding":
334 if value.lower() == "chunked":
335 chunked = True
336 self.headers.append((name, value))
337 elif name == "Connection":
338 connection = value
339 else:
340 if not isinstance(value, types.StringTypes):
341 value = str(value)
342 self.headers.append((name, value))
343
344 self.set_body(body, content_type=content_type,
345 content_length=content_length, chunked=chunked)
346
347 self.ua = ua
348 self.chunked = chunked
349 self.host_hdr = host
350 self.accept_encoding = accept_encoding
351 if connection == "close":
352 self.should_close = True
353 elif self.pool is not None:
354 self.should_close = False
355
356
357 return self.do_send()
358
360
361 if self.version == (1,1):
362 httpver = "HTTP/1.1"
363 else:
364 httpver = "HTTP/1.0"
365
366
367 path = self.uri.path or "/"
368 req_path = urlparse.urlunparse(('','', path, '',
369 self.uri.query, self.uri.fragment))
370
371
372 req_headers = [
373 "%s %s %s\r\n" % (self.method, req_path, httpver),
374 "Host: %s\r\n" % self.host_hdr,
375 "User-Agent: %s\r\n" % self.ua,
376 "Accept-Encoding: %s\r\n" % self.accept_encoding
377 ]
378 req_headers.extend(["%s: %s\r\n" % (k, v) for k, v in self.headers])
379 req_headers.append('\r\n')
380 return req_headers
381
383 tries = 2
384 while True:
385 try:
386
387 self._sock = self.make_connection()
388
389
390 self.filters.apply("on_request", self)
391
392
393 self.req_headers = req_headers = self._req_headers()
394
395
396 log.info('Start request: %s %s', self.method, self.url)
397 log.debug("Request headers: [%s]", req_headers)
398
399 self._sock.sendall("".join(req_headers))
400
401 if self.body is not None:
402 if hasattr(self.body, 'read'):
403 if hasattr(self.body, 'seek'): self.body.seek(0)
404 sock.sendfile(self._sock, self.body, self.chunked)
405 elif isinstance(self.body, types.StringTypes):
406 sock.send(self._sock, self.body, self.chunked)
407 else:
408 sock.sendlines(self._sock, self.body, self.chunked)
409
410 if self.chunked:
411 sock.send_chunk(self._sock, "")
412
413 return self.start_response()
414 except socket.gaierror, e:
415 self.clean_connections()
416 raise
417 except socket.error, e:
418 if e[0] not in (errno.EAGAIN, errno.ECONNABORTED, errno.EPIPE,
419 errno.ECONNREFUSED, errno.ECONNRESET) or tries <= 0:
420 self.clean_connections()
421 raise
422 if e[0] in (errno.EPIPE, errno.ECONNRESET):
423 self.clean_connections()
424 except:
425 if tries <= 0:
426 raise
427
428 self.clean_connections()
429 time.sleep(0.2)
430 tries -= 1
431
432
434 """ follow redirections if needed"""
435 if self.nb_redirections <= 0:
436 raise RedirectLimit("Redirection limit is reached")
437
438 if not location:
439 raise RequestError('no Location header')
440
441 new_uri = urlparse.urlparse(location)
442 if not new_uri.netloc:
443 absolute_uri = "%s://%s" % (self.uri.scheme, self.uri.netloc)
444 location = urlparse.urljoin(absolute_uri, location)
445
446 log.debug("Redirect to %s" % location)
447
448 self.final_url = location
449 response.body.read()
450 self.nb_redirections -= 1
451 sock.close(self._sock)
452 return self.request(location, self.method, self.body, self.headers)
453
455 """
456 Get headers, set Body object and return HttpResponse
457 """
458
459 parser = http.ResponseParser(self._sock,
460 release_source = lambda:self.release_connection(
461 (self.host, self.port), self._sock))
462 resp = parser.next()
463
464 log.debug("Start response: %s", resp.status)
465 log.debug("Response headers: [%s]", resp.headers)
466
467 location = None
468 for hdr_name, hdr_value in resp.headers:
469 if hdr_name.lower() == "location":
470 location = hdr_value
471 break
472
473 if self.follow_redirect:
474 if resp.status_int in (301, 302, 307):
475 if self.method in ('GET', 'HEAD') or \
476 self.force_follow_redirect:
477 if self.method not in ('GET', 'HEAD') and \
478 hasattr(self.body, 'seek'):
479 self.body.seek(0)
480 return self.do_redirect(resp, location)
481 elif resp.status_int == 303 and self.method in ('GET',
482 'HEAD'):
483
484
485 return self.do_redirect(resp, location)
486
487
488
489 self.filters.apply("on_response", self)
490
491 self.final_url = location or self.final_url
492 log.debug("Return response: %s" % self.final_url)
493 if self.method == "HEAD":
494 resp.body = StringIO()
495
496 return self.response_class(resp, self.final_url)
497