1
2
3
4
5
6 import errno
7 import gzip
8 import logging
9 import os
10 import socket
11 import time
12 from StringIO import StringIO
13 import types
14 import urlparse
15
16 from restkit import __version__
17 from restkit.errors import RequestError, InvalidUrl, RedirectLimit
18 from restkit.parser import Parser
19 from restkit import sock
20 from restkit import tee
21 from restkit import util
22
23 MAX_FOLLOW_REDIRECTS = 5
24
25 USER_AGENT = "restkit/%s" % __version__
26
27 log = logging.getLogger(__name__)
30 """ Http Response object returned by HttpConnction"""
31
32 charset = "utf8"
33 unicode_errors = 'strict'
34
36 self.http_client = http_client
37 self.status = self.http_client.parser.status
38 self.status_int = self.http_client.parser.status_int
39 self.version = self.http_client.parser.version
40 self.headerslist = self.http_client.parser.headers
41 self.final_url = self.http_client.final_url
42
43 headers = {}
44 for key, value in self.http_client.parser.headers:
45 headers[key.lower()] = value
46 self.headers = headers
47
48 encoding = headers.get('content-encoding', None)
49 if encoding in ('gzip', 'deflate'):
50 self._body = gzip.GzipFile(fileobj=self.http_client.response_body)
51 else:
52 self._body = self.http_client.response_body
53 self._body_eof = False
54
56 try:
57 return getattr(self, key)
58 except AttributeError:
59 pass
60 return self.headers[key]
61
63 return (key in self.headers)
64
66 for item in list(self.headers.items()):
67 yield item
68
69 @property
71 """ body in bytestring """
72 if self._body_eof:
73 self._body.seek(0)
74 self._body_eof = True
75 ret = self._body.read()
76 self._body.seek(0)
77 return ret
78
79 @property
80 - def body_file(self):
81 """ return body as a file like object"""
82 return self._body
83
84 @property
85 - def unicode_body(self):
86 """ like body but converted to unicode"""
87 if not self.charset:
88 raise AttributeError(
89 "You cannot access HttpResponse.unicode_body unless charset is set")
90 return self.body.decode(self.charset, self.unicode_errors)
91
93 """ Http Connection object. """
94
95 version = (1, 1)
96 response_class = HttpResponse
97
98
103
104 """ HttpConnection constructor
105
106 :param timeout: socket timeout
107 :param filters: list, list of http filters. see the doc of http filters
108 for more info
109 :param follow_redirect: boolean, by default is false. If true,
110 if the HTTP status is 301, 302 or 303 the client will follow
111 the location.
112 :param max_follow_redirect: max number of redirection. If max is reached
113 the RedirectLimit exception is raised.
114 :param key_file: the key fle to use with ssl
115 :param cert_file: the cert file to use with ssl
116 :param pool_instance: a pool instance inherited from
117 `restkit.pool.PoolInterface`
118 """
119 self._sock = None
120 self.timeout = timeout
121 self.headers = []
122 self.req_headers = []
123 self.ua = USER_AGENT
124 self.url = None
125
126 self.follow_redirect = follow_redirect
127 self.nb_redirections = max_follow_redirect
128 self.force_follow_redirect = force_follow_redirect
129 self.method = 'GET'
130 self.body = None
131 self.response_body = StringIO()
132 self.final_url = None
133
134
135 self.filters = filters or []
136 self.request_filters = []
137 self.response_filters = []
138 self.cert_file = cert_file
139 self.key_file = key_file
140
141 for f in self.filters:
142 self._add_filter(f)
143
144 if not pool_instance:
145 self.should_close = True
146 self.connections = None
147 else:
148 self.connections = pool_instance
149 self.should_close = False
150
151 if response_class is not None:
152 self.response_class = response_class
153
157
159 if hasattr(f, 'on_request'):
160 self.request_filters.append(f)
161
162 if hasattr(f, 'on_response'):
163 self.response_filters.append(f)
164
166 for i, f1 in enumerate(self.filters):
167 if f == f1: del self.filters[i]
168
169 if hasattr(f, 'on_request'):
170 for i, f1 in enumerate(self.request_filters):
171 if f == f1: del self.request_filters[i]
172
173 if hasattr(f, 'on_response'):
174 for i, f1 in enumerate(self.response_filters):
175 if f == f1: del self.response_filters[i]
176
178 """ initate a connection if needed or reuse a socket"""
179 addr = (self.host, self.port)
180 s = None
181
182 if self.connections is not None:
183 s = self.connections.get(addr)
184
185 if not s:
186
187 if self.uri.scheme == "https":
188 s = sock.connect(addr, self.timeout, True,
189 self.key_file, self.cert_file)
190 else:
191 s = sock.connect(addr, self.timeout)
192 return s
193
195 sock.close(self._sock)
196 if hasattr(self.connections,'clean'):
197 self.connections.clean((self.host, self.port))
198
200 if not self.connections:
201 sock.close(socket)
202 else:
203 self.connections.put(address, self._sock)
204
206 """ parse url and get host/port"""
207 self.uri = urlparse.urlparse(url)
208 if self.uri.scheme not in ('http', 'https'):
209 raise InvalidUrl("None valid url")
210
211 host = self.uri.netloc
212 i = host.rfind(':')
213 j = host.rfind(']')
214 if i > j:
215 try:
216 port = int(host[i+1:])
217 except ValueError:
218 raise InvalidUrl("nonnumeric port: '%s'" % host[i+1:])
219 host = host[:i]
220 else:
221
222 if self.uri.scheme == "https":
223 port = 443
224 else:
225 port = 80
226
227 if host and host[0] == '[' and host[-1] == ']':
228 host = host[1:-1]
229
230 self.host = host
231 self.port = port
232
233 - def request(self, url, method='GET', body=None, headers=None):
234 """ make effective request
235
236 :param url: str, url string
237 :param method: str, by default GET. http verbs
238 :param body: the body, could be a string, an iterator or a file-like object
239 :param headers: dict or list of tupple, http headers
240 """
241 self.parser = Parser.parse_response(should_close=self.should_close)
242 self._sock = None
243 self.url = url
244 self.final_url = url
245 self.parse_url(url)
246 self.method = method.upper()
247
248
249
250 headers = headers or []
251 if isinstance(headers, dict):
252 headers = list(headers.items())
253
254 ua = USER_AGENT
255 normalized_headers = []
256 content_len = None
257 accept_encoding = 'identity'
258 chunked = False
259
260
261 try:
262 host = self.uri.netloc.encode('ascii')
263 except UnicodeEncodeError:
264 host = self.uri.netloc.encode('idna')
265
266
267 for name, value in headers:
268 name = name.title()
269 if name == "User-Agent":
270 ua = value
271 elif name == "Content-Length":
272 content_len = str(value)
273 elif name == "Accept-Encoding":
274 accept_encoding = 'identity'
275 elif name == "Host":
276 host = value
277 elif name == "Transfer-Encoding":
278 if value.lower() == "chunked":
279 chunked = True
280 normalized_headers.append((name, value))
281 else:
282 if not isinstance(value, types.StringTypes):
283 value = str(value)
284 normalized_headers.append((name, value))
285
286
287 if body and body is not None:
288 if not content_len:
289 if hasattr(body, 'fileno'):
290 try:
291 body.flush()
292 except IOError:
293 pass
294 content_len = str(os.fstat(body.fileno())[6])
295 elif hasattr(body, 'len'):
296 try:
297 content_len = str(body.len)
298 except AttributeError:
299 pass
300 elif isinstance(body, types.StringTypes):
301 body = util.to_bytestring(body)
302 content_len = len(body)
303
304 if content_len:
305 normalized_headers.append(("Content-Length", content_len))
306 elif not chunked:
307 raise RequestError("Can't determine content length and" +
308 "Transfer-Encoding header is not chunked")
309
310 if self.method in ('POST', 'PUT') and not body:
311 normalized_headers.append(("Content-Length", "0"))
312
313 self.body = body
314 self.headers = normalized_headers
315 self.ua = ua
316 self.chunked = chunked
317 self.host_hdr = host
318 self.accept_encoding = accept_encoding
319
320
321 return self.do_send()
322
324
325 if self.version == (1,1):
326 httpver = "HTTP/1.1"
327 else:
328 httpver = "HTTP/1.0"
329
330
331 path = self.uri.path or "/"
332 req_path = urlparse.urlunparse(('','', path, '',
333 self.uri.query, self.uri.fragment))
334
335
336 req_headers = [
337 "%s %s %s\r\n" % (self.method, req_path, httpver),
338 "Host: %s\r\n" % self.host_hdr,
339 "User-Agent: %s\r\n" % self.ua,
340 "Accept-Encoding: %s\r\n" % self.accept_encoding
341 ]
342 req_headers.extend(["%s:%s\r\n" % (k, v) for k, v in self.headers])
343 req_headers.append('\r\n')
344 return req_headers
345
347 tries = 2
348 while True:
349 try:
350
351 self._sock = self.make_connection()
352
353
354 for bf in self.request_filters:
355 bf.on_request(self)
356
357
358 self.req_headers = req_headers = self._req_headers()
359
360
361 log.info('Start request: %s %s' % (self.method, self.url))
362 log.debug("Request headers: [%s]" % str(req_headers))
363
364 self._sock.sendall("".join(req_headers))
365
366 if self.body is not None:
367 if hasattr(self.body, 'read'):
368 if hasattr(self.body, 'seek'): self.body.seek(0)
369 sock.sendfile(self._sock, self.body, self.chunked)
370 elif isinstance(self.body, types.StringTypes):
371 sock.send(self._sock, self.body, self.chunked)
372 else:
373 sock.sendlines(self._sock, self.body, self.chunked)
374
375 if self.chunked:
376 sock.send_chunk(self._sock, "")
377
378 return self.start_response()
379 except socket.gaierror, e:
380 self.clean_connections()
381 raise
382 except socket.error, e:
383 if e[0] not in (errno.EAGAIN, errno.ECONNABORTED, errno.EPIPE,
384 errno.ECONNREFUSED) or tries <= 0:
385 self.clean_connections()
386 raise
387 if e[0] == errno.EPIPE:
388 log.debug("Got EPIPE")
389 self.clean_connections()
390 except:
391 if tries <= 0:
392 raise
393
394 self.clean_connections()
395 time.sleep(0.2)
396 tries -= 1
397
398
400 """ follow redirections if needed"""
401 if self.nb_redirections <= 0:
402 raise RedirectLimit("Redirection limit is reached")
403
404 location = self.parser.headers_dict.get('Location')
405 if not location:
406 raise RequestError('no Location header')
407
408 new_uri = urlparse.urlparse(location)
409 if not new_uri.netloc:
410 absolute_uri = "%s://%s" % (self.uri.scheme, self.uri.netloc)
411 location = urlparse.urljoin(absolute_uri, location)
412
413 log.debug("Redirect to %s" % location)
414
415 self.final_url = location
416 self.response_body.read()
417 self.nb_redirections -= 1
418 sock.close(self._sock)
419 return self.request(location, self.method, self.body, self.headers)
420
422 """
423 Get headers, set Body object and return HttpResponse
424 """
425
426 headers = []
427 buf = StringIO()
428 data = self._sock.recv(sock.CHUNK_SIZE)
429 buf.write(data)
430 buf2 = self.parser.filter_headers(headers, buf)
431 if not buf2:
432 while True:
433 data = self._sock.recv(sock.CHUNK_SIZE)
434 if not data:
435 break
436 buf.write(data)
437 buf2 = self.parser.filter_headers(headers, buf)
438 if buf2:
439 break
440
441 log.debug("Start response: %s" % str(self.parser.status_line))
442 log.debug("Response headers: [%s]" % str(self.parser.headers))
443
444 if (not self.parser.content_len and not self.parser.is_chunked):
445 if self.parser.should_close:
446
447
448 log.debug("No content len an not chunked transfer, get body")
449 while True:
450 try:
451 chunk = self._sock.recv(sock.CHUNK_SIZE)
452 except socket.error:
453 break
454 if not chunk:
455 break
456 buf2.write(chunk)
457 sock.close(self._sock)
458 buf2.seek(0)
459 self.response_body = buf2
460
461 elif self.method == "HEAD":
462 self.response_body = StringIO()
463 sock.close(self._sock)
464 else:
465 self.response_body = tee.TeeInput(self._sock, self.parser, buf2,
466 maybe_close=lambda: self.release_connection(
467 self.uri.netloc, self._sock))
468
469
470 for af in self.response_filters:
471 af.on_response(self)
472
473 if self.follow_redirect:
474 if self.parser.status_int in (301, 302, 307):
475 if self.method in ('GET', 'HEAD') or \
476 self.force_follow_redirect:
477 if self.method not in ('GET', 'HEAD') and \
478 hasattr(self.body, 'seek'):
479 self.body.seek(0)
480 return self.do_redirect()
481 elif self.parser.status_int == 303 and self.method in ('GET',
482 'HEAD'):
483
484
485 return self.do_redirect()
486
487
488 self.final_url = self.parser.headers_dict.get('Location',
489 self.final_url)
490 log.debug("Return response: %s" % self.final_url)
491 return self.response_class(self)
492