1
2
3
4
5
6 import ctypes
7 import errno
8 import gzip
9 import logging
10 import os
11 import socket
12 import StringIO
13 import urlparse
14
15 from restkit import __version__
16 from restkit.errors import RequestError, InvalidUrl, RedirectLimit
17 from restkit.parser import Parser
18 from restkit import sock
19 from restkit import tee
20 from restkit import util
21
22 MAX_FOLLOW_REDIRECTS = 5
23
24 USER_AGENT = "restkit/%s" % __version__
25
26 log = logging.getLogger(__name__)
29 """ Http Response object returned by HttpConnction"""
30
31 charset = "utf8"
32 unicode_errors = 'strict'
33
35 self.http_client = http_client
36 self.status = self.http_client.parser.status
37 self.status_int = self.http_client.parser.status_int
38 self.version = self.http_client.parser.version
39 self.headerslist = self.http_client.parser.headers
40 self.final_url = self.http_client.final_url
41
42 headers = {}
43 for key, value in self.http_client.parser.headers:
44 headers[key.lower()] = value
45 self.headers = headers
46
47 encoding = headers.get('content-encoding', None)
48 if encoding in ('gzip', 'deflate'):
49 self._body = gzip.GzipFile(fileobj=self.http_client.response_body)
50 else:
51 self._body = self.http_client.response_body
52 self._body_eof = False
53
55 try:
56 return getattr(self, key)
57 except AttributeError:
58 pass
59 return self.headers[key]
60
62 return (key in self.headers)
63
65 for item in list(self.headers.items()):
66 yield item
67
68 @property
70 """ body in bytestring """
71 if self._body_eof:
72 self._body.seek(0)
73 self._body_eof = True
74 ret = self._body.read()
75 self._body.seek(0)
76 return ret
77
78 @property
79 - def body_file(self):
80 """ return body as a file like object"""
81 return self._body
82
83 @property
84 - def unicode_body(self):
85 """ like body but converted to unicode"""
86 if not self.charset:
87 raise AttributeError(
88 "You cannot access HttpResponse.unicode_body unless charset is set")
89 return self.body.decode(self.charset, self.unicode_errors)
90
92 """ Http Connection object. """
93
94 version = (1, 1)
95 response_class = HttpResponse
96
97
98 - def __init__(self, timeout=sock._GLOBAL_DEFAULT_TIMEOUT,
99 filters=None, follow_redirect=False, force_follow_redirect=False,
100 max_follow_redirect=MAX_FOLLOW_REDIRECTS, key_file=None,
101 cert_file=None, pool_instance=None, socket=None,
102 response_class=None):
103
104 """ HttpConnection constructor
105
106 :param timeout: socket timeout
107 :param filters: list, list of http filters. see the doc of http filters
108 for more info
109 :param follow_redirect: boolean, by default is false. If true,
110 if the HTTP status is 301, 302 or 303 the client will follow
111 the location.
112 :param max_follow_redirect: max number of redirection. If max is reached
113 the RedirectLimit exception is raised.
114 :param key_file: the key fle to use with ssl
115 :param cert_file: the cert file to use with ssl
116 :param pool_instance: a pool instance inherited from
117 `restkit.pool.PoolInterface`
118 :param socket: eventually you can pass your own socket object to
119 the client.
120 """
121 self.socket = socket
122 self.timeout = timeout
123 self.headers = []
124 self.req_headers = []
125 self.ua = USER_AGENT
126 self.url = None
127
128 self.follow_redirect = follow_redirect
129 self.nb_redirections = max_follow_redirect
130 self.force_follow_redirect = force_follow_redirect
131 self.method = 'GET'
132 self.body = None
133 self.response_body = StringIO.StringIO()
134 self.final_url = None
135
136
137 self.filters = filters or []
138 self.request_filters = []
139 self.response_filters = []
140
141 for f in self.filters:
142 self._add_filter(f)
143
144 if not pool_instance:
145 self.should_close = True
146 self.connections = None
147 else:
148 self.connections = pool_instance
149 self.should_close = False
150
151 if response_class is not None:
152 self.response_class = response_class
153
157
159 if hasattr(f, 'on_request'):
160 self.request_filters.append(f)
161
162 if hasattr(f, 'on_response'):
163 self.response_filters.append(f)
164
166 for i, f1 in enumerate(self.filters):
167 if f == f1: del self.filters[i]
168
169 if hasattr(f, 'on_request'):
170 for i, f1 in enumerate(self.request_filters):
171 if f == f1: del self.request_filters[i]
172
173 if hasattr(f, 'on_response'):
174 for i, f1 in enumerate(self.response_filters):
175 if f == f1: del self.response_filters[i]
176
178 """ initate a connection if needed or reuse a socket"""
179 addr = (self.host, self.port)
180 s = self.socket or None
181
182
183 if self.connections is not None:
184 s = self.connections.get(addr)
185
186 if not s:
187
188 if self.uri.scheme == "https":
189 s = sock.connect(addr, self.timeout, True,
190 self.key_file, self.cert_file)
191 else:
192 s = sock.connect(addr, self.timeout)
193
194 self.socket = s
195 return s
196
198 self.socket = None
199 if hasattr(self.connections,'clean'):
200 self.connections.clean((self.host, self.port))
201
203 if not self.socket: return
204 if self.parser.should_close:
205 sock.close(self.socket)
206 else:
207 self.connections.put((self.host, self.port), self.socket)
208 self.socket = None
209
211 """ parse url and get host/port"""
212 self.uri = urlparse.urlparse(url)
213 if self.uri.scheme not in ('http', 'https'):
214 raise InvalidUrl("None valid url")
215
216 host = self.uri.netloc
217 i = host.rfind(':')
218 j = host.rfind(']')
219 if i > j:
220 try:
221 port = int(host[i+1:])
222 except ValueError:
223 raise InvalidUrl("nonnumeric port: '%s'" % host[i+1:])
224 host = host[:i]
225 else:
226
227 if self.uri.scheme == "https":
228 port = 443
229 else:
230 port = 80
231
232 if host and host[0] == '[' and host[-1] == ']':
233 host = host[1:-1]
234
235 self.host = host
236 self.port = port
237
238 - def request(self, url, method='GET', body=None, headers=None):
239 """ make effective request
240
241 :param url: str, url string
242 :param method: str, by default GET. http verbs
243 :param body: the body, could be a string, an iterator or a file-like object
244 :param headers: dict or list of tupple, http headers
245 """
246 self.parser = Parser.parse_response(should_close=self.should_close)
247 self.url = url
248 self.final_url = url
249 self.parse_url(url)
250 self.method = method.upper()
251
252
253
254 headers = headers or []
255 if isinstance(headers, dict):
256 headers = list(headers.items())
257
258 ua = USER_AGENT
259 normalized_headers = []
260 content_len = None
261 accept_encoding = 'identity'
262 chunked = False
263
264
265 try:
266 host = self.uri.netloc.encode('ascii')
267 except UnicodeEncodeError:
268 host = self.uri.netloc.encode('idna')
269
270
271 for name, value in headers:
272 name = util.normalize_name(name)
273 if name == "User-Agenr":
274 ua = value
275 elif name == "Content-Length":
276 content_len = str(value)
277 elif name == "Accept-Encoding":
278 accept_encoding = 'identity'
279 elif name == "Host":
280 host = value
281 elif name == "Transfer-Encoding":
282 if value.lower() == "chunked":
283 chunked = True
284 normalized_headers.append((name, value))
285 else:
286 if not isinstance(value, basestring):
287 value = str(value)
288 normalized_headers.append((name, value))
289
290
291 if body and body is not None:
292 if not content_len:
293 if hasattr(body, 'fileno'):
294 try:
295 body.flush()
296 except IOError:
297 pass
298 content_len = str(os.fstat(body.fileno())[6])
299 elif hasattr(body, 'len'):
300 try:
301 content_len = str(body.len)
302 except AttributeError:
303 pass
304 elif isinstance(body, basestring):
305 body = util.to_bytestring(body)
306 content_len = len(body)
307
308 if content_len:
309 normalized_headers.append(("Content-Length", content_len))
310 elif not chunked:
311 raise RequestError("Can't determine content length and" +
312 "Transfer-Encoding header is not chunked")
313
314 if self.method in ('POST', 'PUT') and not body:
315 normalized_headers.append(("Content-Length", "0"))
316
317 self.body = body
318 self.headers = normalized_headers
319 self.ua = ua
320
321
322 for bf in self.request_filters:
323 bf.on_request(self)
324
325
326 if self.version == (1,1):
327 httpver = "HTTP/1.1"
328 else:
329 httpver = "HTTP/1.0"
330
331
332 path = self.uri.path or "/"
333 req_path = urlparse.urlunparse(('','', path, '',
334 self.uri.query, self.uri.fragment))
335
336
337 req_headers = []
338 req_headers.append("%s %s %s\r\n" % (method, req_path, httpver))
339 req_headers.append("Host: %s\r\n" % host)
340 req_headers.append("User-Agent: %s\r\n" % self.ua)
341 req_headers.append("Accept-Encoding: %s\r\n" % accept_encoding)
342 for name, value in self.headers:
343 req_headers.append("%s: %s\r\n" % (name, value))
344 req_headers.append("\r\n")
345 self.req_headers = req_headers
346
347
348 return self.do_send(req_headers, self.body, chunked)
349
350 - def do_send(self, req_headers, body=None, chunked=False):
351 tries = 2
352 while True:
353 try:
354 s = self.make_connection()
355
356 sock.sendlines(s, req_headers)
357
358 log.info('Start request: %s %s' % (self.method, self.url))
359 log.debug("Request headers: [%s]" % str(req_headers))
360
361 if body is not None:
362 if hasattr(body, 'read'):
363 if hasattr(body, 'seek'): body.seek(0)
364 sock.sendfile(s, body, chunked)
365 elif isinstance(body, basestring):
366 sock.sendfile(s, StringIO.StringIO(
367 util.to_bytestring(body)), chunked)
368 else:
369 sock.sendlines(s, body, chunked)
370
371 if chunked:
372 sock.send_chunk(s, "")
373
374 return self.start_response()
375 except socket.gaierror, e:
376 self.clean_connections()
377 raise
378 except socket.error, e:
379 if e[0] not in (errno.EAGAIN, errno.ECONNABORTED, errno.EPIPE,
380 errno.ECONNREFUSED) or tries <= 0:
381 self.clean_connections()
382 raise
383 except:
384 if tries <= 0:
385 raise
386 tries -= 1
387
388
390 """ follow redirections if needed"""
391 if self.nb_redirections <= 0:
392 raise RedirectLimit("Redirection limit is reached")
393
394 location = self.parser.headers_dict.get('Location')
395 if not location:
396 raise RequestError('no Location header')
397
398 new_uri = urlparse.urlparse(location)
399 if not new_uri.netloc:
400 absolute_uri = "%s://%s" % (self.uri.scheme, self.uri.netloc)
401 location = urlparse.urljoin(absolute_uri, location)
402
403 log.debug("Redirect to %s" % location)
404
405 self.final_url = location
406 self.response_body.read()
407 self.nb_redirections -= 1
408 self.maybe_close()
409 return self.request(location, self.method, self.body,
410 self.headers)
411
413 """
414 Get headers, set Body object and return HttpResponse
415 """
416
417 headers = []
418 buf = sock.recv(self.socket, sock.CHUNK_SIZE)
419 i = self.parser.filter_headers(headers, buf)
420 if i == -1 and buf:
421 while True:
422 data = sock.recv(self.socket, sock.CHUNK_SIZE)
423 if not data: break
424 buf += data
425 i = self.parser.filter_headers(headers, buf)
426 if i != -1:
427 break
428
429 log.debug("Start response: %s" % str(self.parser.status_line))
430 log.debug("Response headers: [%s]" % str(headers))
431
432 if (not self.parser.content_len and not self.parser.is_chunked):
433 response_body = StringIO.StringIO("".join(buf[i:]))
434 if self.parser.should_close:
435
436
437 log.debug("No content len an not chunked transfer, get body")
438 while True:
439 try:
440 chunk = sock.recv(self.socket, sock.CHUNK_SIZE)
441 except socket.error:
442 break
443 if not chunk: break
444 response_body.write("".join(chunk))
445 self.maybe_close()
446
447 response_body.seek(0)
448 self.response_body = response_body
449 elif self.method == "HEAD":
450 self.response_body = StringIO.StringIO()
451 self.maybe_close()
452 else:
453 self.response_body = tee.TeeInput(self.socket, self.parser,
454 buf[i:], maybe_close=self.maybe_close)
455
456
457 for af in self.response_filters:
458 af.on_response(self)
459
460 if self.follow_redirect:
461 if self.parser.status_int in (301, 302, 307):
462 if self.method in ('GET', 'HEAD') or \
463 self.force_follow_redirect:
464 if self.method not in ('GET', 'HEAD') and \
465 hasattr(self.body, 'seek'):
466 self.body.seek(0)
467 return self.do_redirect()
468 elif self.parser.status_int == 303 and self.method in ('GET',
469 'HEAD'):
470
471
472 return self.do_redirect()
473
474
475 self.final_url = self.parser.headers_dict.get('Location',
476 self.final_url)
477 log.debug("Return response: %s" % self.final_url)
478 return self.response_class(self)
479