1
2
3
4
5
6 import ctypes
7 import errno
8 import gzip
9 import logging
10 import os
11 import socket
12 import StringIO
13 import urlparse
14
15 from restkit import __version__
16 from restkit.errors import RequestError, InvalidUrl, RedirectLimit
17 from restkit.parser import Parser
18 from restkit import sock
19 from restkit import tee
20 from restkit import util
21
22 MAX_FOLLOW_REDIRECTS = 5
23
24 USER_AGENT = "restkit/%s" % __version__
25
26 log = logging.getLogger(__name__)
29 """ Http Response object returned by HttpConnction"""
30
31 charset = "utf8"
32 unicode_errors = 'strict'
33
35 self.http_client = http_client
36 self.status = self.http_client.parser.status
37 self.status_int = self.http_client.parser.status_int
38 self.version = self.http_client.parser.version
39 self.headerslist = self.http_client.parser.headers
40 self.final_url = self.http_client.final_url
41
42 headers = {}
43 for key, value in self.http_client.parser.headers:
44 headers[key.lower()] = value
45 self.headers = headers
46
47 encoding = headers.get('content-encoding', None)
48 if encoding in ('gzip', 'deflate'):
49 self._body = gzip.GzipFile(fileobj=self.http_client.response_body)
50 else:
51 self._body = self.http_client.response_body
52 self._body_eof = False
53
55 try:
56 return getattr(self, key)
57 except AttributeError:
58 pass
59 return self.headers[key]
60
62 return (key in self.headers)
63
65 for item in list(self.headers.items()):
66 yield item
67
68 @property
70 """ body in bytestring """
71 if self._body_eof:
72 self._body.seek(0)
73 self._body_eof = True
74 ret = self._body.read()
75 self._body.seek(0)
76 return ret
77
78 @property
79 - def body_file(self):
80 """ return body as a file like object"""
81 return self._body
82
83 @property
84 - def unicode_body(self):
85 """ like body but converted to unicode"""
86 if not self.charset:
87 raise AttributeError(
88 "You cannot access HttpResponse.unicode_body unless charset is set")
89 return self.body.decode(self.charset, self.unicode_errors)
90
92 """ Http Connection object. """
93
94 version = (1, 1)
95 response_class = HttpResponse
96
97
98 - def __init__(self, timeout=sock._GLOBAL_DEFAULT_TIMEOUT,
99 filters=None, follow_redirect=False, force_follow_redirect=False,
100 max_follow_redirect=MAX_FOLLOW_REDIRECTS, key_file=None,
101 cert_file=None, pool_instance=None, socket=None,
102 response_class=None):
103
104 """ HttpConnection constructor
105
106 :param timeout: socket timeout
107 :param filters: list, list of http filters. see the doc of http filters
108 for more info
109 :param follow_redirect: boolean, by default is false. If true,
110 if the HTTP status is 301, 302 or 303 the client will follow
111 the location.
112 :param max_follow_redirect: max number of redirection. If max is reached
113 the RedirectLimit exception is raised.
114 :param key_file: the key fle to use with ssl
115 :param cert_file: the cert file to use with ssl
116 :param pool_instance: a pool instance inherited from
117 `restkit.pool.PoolInterface`
118 :param socket: eventually you can pass your own socket object to
119 the client.
120 """
121 self.socket = socket
122 self.timeout = timeout
123 self.headers = []
124 self.req_headers = []
125 self.ua = USER_AGENT
126 self.url = None
127
128 self.follow_redirect = follow_redirect
129 self.nb_redirections = max_follow_redirect
130 self.force_follow_redirect = force_follow_redirect
131 self.method = 'GET'
132 self.body = None
133 self.response_body = StringIO.StringIO()
134 self.final_url = None
135
136
137 self.filters = filters or []
138 self.request_filters = []
139 self.response_filters = []
140
141 for f in self.filters:
142 self._add_filter(f)
143
144 if not pool_instance:
145 self.should_close = True
146 self.connections = None
147 else:
148 self.connections = pool_instance
149 self.should_close = False
150
151 if response_class is not None:
152 self.response_class = response_class
153
157
159 if hasattr(f, 'on_request'):
160 self.request_filters.append(f)
161
162 if hasattr(f, 'on_response'):
163 self.response_filters.append(f)
164
166 for i, f1 in enumerate(self.filters):
167 if f == f1: del self.filters[i]
168
169 if hasattr(f, 'on_request'):
170 for i, f1 in enumerate(self.request_filters):
171 if f == f1: del self.request_filters[i]
172
173 if hasattr(f, 'on_response'):
174 for i, f1 in enumerate(self.response_filters):
175 if f == f1: del self.response_filters[i]
176
178 """ initate a connection if needed or reuse a socket"""
179 addr = (self.host, self.port)
180 s = self.socket or None
181
182
183 if self.connections is not None:
184 s = self.connections.get(addr)
185
186 if not s:
187
188 if self.uri.scheme == "https":
189 s = sock.connect(addr, self.timeout, True,
190 self.key_file, self.cert_file)
191 else:
192 s = sock.connect(addr, self.timeout)
193
194 self.socket = s
195 return s
196
198 self.socket = None
199 if hasattr(self.connections,'clean'):
200 self.connections.clean((self.host, self.port))
201
203 if not self.socket: return
204 if not hasattr(self.connections,'clear') or self.parser.should_close:
205 sock.close(self.socket)
206 else:
207 self.connections.put((self.host, self.port), self.socket)
208 self.socket = None
209
211 """ parse url and get host/port"""
212 self.uri = urlparse.urlparse(url)
213
214 host = self.uri.netloc
215 i = host.rfind(':')
216 j = host.rfind(']')
217 if i > j:
218 try:
219 port = int(host[i+1:])
220 except ValueError:
221 raise InvalidUrl("nonnumeric port: '%s'" % host[i+1:])
222 host = host[:i]
223 else:
224
225 if self.uri.scheme == "https":
226 port = 443
227 else:
228 port = 80
229
230 if host and host[0] == '[' and host[-1] == ']':
231 host = host[1:-1]
232
233 self.host = host
234 self.port = port
235
236 - def request(self, url, method='GET', body=None, headers=None):
237 """ make effective request
238
239 :param url: str, url string
240 :param method: str, by default GET. http verbs
241 :param body: the body, could be a string, an iterator or a file-like object
242 :param headers: dict or list of tupple, http headers
243 """
244 self.parser = Parser.parse_response(should_close=self.should_close)
245 self.url = url
246 self.final_url = url
247 self.parse_url(url)
248 self.method = method.upper()
249
250
251
252 headers = headers or []
253 if isinstance(headers, dict):
254 headers = list(headers.items())
255
256 ua = USER_AGENT
257 normalized_headers = []
258 content_len = None
259 accept_encoding = 'identity'
260 chunked = False
261
262
263 try:
264 host = self.uri.netloc.encode('ascii')
265 except UnicodeEncodeError:
266 host = self.uri.netloc.encode('idna')
267
268
269 for name, value in headers:
270 name = util.normalize_name(name)
271 if name == "User-Agenr":
272 ua = value
273 elif name == "Content-Length":
274 content_len = str(value)
275 elif name == "Accept-Encoding":
276 accept_encoding = 'identity'
277 elif name == "Host":
278 host = value
279 elif name == "Transfer-Encoding":
280 if value.lower() == "chunked":
281 chunked = True
282 normalized_headers.append((name, value))
283 else:
284 if not isinstance(value, basestring):
285 value = str(value)
286 normalized_headers.append((name, value))
287
288
289 if body and body is not None:
290 if not content_len:
291 if hasattr(body, 'fileno'):
292 try:
293 body.flush()
294 except IOError:
295 pass
296 content_len = str(os.fstat(body.fileno())[6])
297 elif hasattr(body, 'len'):
298 try:
299 content_len = str(body.len)
300 except AttributeError:
301 pass
302 elif isinstance(body, basestring):
303 body = util.to_bytestring(body)
304 content_len = len(body)
305
306 if content_len:
307 normalized_headers.append(("Content-Length", content_len))
308 elif not chunked:
309 raise RequestError("Can't determine content length and" +
310 "Transfer-Encoding header is not chunked")
311
312 if self.method in ('POST', 'PUT') and not body:
313 normalized_headers.append(("Content-Length", "0"))
314
315 self.body = body
316 self.headers = normalized_headers
317 self.ua = ua
318
319
320 for bf in self.request_filters:
321 bf.on_request(self)
322
323
324 if self.version == (1,1):
325 httpver = "HTTP/1.1"
326 else:
327 httpver = "HTTP/1.0"
328
329
330 path = self.uri.path or "/"
331 req_path = urlparse.urlunparse(('','', path, '',
332 self.uri.query, self.uri.fragment))
333
334
335 req_headers = []
336 req_headers.append("%s %s %s\r\n" % (method, req_path, httpver))
337 req_headers.append("Host: %s\r\n" % host)
338 req_headers.append("User-Agent: %s\r\n" % self.ua)
339 req_headers.append("Accept-Encoding: %s\r\n" % accept_encoding)
340 for name, value in self.headers:
341 req_headers.append("%s: %s\r\n" % (name, value))
342 req_headers.append("\r\n")
343 self.req_headers = req_headers
344
345
346 return self.do_send(req_headers, self.body, chunked)
347
348 - def do_send(self, req_headers, body=None, chunked=False):
349 for i in range(2):
350 try:
351 s = self.make_connection()
352
353 sock.sendlines(s, req_headers)
354
355 log.info('%s %s\n\n' % (self.method, self.url))
356 log.debug("Headers: [%s]" % "".join(req_headers))
357
358 if body is not None:
359 if hasattr(body, 'read'):
360 sock.sendfile(s, body, chunked)
361 elif isinstance(body, basestring):
362 sock.sendfile(s, StringIO.StringIO(
363 util.to_bytestring(body)), chunked)
364 else:
365 sock.sendlines(s, body, chunked)
366
367 if chunked:
368 sock.send_chunk(s, "")
369
370 return self.start_response()
371 except socket.gaierror, e:
372 self.clean_connections()
373 raise
374 except socket.error, e:
375 if e[0] not in (errno.EAGAIN, errno.ECONNABORTED, errno.EPIPE,
376 errno.ECONNREFUSED):
377 self.clean_connections()
378 raise
379
381 """ follow redirections if needed"""
382 if self.nb_redirections <= 0:
383 raise RedirectLimit("Redirection limit is reached")
384
385 location = self.parser.headers_dict.get('Location')
386 if not location:
387 raise RequestError('no Location header')
388
389 new_uri = urlparse.urlparse(location)
390 if not new_uri.netloc:
391 absolute_uri = "%s://%s" % (self.uri.scheme, self.uri.netloc)
392 location = urlparse.urljoin(absolute_uri, location)
393
394 self.final_url = location
395 self.response_body.read()
396 self.nb_redirections -= 1
397 self.maybe_close()
398 return self.request(location, self.method, self.body,
399 self.headers)
400
402 """
403 Get headers, set Body object and return HttpResponse
404 """
405
406 headers = []
407 buf = sock.recv(self.socket, sock.CHUNK_SIZE)
408 i = self.parser.filter_headers(headers, buf)
409 if i == -1 and buf:
410 while True:
411 data = sock.recv(self.socket, sock.CHUNK_SIZE)
412 if not data: break
413 buf += data
414 i = self.parser.filter_headers(headers, buf)
415 if i != -1:
416 break
417 log.debug("Start response")
418 log.debug("Response headers: [%s]" % str(headers))
419
420 if (not self.parser.content_len and not self.parser.is_chunked):
421 response_body = StringIO.StringIO("".join(buf[i:]))
422
423 if self.parser.should_close:
424
425
426 while True:
427 try:
428 chunk = sock.recv(self.socket, sock.CHUNK_SIZE)
429 except socket.error:
430 break
431 response_body.write("".join(chunk))
432 if not chunk: break
433 self.maybe_close()
434
435 response_body.seek(0)
436 self.response_body = response_body
437 elif self.method == "HEAD":
438 self.response_body = StringIO.StringIO()
439 else:
440 self.response_body = tee.TeeInput(self.socket, self.parser,
441 buf[i:], maybe_close=self.maybe_close)
442
443
444 for af in self.response_filters:
445 af.on_response(self)
446
447 if self.follow_redirect:
448 if self.parser.status_int in (301, 302, 307):
449 if self.method in ('GET', 'HEAD') or \
450 self.force_follow_redirect:
451 if self.method not in ('GET', 'HEAD') and \
452 hasattr(self.body, 'seek'):
453 self.body.seek(0)
454 return self.do_redirect()
455 elif self.parser.status_int == 303 and self.method in ('GET',
456 'HEAD'):
457
458
459 return self.do_redirect()
460
461 self.final_url = self.parser.headers_dict.get('Location',
462 self.final_url)
463 return self.response_class(self)
464