1
2
3
4
5
6 import copy
7 import cgi
8 import errno
9 import logging
10 import mimetypes
11 import os
12 import socket
13 import time
14 try:
15 from cStringIO import StringIO
16 except ImportError:
17 from StringIO import StringIO
18 import types
19 import urlparse
20 import uuid
21
22 from restkit import __version__
23 from restkit.client.response import HttpResponse
24 from restkit.conn import get_default_manager, \
25 DEFAULT_CONN_NB_CONNECTIONS, DEFAULT_CONN_TIMEOUT
26 from restkit.errors import RequestError, InvalidUrl, RedirectLimit, \
27 RequestTimeout
28 from restkit.filters import Filters
29 from restkit.forms import multipart_form_encode, form_encode
30 from restkit.util import sock
31 from restkit import util
32 from restkit import http
33
34 MAX_FOLLOW_REDIRECTS = 5
35
36 USER_AGENT = "restkit/%s" % __version__
37
38 log = logging.getLogger(__name__)
39
41 """ Http Connection object. """
42
43 version = (1, 1)
44 response_class = HttpResponse
45
46 - def __init__(self,
47 timeout=DEFAULT_CONN_TIMEOUT,
48 filters=None,
49 follow_redirect=False,
50 force_follow_redirect=False,
51 max_follow_redirect=MAX_FOLLOW_REDIRECTS,
52 decompress=True,
53 pool_instance=None,
54 response_class=None,
55 conn_manager=None,
56 nb_connections=DEFAULT_CONN_NB_CONNECTIONS,
57 **ssl_args):
58
59 """ HttpRequest constructor
60
61 :param timeout: socket timeout
62 :param filters: list, list of http filters. see the doc of http filters
63 for more info
64 :param follow_redirect: boolean, by default is false. If true,
65 if the HTTP status is 301, 302 or 303 the client will follow
66 the location.
67 :param max_follow_redirect: max number of redirection. If max is reached
68 the RedirectLimit exception is raised.
69 :param conn_manager: a connectoin manager instance inherited from
70 `restkit.conn.base.ConnectioManager`
71 :param ssl_args: ssl arguments. See http://docs.python.org/library/ssl.html
72 for more information.
73 """
74 self._conn = None
75 self.timeout = timeout
76 self.headers = []
77 self.req_headers = []
78 self.ua = USER_AGENT
79 self.url = None
80
81 self.follow_redirect = follow_redirect
82 self.nb_redirections = max_follow_redirect
83 self.force_follow_redirect = force_follow_redirect
84 self.decompress = decompress
85 self.method = 'GET'
86 self.body = None
87 self.response_body = StringIO()
88 self.final_url = None
89
90
91 self.filters = Filters(filters)
92 self.ssl_args = ssl_args or {}
93
94 if pool_instance is not None:
95 self.conn_manager = pool_instance
96 elif conn_manager is not None:
97 self.conn_manager = conn_manager
98 else:
99 self.conn_manager = get_default_manager(
100 timeout=timeout,
101 nb_connections=nb_connections
102 )
103
104 if response_class is not None:
105 self.response_class = response_class
106
108 """ parse url and get host/port"""
109 self.uri = urlparse.urlparse(url)
110 if self.uri.scheme not in ('http', 'https'):
111 raise InvalidUrl("None valid url")
112
113 host, port = util.parse_netloc(self.uri)
114 self.host = host
115 self.port = port
116
117 - def set_body(self, body, headers, chunked=False):
118 """ set HTTP body and manage form if needed """
119 content_type = headers.get('CONTENT-TYPE')
120 content_length = headers.get('CONTENT-LENGTH')
121 if not body:
122 if content_type is not None:
123 self.headers.append(('Content-Type', content_type))
124 if self.method in ('POST', 'PUT'):
125 self.headers.append(("Content-Length", "0"))
126 return
127
128
129 if isinstance(body, dict):
130 if content_type is not None and \
131 content_type.startswith("multipart/form-data"):
132 type_, opts = cgi.parse_header(content_type)
133 boundary = opts.get('boundary', uuid.uuid4().hex)
134 body, self.headers = multipart_form_encode(body,
135 self.headers, boundary)
136 else:
137 content_type = "application/x-www-form-urlencoded; charset=utf-8"
138 body = form_encode(body)
139 elif hasattr(body, "boundary"):
140 content_type = "multipart/form-data; boundary=%s" % body.boundary
141 content_length = body.get_size()
142
143 if not content_type:
144 content_type = 'application/octet-stream'
145 if hasattr(body, 'name'):
146 content_type = mimetypes.guess_type(body.name)[0]
147
148 if not content_length:
149 if hasattr(body, 'fileno'):
150 try:
151 body.flush()
152 except IOError:
153 pass
154 content_length = str(os.fstat(body.fileno())[6])
155 elif hasattr(body, 'getvalue'):
156 try:
157 content_length = str(len(body.getvalue()))
158 except AttributeError:
159 pass
160 elif isinstance(body, types.StringTypes):
161 body = util.to_bytestring(body)
162 content_length = len(body)
163
164 if content_length:
165 self.headers.append(("Content-Length", content_length))
166 if content_type is not None:
167 self.headers.append(('Content-Type', content_type))
168
169 elif not chunked:
170 raise RequestError("Can't determine content length and " +
171 "Transfer-Encoding header is not chunked")
172
173 self.body = body
174
175
176 - def request(self, url, method='GET', body=None, headers=None):
177 """ make effective request
178
179 :param url: str, url string
180 :param method: str, by default GET. http verbs
181 :param body: the body, could be a string, an iterator or a file-like object
182 :param headers: dict or list of tupple, http headers
183 """
184 self._conn = None
185 self.url = url
186 self.final_url = url
187 self.parse_url(url)
188 self.method = method.upper()
189
190 self.init_headers = copy.copy(headers or [])
191 self.headers = []
192
193
194 headers = headers or []
195 if isinstance(headers, dict):
196 headers = headers.items()
197
198 chunked = False
199
200
201 search_headers = ('USER-AGENT', 'CONTENT-TYPE',
202 'CONTENT-LENGTH', 'ACCEPT-ENCODING',
203 'TRANSFER-ENCODING', 'CONNECTION', 'HOST')
204 found_headers = {}
205 new_headers = copy.copy(headers)
206 for (name, value) in headers:
207 uname = name.upper()
208 if uname in search_headers:
209 if uname == 'TRANSFER-ENCODING':
210 if value.lower() == "chunked":
211 chunked = True
212 else:
213 found_headers[uname] = value
214 new_headers.remove((name, value))
215
216 self.headers = new_headers
217 self.chunked = chunked
218
219
220 self.set_body(body, found_headers, chunked=chunked)
221
222 self.found_headers = found_headers
223
224
225 return self.do_send()
226
228
229 if self.version == (1,1):
230 httpver = "HTTP/1.1"
231 else:
232 httpver = "HTTP/1.0"
233
234
235 path = self.uri.path or "/"
236 req_path = urlparse.urlunparse(('','', path, self.uri.params,
237 self.uri.query, self.uri.fragment))
238
239
240 ua = self.found_headers.get('USER-AGENT')
241 accept_encoding = self.found_headers.get('ACCEPT-ENCODING')
242 connection = self.found_headers.get('CONNECTION')
243
244
245 try:
246 host = self.uri.netloc.encode('ascii')
247 except UnicodeEncodeError:
248 host = self.uri.netloc.encode('idna')
249 host = self.found_headers.get('HOST') or host
250
251
252 req_headers = [
253 "%s %s %s\r\n" % (self.method, req_path, httpver),
254 "Host: %s\r\n" % host,
255 "User-Agent: %s\r\n" % ua or USER_AGENT,
256 "Accept-Encoding: %s\r\n" % accept_encoding or 'identity'
257 ]
258
259 if connection is not None:
260 req_headers.append("Connection: %s\r\n" % connection)
261
262 req_headers.extend(["%s: %s\r\n" % (k, v) for k, v in self.headers])
263 req_headers.append('\r\n')
264 return req_headers
265
267 self._pool.shutdown()
268 if not self._conn:
269 return
270 self._conn.close()
271 self._conn = None
272
274 addr = (self.host, self.port)
275 is_ssl = (self.uri.scheme == "https")
276 route = (addr, is_ssl, self.filters, self.ssl_args)
277 self._pool = self.conn_manager.get_pool(route)
278 tries = 2
279 while True:
280 try:
281 if not self._conn:
282
283 self._conn = self._pool.request()
284
285 s = self._conn.socket()
286 self.headers.extend(self._conn.headers)
287
288
289 self.filters.apply("on_request", self, tries)
290
291
292 self.req_headers = req_headers = self._req_headers()
293
294
295 log.info('Start request: %s %s', self.method, self.url)
296 log.debug("Request headers: [%s]", req_headers)
297
298 s.sendall("".join(req_headers))
299
300 if self.body is not None:
301 if hasattr(self.body, 'read'):
302 if hasattr(self.body, 'seek'): self.body.seek(0)
303 sock.sendfile(s, self.body, self.chunked)
304 elif isinstance(self.body, types.StringTypes):
305 sock.send(s, self.body, self.chunked)
306 else:
307 sock.sendlines(s, self.body, self.chunked)
308
309 if self.chunked:
310 sock.send_chunk(s, "")
311
312 return self.start_response()
313 except socket.gaierror, e:
314 self.shutdown_connection()
315 raise RequestError(str(e))
316 except socket.timeout, e:
317 self.shutdown_connection()
318 raise RequestTimeout(str(e))
319 except socket.error, e:
320 if e[0] not in (errno.EAGAIN, errno.ECONNABORTED,
321 errno.EPIPE, errno.ECONNREFUSED,
322 errno.ECONNRESET) or tries <= 0:
323 self.shutdown_connection()
324 raise RequestError(str(e))
325 if e[0] in (errno.EPIPE, errno.ECONNRESET):
326 self.shutdown_connection()
327 except (KeyboardInterrupt, SystemExit):
328 break
329 except:
330 if tries < 0:
331 raise
332
333 self.shutdown_connection()
334 time.sleep(0.2)
335 tries -= 1
336
338 """ follow redirections if needed"""
339 if self.nb_redirections <= 0:
340 raise RedirectLimit("Redirection limit is reached")
341
342 if not location:
343 raise RequestError('no Location header')
344
345 new_uri = urlparse.urlparse(location)
346 if not new_uri.netloc:
347 absolute_uri = "%s://%s" % (self.uri.scheme, self.uri.netloc)
348 location = urlparse.urljoin(absolute_uri, location)
349
350 log.debug("Redirect to %s" % location)
351
352 self.final_url = location
353 response.body.read()
354 self.nb_redirections -= 1
355 if response.should_close:
356 self._conn.close()
357 self._conn = None
358 return self.request(location, self.method, self.body, self.init_headers)
359
361 """
362 Get headers, set Body object and return HttpResponse
363 """
364
365 while True:
366 parser = http.ResponseParser(self._conn,
367 release_source=self._pool.release,
368 decompress=self.decompress)
369 resp = parser.next()
370 if resp.status_int != 100:
371 break
372
373 log.debug("Start response: %s", resp.status)
374 log.debug("Response headers: [%s]", resp.headers)
375
376 location = None
377 for hdr_name, hdr_value in resp.headers:
378 if hdr_name.lower() == "location":
379 location = hdr_value
380 break
381
382 if self.follow_redirect:
383 if resp.status_int in (301, 302, 307):
384 if self.method in ('GET', 'HEAD') or \
385 self.force_follow_redirect:
386 if self.method not in ('GET', 'HEAD') and \
387 hasattr(self.body, 'seek'):
388 self.body.seek(0)
389 return self.do_redirect(resp, location)
390 elif resp.status_int == 303 and self.method in ('GET',
391 'HEAD'):
392
393
394 return self.do_redirect(resp, location)
395
396
397
398 self.filters.apply("on_response", self)
399
400 self.final_url = location or self.final_url
401 log.debug("Return response: %s" % self.final_url)
402 if self.method == "HEAD":
403 resp.body = StringIO()
404 self._conn.close()
405 self._conn = None
406 return self.response_class(resp, self.final_url)
407