1
2
3
4
5
6 import copy
7 import cgi
8 import errno
9 import logging
10 import mimetypes
11 import os
12 import socket
13 import time
14 try:
15 from cStringIO import StringIO
16 except ImportError:
17 from StringIO import StringIO
18 import types
19 import urlparse
20 import uuid
21
22 from restkit import __version__
23 from restkit.client.response import HttpResponse
24 from restkit.errors import RequestError, InvalidUrl, RedirectLimit
25 from restkit.filters import Filters
26 from restkit.forms import multipart_form_encode, form_encode
27 from restkit.util import sock
28 from restkit import util
29 from restkit import http
30
31 MAX_FOLLOW_REDIRECTS = 5
32
33 USER_AGENT = "restkit/%s" % __version__
34
35 log = logging.getLogger(__name__)
36
38 """ Http Connection object. """
39
40 version = (1, 1)
41 response_class = HttpResponse
42
48
49 """ HttpRequest constructor
50
51 :param timeout: socket timeout
52 :param filters: list, list of http filters. see the doc of http filters
53 for more info
54 :param follow_redirect: boolean, by default is false. If true,
55 if the HTTP status is 301, 302 or 303 the client will follow
56 the location.
57 :param max_follow_redirect: max number of redirection. If max is reached
58 the RedirectLimit exception is raised.
59 :param pool_instance: a pool instance inherited from
60 `restkit.pool.PoolInterface`
61 :param ssl_args: ssl arguments. See http://docs.python.org/library/ssl.html
62 for more information.
63 """
64 self._sock = None
65 self.timeout = timeout
66 self.headers = []
67 self.req_headers = []
68 self.ua = USER_AGENT
69 self.url = None
70
71 self.follow_redirect = follow_redirect
72 self.nb_redirections = max_follow_redirect
73 self.force_follow_redirect = force_follow_redirect
74 self.method = 'GET'
75 self.body = None
76 self.response_body = StringIO()
77 self.final_url = None
78
79
80 self.filters = Filters(filters)
81 self.ssl_args = ssl_args or {}
82
83 if not pool_instance:
84 self.should_close = True
85 self.pool = None
86 else:
87 self.pool = pool_instance
88 self.should_close = False
89
90 if response_class is not None:
91 self.response_class = response_class
92
94 """ initate a connection if needed or reuse a socket"""
95
96
97 self.filters.apply("on_connect", self)
98 if self._sock is not None:
99 return self._sock
100
101 addr = (self.host, self.port)
102 s = None
103
104 if self.pool is not None:
105 s = self.pool.get(addr)
106
107 if not s:
108
109 if self.uri.scheme == "https":
110 s = sock.connect(addr, True, self.timeout, **self.ssl_args)
111 else:
112 s = sock.connect(addr, False, self.timeout)
113 return s
114
120
127
129 """ parse url and get host/port"""
130 self.uri = urlparse.urlparse(url)
131 if self.uri.scheme not in ('http', 'https'):
132 raise InvalidUrl("None valid url")
133
134 host, port = util.parse_netloc(self.uri)
135 self.host = host
136 self.port = port
137
138 - def set_body(self, body, headers, chunked=False):
139 """ set HTTP body and manage form if needed """
140 content_type = headers.get('CONTENT-TYPE')
141 content_length = headers.get('CONTENT-LENGTH')
142 if not body:
143 if content_type is not None:
144 self.headers.append(('Content-Type', content_type))
145 if self.method in ('POST', 'PUT'):
146 self.headers.append(("Content-Length", "0"))
147 return
148
149
150 if isinstance(body, dict):
151 if content_type is not None and \
152 content_type.startswith("multipart/form-data"):
153 type_, opts = cgi.parse_header(content_type)
154 boundary = opts.get('boundary', uuid.uuid4().hex)
155 body, self.headers = multipart_form_encode(body,
156 self.headers, boundary)
157 else:
158 content_type = "application/x-www-form-urlencoded; charset=utf-8"
159 body = form_encode(body)
160 elif hasattr(body, "boundary"):
161 content_type = "multipart/form-data; boundary=%s" % body.boundary
162 content_length = body.get_size()
163
164 if not content_type:
165 content_type = 'application/octet-stream'
166 if hasattr(body, 'name'):
167 content_type = mimetypes.guess_type(body.name)[0]
168
169 if not content_length:
170 if hasattr(body, 'fileno'):
171 try:
172 body.flush()
173 except IOError:
174 pass
175 content_length = str(os.fstat(body.fileno())[6])
176 elif hasattr(body, 'getvalue'):
177 try:
178 content_length = str(len(body.getvalue()))
179 except AttributeError:
180 pass
181 elif isinstance(body, types.StringTypes):
182 body = util.to_bytestring(body)
183 content_length = len(body)
184
185 if content_length:
186 self.headers.append(("Content-Length", content_length))
187 if content_type is not None:
188 self.headers.append(('Content-Type', content_type))
189
190 elif not chunked:
191 raise RequestError("Can't determine content length and " +
192 "Transfer-Encoding header is not chunked")
193
194 self.body = body
195
196
197 - def request(self, url, method='GET', body=None, headers=None):
198 """ make effective request
199
200 :param url: str, url string
201 :param method: str, by default GET. http verbs
202 :param body: the body, could be a string, an iterator or a file-like object
203 :param headers: dict or list of tupple, http headers
204 """
205 self._sock = None
206 self.url = url
207 self.final_url = url
208 self.parse_url(url)
209 self.method = method.upper()
210
211 self.init_headers = copy.copy(headers or [])
212 self.headers = []
213
214
215 headers = headers or []
216 if isinstance(headers, dict):
217 headers = headers.items()
218
219 chunked = False
220
221
222 search_headers = ('USER-AGENT', 'CONTENT-TYPE',
223 'CONTENT-LENGTH', 'ACCEPT-ENCODING',
224 'TRANSFER-ENCODING', 'CONNECTION', 'HOST')
225 found_headers = {}
226 new_headers = copy.copy(headers)
227 for (name, value) in headers:
228 uname = name.upper()
229 if uname in search_headers:
230 if uname == 'TRANSFER-ENCODING':
231 if value.lower() == "chunked":
232 chunked = True
233 else:
234 found_headers[uname] = value
235 new_headers.remove((name, value))
236
237 self.headers = new_headers
238 self.chunked = chunked
239
240
241 self.set_body(body, found_headers, chunked=chunked)
242
243
244 if found_headers.get('CONNECTION') == "close":
245 self.should_close = True
246 elif self.pool is None:
247 found_headers['CONNECTION'] = "close"
248
249 self.found_headers = found_headers
250
251
252 return self.do_send()
253
255
256 if self.version == (1,1):
257 httpver = "HTTP/1.1"
258 else:
259 httpver = "HTTP/1.0"
260
261
262 path = self.uri.path or "/"
263 req_path = urlparse.urlunparse(('','', path, '',
264 self.uri.query, self.uri.fragment))
265
266
267 ua = self.found_headers.get('USER-AGENT')
268 accept_encoding = self.found_headers.get('ACCEPT-ENCODING')
269 connection = self.found_headers.get('CONNECTION')
270
271
272 try:
273 host = self.uri.netloc.encode('ascii')
274 except UnicodeEncodeError:
275 host = self.uri.netloc.encode('idna')
276 host = self.found_headers.get('HOST') or host
277
278
279 req_headers = [
280 "%s %s %s\r\n" % (self.method, req_path, httpver),
281 "Host: %s\r\n" % host,
282 "User-Agent: %s\r\n" % ua or USER_AGENT,
283 "Accept-Encoding: %s\r\n" % accept_encoding or 'identity'
284 ]
285
286 if connection is not None:
287 req_headers.append("Connection: %s\r\n" % connection)
288
289 req_headers.extend(["%s: %s\r\n" % (k, v) for k, v in self.headers])
290 req_headers.append('\r\n')
291 return req_headers
292
294 tries = 2
295 while True:
296 try:
297
298 self._sock = self.make_connection()
299
300
301 self.filters.apply("on_request", self, tries)
302
303
304 self.req_headers = req_headers = self._req_headers()
305
306
307 log.info('Start request: %s %s', self.method, self.url)
308 log.debug("Request headers: [%s]", req_headers)
309
310 self._sock.sendall("".join(req_headers))
311
312 if self.body is not None:
313 if hasattr(self.body, 'read'):
314 if hasattr(self.body, 'seek'): self.body.seek(0)
315 sock.sendfile(self._sock, self.body, self.chunked)
316 elif isinstance(self.body, types.StringTypes):
317 sock.send(self._sock, self.body, self.chunked)
318 else:
319 sock.sendlines(self._sock, self.body, self.chunked)
320
321 if self.chunked:
322 sock.send_chunk(self._sock, "")
323
324 return self.start_response()
325 except socket.gaierror, e:
326 self.clean_connections()
327 raise
328 except socket.error, e:
329 if e[0] not in (errno.EAGAIN, errno.ECONNABORTED, errno.EPIPE,
330 errno.ECONNREFUSED, errno.ECONNRESET) or tries <= 0:
331 self.clean_connections()
332 raise
333 if e[0] in (errno.EPIPE, errno.ECONNRESET):
334 self.clean_connections()
335 except (KeyboardInterrupt, SystemExit):
336 break
337 except:
338 if tries <= 0:
339 raise
340
341 self.clean_connections()
342 time.sleep(0.2)
343 tries -= 1
344
345
347 """ follow redirections if needed"""
348 if self.nb_redirections <= 0:
349 raise RedirectLimit("Redirection limit is reached")
350
351 if not location:
352 raise RequestError('no Location header')
353
354 new_uri = urlparse.urlparse(location)
355 if not new_uri.netloc:
356 absolute_uri = "%s://%s" % (self.uri.scheme, self.uri.netloc)
357 location = urlparse.urljoin(absolute_uri, location)
358
359 log.debug("Redirect to %s" % location)
360
361 self.final_url = location
362 response.body.read()
363 self.nb_redirections -= 1
364
365 self.release_connection((self.host, self.port), self._sock)
366 return self.request(location, self.method, self.body, self.init_headers)
367
369 """
370 Get headers, set Body object and return HttpResponse
371 """
372
373 parser = http.ResponseParser(self._sock,
374 release_source = lambda:self.release_connection(
375 (self.host, self.port), self._sock))
376 resp = parser.next()
377
378 log.debug("Start response: %s", resp.status)
379 log.debug("Response headers: [%s]", resp.headers)
380
381 location = None
382 for hdr_name, hdr_value in resp.headers:
383 if hdr_name.lower() == "location":
384 location = hdr_value
385 break
386
387 if self.follow_redirect:
388 if resp.status_int in (301, 302, 307):
389 if self.method in ('GET', 'HEAD') or \
390 self.force_follow_redirect:
391 if self.method not in ('GET', 'HEAD') and \
392 hasattr(self.body, 'seek'):
393 self.body.seek(0)
394 return self.do_redirect(resp, location)
395 elif resp.status_int == 303 and self.method in ('GET',
396 'HEAD'):
397
398
399 return self.do_redirect(resp, location)
400
401
402
403 self.filters.apply("on_response", self)
404
405 self.final_url = location or self.final_url
406 log.debug("Return response: %s" % self.final_url)
407 if self.method == "HEAD":
408 resp.body = StringIO()
409
410 return self.response_class(resp, self.final_url)
411