1
2
3
4
5
6 import copy
7 import cgi
8 import errno
9 import logging
10 import mimetypes
11 import os
12 import socket
13 import time
14 try:
15 from cStringIO import StringIO
16 except ImportError:
17 from StringIO import StringIO
18 import types
19 import urlparse
20 import uuid
21
22 from restkit import __version__
23 from restkit.client.response import HttpResponse
24 from restkit.errors import RequestError, InvalidUrl, RedirectLimit
25 from restkit.filters import Filters
26 from restkit.forms import multipart_form_encode, form_encode
27 from restkit.util import sock
28 from restkit import util
29 from restkit import http
30
31 MAX_FOLLOW_REDIRECTS = 5
32
33 USER_AGENT = "restkit/%s" % __version__
34
35 log = logging.getLogger(__name__)
36
38 """ Http Connection object. """
39
40 version = (1, 1)
41 response_class = HttpResponse
42
49
50 """ HttpRequest constructor
51
52 :param timeout: socket timeout
53 :param filters: list, list of http filters. see the doc of http filters
54 for more info
55 :param follow_redirect: boolean, by default is false. If true,
56 if the HTTP status is 301, 302 or 303 the client will follow
57 the location.
58 :param max_follow_redirect: max number of redirection. If max is reached
59 the RedirectLimit exception is raised.
60 :param pool_instance: a pool instance inherited from
61 `restkit.pool.PoolInterface`
62 :param ssl_args: ssl arguments. See http://docs.python.org/library/ssl.html
63 for more information.
64 """
65 self._sock = None
66 self.timeout = timeout
67 self.headers = []
68 self.req_headers = []
69 self.ua = USER_AGENT
70 self.url = None
71
72 self.follow_redirect = follow_redirect
73 self.nb_redirections = max_follow_redirect
74 self.force_follow_redirect = force_follow_redirect
75 self.decompress = decompress
76 self.method = 'GET'
77 self.body = None
78 self.response_body = StringIO()
79 self.final_url = None
80
81
82 self.filters = Filters(filters)
83 self.ssl_args = ssl_args or {}
84
85 if not pool_instance:
86 self.should_close = True
87 self.pool = None
88 else:
89 self.pool = pool_instance
90 self.should_close = False
91
92 if response_class is not None:
93 self.response_class = response_class
94
96 """ initate a connection if needed or reuse a socket"""
97
98
99 self.filters.apply("on_connect", self)
100 if self._sock is not None:
101 return self._sock
102
103 addr = (self.host, self.port)
104 s = None
105
106 if self.pool is not None:
107 s = self.pool.get(addr)
108
109 if not s:
110
111 if self.uri.scheme == "https":
112 s = sock.connect(addr, True, self.timeout, **self.ssl_args)
113 else:
114 s = sock.connect(addr, False, self.timeout)
115 return s
116
122
129
131 """ parse url and get host/port"""
132 self.uri = urlparse.urlparse(url)
133 if self.uri.scheme not in ('http', 'https'):
134 raise InvalidUrl("None valid url")
135
136 host, port = util.parse_netloc(self.uri)
137 self.host = host
138 self.port = port
139
140 - def set_body(self, body, headers, chunked=False):
141 """ set HTTP body and manage form if needed """
142 content_type = headers.get('CONTENT-TYPE')
143 content_length = headers.get('CONTENT-LENGTH')
144 if not body:
145 if content_type is not None:
146 self.headers.append(('Content-Type', content_type))
147 if self.method in ('POST', 'PUT'):
148 self.headers.append(("Content-Length", "0"))
149 return
150
151
152 if isinstance(body, dict):
153 if content_type is not None and \
154 content_type.startswith("multipart/form-data"):
155 type_, opts = cgi.parse_header(content_type)
156 boundary = opts.get('boundary', uuid.uuid4().hex)
157 body, self.headers = multipart_form_encode(body,
158 self.headers, boundary)
159 else:
160 content_type = "application/x-www-form-urlencoded; charset=utf-8"
161 body = form_encode(body)
162 elif hasattr(body, "boundary"):
163 content_type = "multipart/form-data; boundary=%s" % body.boundary
164 content_length = body.get_size()
165
166 if not content_type:
167 content_type = 'application/octet-stream'
168 if hasattr(body, 'name'):
169 content_type = mimetypes.guess_type(body.name)[0]
170
171 if not content_length:
172 if hasattr(body, 'fileno'):
173 try:
174 body.flush()
175 except IOError:
176 pass
177 content_length = str(os.fstat(body.fileno())[6])
178 elif hasattr(body, 'getvalue'):
179 try:
180 content_length = str(len(body.getvalue()))
181 except AttributeError:
182 pass
183 elif isinstance(body, types.StringTypes):
184 body = util.to_bytestring(body)
185 content_length = len(body)
186
187 if content_length:
188 self.headers.append(("Content-Length", content_length))
189 if content_type is not None:
190 self.headers.append(('Content-Type', content_type))
191
192 elif not chunked:
193 raise RequestError("Can't determine content length and " +
194 "Transfer-Encoding header is not chunked")
195
196 self.body = body
197
198
199 - def request(self, url, method='GET', body=None, headers=None):
200 """ make effective request
201
202 :param url: str, url string
203 :param method: str, by default GET. http verbs
204 :param body: the body, could be a string, an iterator or a file-like object
205 :param headers: dict or list of tupple, http headers
206 """
207 self._sock = None
208 self.url = url
209 self.final_url = url
210 self.parse_url(url)
211 self.method = method.upper()
212
213 self.init_headers = copy.copy(headers or [])
214 self.headers = []
215
216
217 headers = headers or []
218 if isinstance(headers, dict):
219 headers = headers.items()
220
221 chunked = False
222
223
224 search_headers = ('USER-AGENT', 'CONTENT-TYPE',
225 'CONTENT-LENGTH', 'ACCEPT-ENCODING',
226 'TRANSFER-ENCODING', 'CONNECTION', 'HOST')
227 found_headers = {}
228 new_headers = copy.copy(headers)
229 for (name, value) in headers:
230 uname = name.upper()
231 if uname in search_headers:
232 if uname == 'TRANSFER-ENCODING':
233 if value.lower() == "chunked":
234 chunked = True
235 else:
236 found_headers[uname] = value
237 new_headers.remove((name, value))
238
239 self.headers = new_headers
240 self.chunked = chunked
241
242
243 self.set_body(body, found_headers, chunked=chunked)
244
245
246 if found_headers.get('CONNECTION') == "close":
247 self.should_close = True
248 elif self.pool is None:
249 found_headers['CONNECTION'] = "close"
250
251 self.found_headers = found_headers
252
253
254 return self.do_send()
255
257
258 if self.version == (1,1):
259 httpver = "HTTP/1.1"
260 else:
261 httpver = "HTTP/1.0"
262
263
264 path = self.uri.path or "/"
265 req_path = urlparse.urlunparse(('','', path, self.uri.params,
266 self.uri.query, self.uri.fragment))
267
268
269 ua = self.found_headers.get('USER-AGENT')
270 accept_encoding = self.found_headers.get('ACCEPT-ENCODING')
271 connection = self.found_headers.get('CONNECTION')
272
273
274 try:
275 host = self.uri.netloc.encode('ascii')
276 except UnicodeEncodeError:
277 host = self.uri.netloc.encode('idna')
278 host = self.found_headers.get('HOST') or host
279
280
281 req_headers = [
282 "%s %s %s\r\n" % (self.method, req_path, httpver),
283 "Host: %s\r\n" % host,
284 "User-Agent: %s\r\n" % ua or USER_AGENT,
285 "Accept-Encoding: %s\r\n" % accept_encoding or 'identity'
286 ]
287
288 if connection is not None:
289 req_headers.append("Connection: %s\r\n" % connection)
290
291 req_headers.extend(["%s: %s\r\n" % (k, v) for k, v in self.headers])
292 req_headers.append('\r\n')
293 return req_headers
294
296 tries = 2
297 while True:
298 try:
299
300 self._sock = self.make_connection()
301
302
303 self.filters.apply("on_request", self, tries)
304
305
306 self.req_headers = req_headers = self._req_headers()
307
308
309 log.info('Start request: %s %s', self.method, self.url)
310 log.debug("Request headers: [%s]", req_headers)
311
312 self._sock.sendall("".join(req_headers))
313
314 if self.body is not None:
315 if hasattr(self.body, 'read'):
316 if hasattr(self.body, 'seek'): self.body.seek(0)
317 sock.sendfile(self._sock, self.body, self.chunked)
318 elif isinstance(self.body, types.StringTypes):
319 sock.send(self._sock, self.body, self.chunked)
320 else:
321 sock.sendlines(self._sock, self.body, self.chunked)
322
323 if self.chunked:
324 sock.send_chunk(self._sock, "")
325
326 return self.start_response()
327 except socket.gaierror, e:
328 self.clean_connections()
329 raise
330 except socket.error, e:
331 if e[0] not in (errno.EAGAIN, errno.ECONNABORTED, errno.EPIPE,
332 errno.ECONNREFUSED, errno.ECONNRESET) or tries <= 0:
333 self.clean_connections()
334 raise
335 if e[0] in (errno.EPIPE, errno.ECONNRESET):
336 self.clean_connections()
337 except (KeyboardInterrupt, SystemExit):
338 break
339 except:
340 if tries <= 0:
341 raise
342
343 self.clean_connections()
344 time.sleep(0.2)
345 tries -= 1
346
347
349 """ follow redirections if needed"""
350 if self.nb_redirections <= 0:
351 raise RedirectLimit("Redirection limit is reached")
352
353 if not location:
354 raise RequestError('no Location header')
355
356 new_uri = urlparse.urlparse(location)
357 if not new_uri.netloc:
358 absolute_uri = "%s://%s" % (self.uri.scheme, self.uri.netloc)
359 location = urlparse.urljoin(absolute_uri, location)
360
361 log.debug("Redirect to %s" % location)
362
363 self.final_url = location
364 response.body.read()
365 self.nb_redirections -= 1
366
367 self.release_connection((self.host, self.port), self._sock)
368 return self.request(location, self.method, self.body, self.init_headers)
369
371 """
372 Get headers, set Body object and return HttpResponse
373 """
374
375 release_fun = lambda:self.release_connection(
376 (self.host, self.port), self._sock)
377 while True:
378 parser = http.ResponseParser(self._sock,
379 release_source=release_fun,
380 decompress=self.decompress)
381 resp = parser.next()
382 if resp.status_int != 100:
383 break
384
385 log.debug("Start response: %s", resp.status)
386 log.debug("Response headers: [%s]", resp.headers)
387
388 location = None
389 for hdr_name, hdr_value in resp.headers:
390 if hdr_name.lower() == "location":
391 location = hdr_value
392 break
393
394 if self.follow_redirect:
395 if resp.status_int in (301, 302, 307):
396 if self.method in ('GET', 'HEAD') or \
397 self.force_follow_redirect:
398 if self.method not in ('GET', 'HEAD') and \
399 hasattr(self.body, 'seek'):
400 self.body.seek(0)
401 return self.do_redirect(resp, location)
402 elif resp.status_int == 303 and self.method in ('GET',
403 'HEAD'):
404
405
406 return self.do_redirect(resp, location)
407
408
409
410 self.filters.apply("on_response", self)
411
412 self.final_url = location or self.final_url
413 log.debug("Return response: %s" % self.final_url)
414 if self.method == "HEAD":
415 resp.body = StringIO()
416
417 return self.response_class(resp, self.final_url)
418