1
2
3
4
5
6 import copy
7 import errno
8 import logging
9 import mimetypes
10 import os
11 import time
12 import socket
13 import types
14 import urlparse
15 import uuid
16
17 try:
18 from cStringIO import StringIO
19 except ImportError:
20 from StringIO import StringIO
21
22 try:
23 import ssl
24 have_ssl = True
25 except ImportError:
26 have_ssl = False
27
28 from . import __version__
29 from .datastructures import MultiDict
30 from .errors import *
31 from .filters import Filters
32 from .forms import multipart_form_encode, form_encode
33 from .globals import get_manager
34 from . import http
35
36 from .sock import close, send, sendfile, sendlines, send_chunk
37 from .tee import TeeInput
38 from .util import parse_netloc, to_bytestring, rewrite_location
39
40 MAX_CLIENT_TIMEOUT=300
41 MAX_CLIENT_CONNECTIONS = 5
42 MAX_CLIENT_TRIES = 5
43 CLIENT_WAIT_TRIES = 0.3
44 MAX_FOLLOW_REDIRECTS = 5
45 USER_AGENT = "restkit/%s" % __version__
46
47 log = logging.getLogger(__name__)
48
49 -class BodyWrapper(object):
50
51 - def __init__(self, resp, client):
52 self.resp = resp
53 self.body = resp._body
54 self.client = client
55 self._sock = client._sock
56 self._sock_key = copy.copy(client._sock_key)
57
58 - def __enter__(self):
60
61 - def __exit__(self, exc_type, exc_val, traceback):
63
65 """ release connection """
66 self.client.release_connection(self._sock_key,
67 self._sock, self.resp.should_close)
68
71
73 try:
74 return self.body.next()
75 except StopIteration:
76 self.close()
77 raise
78
79 - def read(self, size=None):
80 data = self.body.read(size=size)
81 if not data:
82 self.close()
83 return data
84
85 - def readline(self, size=None):
86 line = self.body.readline(size=size)
87 if not line:
88 self.close()
89 return line
90
91 - def readlines(self, size=None):
92 lines = self.body.readlines(size=size)
93 if self.body.close:
94 self.close()
95 return line
96
97
99
100 charset = "utf8"
101 unicode_errors = 'strict'
102
126
128 try:
129 return getattr(self, key)
130 except AttributeError:
131 pass
132 return self.headers.iget(key)
133
136
139
145
147 return not self._closed and not self._already_read
148
149 - def body_string(self, charset=None, unicode_errors="strict"):
150 """ return body string, by default in bytestring """
151
152 if not self.can_read():
153 raise AlreadyRead()
154
155 body = self._body.read()
156 self._already_read = True
157
158
159 self.release_connection()
160
161 if charset is not None:
162 try:
163 body = body.decode(charset, unicode_errors)
164 except UnicodeDecodeError:
165 pass
166 return body
167
168 - def body_stream(self):
169 """ stream body """
170 if not self.can_read():
171 raise AlreadyRead()
172
173 self._already_read = True
174
175 return BodyWrapper(self, self.client)
176
178 """ copy response input to standard output or a file if length >
179 sock.MAX_BODY. This make possible to reuse it in your
180 appplication. When all the input has been read, connection is
181 released """
182 if not hasattr(self._body, "reader"):
183
184 return self._body
185
186 return TeeInput(self, self.client)
187
188
190
191 """ A client handle a connection at a time. A client is threadsafe,
192 but an handled shouldn't be shared between threads. All connections
193 are shared between threads via a pool.
194
195 >>> from restkit import *
196 >>> c = Client()
197 >>> c.url = "http://google.com"
198 >>> r = c.perform()
199 r>>> r.status
200 '301 Moved Permanently'
201 >>> r.body_string()
202 '<HTML><HEAD><meta http-equiv="content-type" content="text/html;charset=utf-8">\n<TITLE>301 Moved</TITLE></HEAD><BODY>\n<H1>301 Moved</H1>\nThe document has moved\n<A HREF="http://www.google.com/">here</A>.\r\n</BODY></HTML>\r\n'
203 >>> c.follow_redirect = True
204 >>> r = c.perform()
205 >>> r.status
206 '200 OK'
207
208 """
209
210 version = (1, 1)
211 response_class=ClientResponse
212
213 - def __init__(self,
214 follow_redirect=False,
215 force_follow_redirect=False,
216 max_follow_redirect=MAX_FOLLOW_REDIRECTS,
217 filters=None,
218 decompress=True,
219 manager=None,
220 response_class=None,
221 timeout=MAX_CLIENT_TIMEOUT,
222 force_dns=False,
223 max_tries=5,
224 wait_tries=1.0,
225 **ssl_args):
226
227 self.follow_redirect = follow_redirect
228 self.force_follow_redirect = force_follow_redirect
229 self.max_follow_redirect = max_follow_redirect
230 self.filters = Filters(filters)
231 self.decompress = decompress
232
233
234 if manager is None:
235 manager = get_manager()
236 self._manager = manager
237
238
239 if response_class is not None:
240 self.response_class = response_class
241
242 self.max_tries = max_tries
243 self.wait_tries = wait_tries
244 self.timeout = timeout
245
246 self._nb_redirections = self.max_follow_redirect
247 self._url = None
248 self._initial_url = None
249 self._write_cb = None
250 self._headers = None
251 self._sock_key = None
252 self._sock = None
253 self._original = None
254
255 self.method = 'GET'
256 self.body = None
257 self.ssl_args = ssl_args or {}
258
259
261 if not isinstance(self._headers, MultiDict):
262 self._headers = MultiDict(self._headers or [])
263 return self._headers
266 headers = property(_headers__get, _headers__set, doc=_headers__get.__doc__)
267
268
270 if not callable(cb):
271 raise ValueError("%s isn't a callable" % str(cb))
272 self._write_cb = cb
273
275 if self._url is None:
276 raise ValueError("url isn't set")
277 return urlparse.urlunparse(self._url)
279 self._url = urlparse.urlparse(string)
280 url = property(_url__get, _url__set, doc="current url to request")
281
283 if self._url is None:
284 raise ValueError("url isn't set")
285 return self._url
286 parsed_url = property(_parsed_url__get)
287
289 try:
290 h = self.parsed_url.netloc.encode('ascii')
291 except UnicodeEncodeError:
292 h = self.parsed_url.netloc.encode('idna')
293
294 hdr_host = self.headers.iget("host")
295 if not hdr_host:
296 return h
297 return hdr_host
298 host = property(_host__get)
299
301 path = self.parsed_url.path or '/'
302
303 return urlparse.urlunparse(('','', path, self._url.params,
304 self._url.query, self._url.fragment))
305 path = property(_path__get, doc="request path")
306
308 te = self.headers.iget("transfer-encoding")
309 return (te is not None and te.lower() == "chunked")
310
313
315 """ create a socket """
316 log.debug("create new connection")
317 for res in socket.getaddrinfo(addr[0], addr[1], 0,
318 socket.SOCK_STREAM):
319 af, socktype, proto, canonname, sa = res
320
321 try:
322 sck = socket.socket(af, socktype, proto)
323
324 sck.settimeout(self.timeout)
325 sck.connect(sa)
326
327 if ssl:
328 if not have_ssl:
329 raise ValueError("https isn't supported. On python 2.5x,"
330 + " https support requires ssl module "
331 + "(http://pypi.python.org/pypi/ssl) "
332 + "to be intalled.")
333 validate_ssl_args(self.ssl_args)
334 sock = ssl.wrap_socket(sck, **self.ssl_args)
335
336
337 self.filters.apply("on_connect", self, sck, ssl)
338
339 return sck
340 except socket.error:
341 close(sck)
342 raise socket.error, "getaddrinfo returns an empty list"
343
354
365
367 """ close a connection """
368 log.debug("close connection")
369 close(self._sock)
370 self._sock = None
371
372 - def parse_body(self):
373 """ transform a body if needed and set appropriate headers """
374 if not self.body:
375 if self.method in ('POST', 'PUT',):
376 self.headers['Content-Length'] = 0
377 return
378
379 ctype = self.headers.iget('content-type')
380 clen = self.headers.iget('content-length')
381
382 if isinstance(self.body, dict):
383 if ctype is not None and \
384 ctype.startswith("multipart/form-data"):
385 type_, opts = cgi.parse_header(ctype)
386 boundary = opts.get('boundary', uuid.uuid4().hex)
387 self.body, self.headers = multipart_form_encode(body,
388 self.headers, boundary)
389 else:
390 ctype = "application/x-www-form-urlencoded; charset=utf-8"
391 self.body = form_encode(self.body)
392 elif hasattr(self.body, "boundary"):
393 ctype = "multipart/form-data; boundary=%s" % self.body.boundary
394 clen = self.body.get_size()
395
396 if not ctype:
397 ctype = 'application/octet-stream'
398 if hasattr(self.body, 'name'):
399 ctype = mimetypes.guess_type(self.body.name)[0]
400
401 if not clen:
402 if hasattr(self.body, 'fileno'):
403 try:
404 self.body.flush()
405 except IOError:
406 pass
407 try:
408 fno = self.body.fileno()
409 clen = str(os.fstat(fno)[6])
410 except IOError:
411 if not self.req_is_chunked():
412 clen = len(self.body.read())
413 elif hasattr(self.body, 'getvalue') and not \
414 self.req_is_chunked():
415 clen = len(self.body.getvalue())
416 elif isinstance(self.body, types.StringTypes):
417 self.body = to_bytestring(self.body)
418 clen = len(self.body)
419
420 if clen is not None:
421 self.headers['Content-Length'] = clen
422 elif not self.req_is_chunked():
423 raise RequestError("Can't determine content length and " +
424 "Transfer-Encoding header is not chunked")
425
426 if ctype is not None:
427 self.headers['Content-Type'] = ctype
428
430 """ create final header string """
431 if self.version == (1,1):
432 httpver = "HTTP/1.1"
433 else:
434 httpver = "HTTP/1.0"
435
436 ua = self.headers.iget('user_agent')
437 host = self.host
438 accept_encoding = self.headers.iget('accept-encoding')
439
440 headers = [
441 "%s %s %s\r\n" % (self.method, self.path, httpver),
442 "Host: %s\r\n" % host,
443 "User-Agent: %s\r\n" % ua or USER_AGENT,
444 "Accept-Encoding: %s\r\n" % accept_encoding or 'identity'
445 ]
446
447 headers.extend(["%s: %s\r\n" % (k, str(v)) for k, v in \
448 self.headers.items() if k.lower() not in \
449 ('user-agent', 'host', 'accept-encoding',)])
450
451 log.debug("Send headers: %s" % headers)
452 return "%s\r\n" % "".join(headers)
453
455 """ reset a client handle to its intial state before performing.
456 It doesn't handle case where body has already been consumed """
457 if self._original is None:
458 return
459
460 self.url = self._original["url"]
461 self.method = self._original["method"]
462 self.body = self._original["body"]
463 self.headers = self._original["headers"]
464 self._nb_redirections = self.max_follow_redirect
465
548
549 - def request(self, url, method='GET', body=None, headers=None):
557
558 - def redirect(self, resp, location, method=None):
559 """ reset request, set new url of request and perform it """
560 if self._nb_redirections <= 0:
561 raise RedirectLimit("Redirection limit is reached")
562
563 if self._initial_url is None:
564 self._initial_url = self.url
565
566
567 if hasattr(resp, "_body"):
568 resp._body.discard()
569 else:
570 resp.body.discard()
571 self.reset_request()
572
573
574 location = rewrite_location(self.url, location)
575
576 log.debug("Redirect to %s" % location)
577
578
579 self.url = location
580 if method is not None:
581 self.method = "GET"
582
583 self._nb_redirections -= 1
584 return self.perform()
585
587 """ return final respons, it is only accessible via peform
588 method """
589 unreader = http.Unreader(self._sock)
590
591 log.info("Start to parse response")
592 while True:
593 resp = http.Request(unreader)
594 if resp.status_int != 100:
595 break
596 resp.body.discard()
597 log.debug("Go 100-Continue header")
598
599 log.info("Got response: %s" % resp.status)
600 log.info("headers: [%s]" % resp.headers)
601
602 location = resp.headers.iget('location')
603
604 if self.follow_redirect:
605 if resp.status_int in (301, 302, 307,):
606 if self.method in ('GET', 'HEAD',) or \
607 self.force_follow_redirect:
608 if hasattr(self.body, 'read'):
609 try:
610 self.body.seek(0)
611 except AttributeError:
612 raise RequestError("Can't redirect %s to %s "
613 "because body has already been read"
614 % (self.url, location))
615 return self.redirect(resp, location)
616
617 elif resp.status_int == 303 and self.method == "POST":
618 return self.redirect(resp, location, method="GET")
619
620
621 self.filters.apply("on_response", self, resp)
622
623
624 self.reset_request()
625
626 log.debug("return response class")
627 return self.response_class(self, resp)
628