1
2
3
4
5 import base64
6 import errno
7 import logging
8 import os
9 import time
10 import socket
11 import traceback
12 import types
13 import urlparse
14
15 try:
16 import ssl
17 _ssl_wrapper = ssl.wrap_socket
18 have_ssl = True
19 except ImportError:
20 if hasattr(socket, "ssl"):
21 from httplib import FakeSocket
22 from restkit.sock import trust_all_certificates
23
24 @trust_all_certificates
26 ssl_sck = socket.ssl(sck, **kwargs)
27 return FakeSocket(sck, ssl_sck)
28 have_ssl = True
29 else:
30 have_ssl = False
31
32 try:
33 from http_parser.http import HttpStream
34 from http_parser.reader import SocketReader
35 except ImportError:
36 raise ImportError("""http-parser isn't installed.
37
38 pip install http-parser""")
39
40 from restkit import __version__
41
42 from restkit.conn import Connection
43 from restkit.errors import RequestError, RequestTimeout, RedirectLimit, \
44 NoMoreData, ProxyError
45 from restkit.globals import get_manager
46 from restkit.sock import close, send, sendfile, sendlines, send_chunk, \
47 validate_ssl_args
48 from restkit.util import parse_netloc, rewrite_location
49 from restkit.wrappers import Request, Response
50
51 MAX_CLIENT_TIMEOUT=300
52 MAX_CLIENT_CONNECTIONS = 5
53 MAX_CLIENT_TRIES = 5
54 CLIENT_WAIT_TRIES = 0.3
55 MAX_FOLLOW_REDIRECTS = 5
56 USER_AGENT = "restkit/%s" % __version__
57
58 log = logging.getLogger(__name__)
61
62 """ A client handle a connection at a time. A client is threadsafe,
63 but an handled shouldn't be shared between threads. All connections
64 are shared between threads via a pool.
65
66 >>> from restkit import *
67 >>> c = Client()
68 >>> r = c.request("http://google.com")
69 r>>> r.status
70 '301 Moved Permanently'
71 >>> r.body_string()
72 '<HTML><HEAD><meta http-equiv="content-type" content="text/html;charset=utf-8">\n<TITLE>301 Moved</TITLE></HEAD><BODY>\n<H1>301 Moved</H1>\nThe document has moved\n<A HREF="http://www.google.com/">here</A>.\r\n</BODY></HTML>\r\n'
73 >>> c.follow_redirect = True
74 >>> r = c.request("http://google.com")
75 >>> r.status
76 '200 OK'
77
78 """
79
80 version = (1, 1)
81 response_class=Response
82
83 - def __init__(self,
84 follow_redirect=False,
85 force_follow_redirect=False,
86 max_follow_redirect=MAX_FOLLOW_REDIRECTS,
87 filters=None,
88 decompress=True,
89 max_status_line_garbage=None,
90 max_header_count=0,
91 manager=None,
92 response_class=None,
93 timeout=None,
94 use_proxy=False,
95 max_tries=5,
96 wait_tries=1.0,
97 **ssl_args):
98 """
99 Client parameters
100 ~~~~~~~~~~~~~~~~~
101
102 :param follow_redirect: follow redirection, by default False
103 :param max_ollow_redirect: number of redirections available
104 :filters: http filters to pass
105 :param decompress: allows the client to decompress the response
106 body
107 :param max_status_line_garbage: defines the maximum number of ignorable
108 lines before we expect a HTTP response's status line. With
109 HTTP/1.1 persistent connections, the problem arises that broken
110 scripts could return a wrong Content-Length (there are more
111 bytes sent than specified). Unfortunately, in some cases, this
112 cannot be detected after the bad response, but only before the
113 next one. So the client is abble to skip bad lines using this
114 limit. 0 disable garbage collection, None means unlimited number
115 of tries.
116 :param max_header_count: determines the maximum HTTP header count
117 allowed. by default no limit.
118 :param manager: the manager to use. By default we use the global
119 one.
120 :parama response_class: the response class to use
121 :param timeout: the default timeout of the connection
122 (SO_TIMEOUT)
123
124 :param max_tries: the number of tries before we give up a
125 connection
126 :param wait_tries: number of time we wait between each tries.
127 :param ssl_args: named argument, see ssl module for more
128 informations
129 """
130 self.follow_redirect = follow_redirect
131 self.force_follow_redirect = force_follow_redirect
132 self.max_follow_redirect = max_follow_redirect
133 self.decompress = decompress
134 self.filters = filters or []
135 self.max_status_line_garbage = max_status_line_garbage
136 self.max_header_count = max_header_count
137 self.use_proxy = use_proxy
138
139 self.request_filters = []
140 self.response_filters = []
141 self.load_filters()
142
143
144
145 if manager is None:
146 manager = get_manager()
147 self._manager = manager
148
149
150 if response_class is not None:
151 self.response_class = response_class
152
153 self.max_tries = max_tries
154 self.wait_tries = wait_tries
155 self.timeout = timeout
156
157 self._nb_redirections = self.max_follow_redirect
158 self._url = None
159 self._initial_url = None
160 self._write_cb = None
161 self._headers = None
162 self._sock_key = None
163 self._sock = None
164 self._original = None
165
166 self.method = 'GET'
167 self.body = None
168 self.ssl_args = ssl_args or {}
169
171 """ Populate filters from self.filters.
172 Must be called each time self.filters is updated.
173 """
174 for f in self.filters:
175 if hasattr(f, "on_request"):
176 self.request_filters.append(f)
177 if hasattr(f, "on_response"):
178 self.response_filters.append(f)
179
181 """ create a socket """
182 if log.isEnabledFor(logging.DEBUG):
183 log.debug("create new connection")
184 for res in socket.getaddrinfo(addr[0], addr[1], 0,
185 socket.SOCK_STREAM):
186 af, socktype, proto, canonname, sa = res
187
188 try:
189 sck = socket.socket(af, socktype, proto)
190
191 if self.timeout is not None:
192 sck.settimeout(self.timeout)
193
194 sck.connect(sa)
195
196 if is_ssl:
197 if not have_ssl:
198 raise ValueError("https isn't supported. On python 2.5x,"
199 + " https support requires ssl module "
200 + "(http://pypi.python.org/pypi/ssl) "
201 + "to be intalled.")
202 validate_ssl_args(self.ssl_args)
203 sck = _ssl_wrapper(sck, **self.ssl_args)
204
205 return sck
206 except socket.error, ex:
207 if ex == 24:
208 raise
209 else:
210 close(sck)
211 raise socket.error, "Can't connect to %s" % str(addr)
212
214 """ get a connection from the pool or create new one. """
215
216 addr = parse_netloc(request.parsed_url)
217 is_ssl = request.is_ssl()
218
219 extra_headers = []
220 sck = None
221 if self.use_proxy:
222 sck, addr, extra_headers = self.proxy_connection(request,
223 addr, is_ssl)
224 if not sck:
225 sck = self._manager.find_socket(addr, is_ssl)
226 if sck is None:
227 sck = self.connect(addr, is_ssl)
228
229
230 if self.timeout is not None:
231 sck.settimeout(self.timeout)
232
233 connection = Connection(sck, self._manager, addr,
234 ssl=is_ssl, extra_headers=extra_headers)
235 return connection
236
238 """ do the proxy connection """
239 proxy_settings = os.environ.get('%s_proxy' %
240 request.parsed_url.scheme)
241
242 if proxy_settings and proxy_settings is not None:
243 proxy_settings, proxy_auth = _get_proxy_auth(proxy_settings)
244 addr = parse_netloc(urlparse.urlparse(proxy_settings))
245
246 if is_ssl:
247 if proxy_auth:
248 proxy_auth = 'Proxy-authorization: %s' % proxy_auth
249 proxy_connect = 'CONNECT %s:%s HTTP/1.0\r\n' % req_addr
250
251 user_agent = request.headers.iget('user_agent')
252 if not user_agent:
253 user_agent = "User-Agent: restkit/%s\r\n" % __version__
254
255 proxy_pieces = '%s%s%s\r\n' % (proxy_connect, proxy_auth,
256 user_agent)
257
258 sck = self._manager.find_socket(addr, ssl)
259 if sck is None:
260 self = self.connect(addr, ssl)
261
262 send(sck, proxy_pieces)
263 unreader = http.Unreader(sck)
264 resp = http.Request(unreader)
265 body = resp.body.read()
266 if resp.status_int != 200:
267 raise ProxyError("Tunnel connection failed: %d %s" %
268 (resp.status_int, body))
269
270 return sck, addr, []
271 else:
272 headers = []
273 if proxy_auth:
274 headers = [('Proxy-authorization', proxy_auth)]
275
276 sck = self._manager.find_socket(addr, ssl)
277 if sck is None:
278 sck = self.connect(addr, ssl)
279 return sck, addr, headers
280 return None, req_addr, []
281
283 """ create final header string """
284 headers = request.headers.copy()
285 if extra_headers is not None:
286 for k, v in extra_headers:
287 headers[k] = v
288
289 if not request.body and request.method in ('POST', 'PUT',):
290 headers['Content-Length'] = 0
291
292 if self.version == (1,1):
293 httpver = "HTTP/1.1"
294 else:
295 httpver = "HTTP/1.0"
296
297 ua = headers.iget('user_agent')
298 if not ua:
299 ua = USER_AGENT
300 host = request.host
301
302 accept_encoding = headers.iget('accept-encoding')
303 if not accept_encoding:
304 accept_encoding = 'identity'
305
306 lheaders = [
307 "%s %s %s\r\n" % (request.method, request.path, httpver),
308 "Host: %s\r\n" % host,
309 "User-Agent: %s\r\n" % ua,
310 "Accept-Encoding: %s\r\n" % accept_encoding
311 ]
312
313 lheaders.extend(["%s: %s\r\n" % (k, str(v)) for k, v in \
314 headers.items() if k.lower() not in \
315 ('user-agent', 'host', 'accept-encoding',)])
316 if log.isEnabledFor(logging.DEBUG):
317 log.debug("Send headers: %s" % lheaders)
318 return "%s\r\n" % "".join(lheaders)
319
437
438
439 - def request(self, url, method='GET', body=None, headers=None):
457
458 - def redirect(self, resp, location, request):
459 """ reset request, set new url of request and perform it """
460 if self._nb_redirections <= 0:
461 raise RedirectLimit("Redirection limit is reached")
462
463 if request.initial_url is None:
464 request.initial_url = self.url
465
466
467 location = rewrite_location(request.url, location)
468
469 if log.isEnabledFor(logging.DEBUG):
470 log.debug("Redirect to %s" % location)
471
472
473 request.url = location
474
475 self._nb_redirections -= 1
476
477
478 return self.perform(request)
479
481 """ return final respons, it is only accessible via peform
482 method """
483 if log.isEnabledFor(logging.DEBUG):
484 log.debug("Start to parse response")
485
486 p = HttpStream(SocketReader(connection.socket()), kind=1,
487 decompress=self.decompress)
488
489 if log.isEnabledFor(logging.DEBUG):
490 log.debug("Got response: %s" % p.status())
491 log.debug("headers: [%s]" % p.headers())
492
493 location = p.headers().get('location')
494
495 if self.follow_redirect:
496 if p.status_code() in (301, 302, 307,):
497 if request.method in ('GET', 'HEAD',) or \
498 self.force_follow_redirect:
499 if hasattr(self.body, 'read'):
500 try:
501 self.body.seek(0)
502 except AttributeError:
503 connection.release()
504 raise RequestError("Can't redirect %s to %s "
505 "because body has already been read"
506 % (self.url, location))
507 connection.release()
508 return self.redirect(p, location, request)
509
510 elif p.status_code() == 303 and self.method == "POST":
511 connection.release()
512 request.method = "GET"
513 request.body = None
514 return self.redirect(p, location, request)
515
516
517 resp = self.response_class(connection, request, p)
518
519
520 for f in self.response_filters:
521 f.on_response(resp, request)
522
523 if log.isEnabledFor(logging.DEBUG):
524 log.debug("return response class")
525
526
527 return resp
528
531 proxy_username = os.environ.get('proxy-username')
532 if not proxy_username:
533 proxy_username = os.environ.get('proxy_username')
534 proxy_password = os.environ.get('proxy-password')
535 if not proxy_password:
536 proxy_password = os.environ.get('proxy_password')
537
538 proxy_password = proxy_password or ""
539
540 if not proxy_username:
541 u = urlparse.urlparse(proxy_settings)
542 if u.username:
543 proxy_password = u.password or proxy_password
544 proxy_settings = urlparse.urlunparse((u.scheme,
545 u.netloc.split("@")[-1], u.path, u.params, u.query,
546 u.fragment))
547
548 if proxy_username:
549 user_auth = base64.encodestring('%s:%s' % (proxy_username,
550 proxy_password))
551 return proxy_settings, 'Basic %s\r\n' % (user_auth.strip())
552 else:
553 return proxy_settings, ''
554