1
2
3
4
5 import base64
6 import errno
7 import logging
8 import os
9 import time
10 import socket
11 import ssl
12 import traceback
13 import types
14 import urlparse
15
16 try:
17 from http_parser.http import HttpStream, BadStatusLine
18 from http_parser.reader import SocketReader
19 except ImportError:
20 raise ImportError("""http-parser isn't installed.
21
22 pip install http-parser""")
23
24 from restkit import __version__
25
26 from restkit.conn import Connection
27 from restkit.errors import RequestError, RequestTimeout, RedirectLimit, \
28 NoMoreData, ProxyError
29 from restkit.session import get_session
30 from restkit.util import parse_netloc, rewrite_location
31 from restkit.wrappers import Request, Response
32
33 MAX_CLIENT_TIMEOUT=300
34 MAX_CLIENT_CONNECTIONS = 5
35 MAX_CLIENT_TRIES =3
36 CLIENT_WAIT_TRIES = 0.3
37 MAX_FOLLOW_REDIRECTS = 5
38 USER_AGENT = "restkit/%s" % __version__
39
40 log = logging.getLogger(__name__)
41
43
44 """ A client handle a connection at a time. A client is threadsafe,
45 but an handled shouldn't be shared between threads. All connections
46 are shared between threads via a pool.
47
48 >>> from restkit import *
49 >>> c = Client()
50 >>> r = c.request("http://google.com")
51 r>>> r.status
52 '301 Moved Permanently'
53 >>> r.body_string()
54 '<HTML><HEAD><meta http-equiv="content-type" content="text/html;charset=utf-8">\n<TITLE>301 Moved</TITLE></HEAD><BODY>\n<H1>301 Moved</H1>\nThe document has moved\n<A HREF="http://www.google.com/">here</A>.\r\n</BODY></HTML>\r\n'
55 >>> c.follow_redirect = True
56 >>> r = c.request("http://google.com")
57 >>> r.status
58 '200 OK'
59
60 """
61
62 version = (1, 1)
63 response_class=Response
64
65 - def __init__(self,
66 follow_redirect=False,
67 force_follow_redirect=False,
68 max_follow_redirect=MAX_FOLLOW_REDIRECTS,
69 filters=None,
70 decompress=True,
71 max_status_line_garbage=None,
72 max_header_count=0,
73 pool=None,
74 response_class=None,
75 timeout=None,
76 use_proxy=False,
77 max_tries=3,
78 wait_tries=0.3,
79 backend="thread",
80 **ssl_args):
81 """
82 Client parameters
83 ~~~~~~~~~~~~~~~~~
84
85 :param follow_redirect: follow redirection, by default False
86 :param max_ollow_redirect: number of redirections available
87 :filters: http filters to pass
88 :param decompress: allows the client to decompress the response
89 body
90 :param max_status_line_garbage: defines the maximum number of ignorable
91 lines before we expect a HTTP response's status line. With
92 HTTP/1.1 persistent connections, the problem arises that broken
93 scripts could return a wrong Content-Length (there are more
94 bytes sent than specified). Unfortunately, in some cases, this
95 cannot be detected after the bad response, but only before the
96 next one. So the client is abble to skip bad lines using this
97 limit. 0 disable garbage collection, None means unlimited number
98 of tries.
99 :param max_header_count: determines the maximum HTTP header count
100 allowed. by default no limit.
101 :param manager: the manager to use. By default we use the global
102 one.
103 :parama response_class: the response class to use
104 :param timeout: the default timeout of the connection
105 (SO_TIMEOUT)
106
107 :param max_tries: the number of tries before we give up a
108 connection
109 :param wait_tries: number of time we wait between each tries.
110 :param ssl_args: named argument, see ssl module for more
111 informations
112 """
113 self.follow_redirect = follow_redirect
114 self.force_follow_redirect = force_follow_redirect
115 self.max_follow_redirect = max_follow_redirect
116 self.decompress = decompress
117 self.filters = filters or []
118 self.max_status_line_garbage = max_status_line_garbage
119 self.max_header_count = max_header_count
120 self.use_proxy = use_proxy
121
122 self.request_filters = []
123 self.response_filters = []
124 self.load_filters()
125
126
127
128
129 session_options = dict(
130 retry_delay=wait_tries,
131 retry_max = max_tries,
132 timeout = timeout)
133
134
135 if pool is None:
136 pool = get_session(backend, **session_options)
137 self._pool = pool
138 self.backend = backend
139
140
141 if response_class is not None:
142 self.response_class = response_class
143
144 self.max_tries = max_tries
145 self.wait_tries = wait_tries
146 self.timeout = timeout
147
148 self._nb_redirections = self.max_follow_redirect
149 self._url = None
150 self._initial_url = None
151 self._write_cb = None
152 self._headers = None
153 self._sock_key = None
154 self._sock = None
155 self._original = None
156
157 self.method = 'GET'
158 self.body = None
159 self.ssl_args = ssl_args or {}
160
162 """ Populate filters from self.filters.
163 Must be called each time self.filters is updated.
164 """
165 for f in self.filters:
166 if hasattr(f, "on_request"):
167 self.request_filters.append(f)
168 if hasattr(f, "on_response"):
169 self.response_filters.append(f)
170
171
172
191
193 """ do the proxy connection """
194 proxy_settings = os.environ.get('%s_proxy' %
195 request.parsed_url.scheme)
196
197 if proxy_settings and proxy_settings is not None:
198 request.is_proxied = True
199
200 proxy_settings, proxy_auth = _get_proxy_auth(proxy_settings)
201 addr = parse_netloc(urlparse.urlparse(proxy_settings))
202
203 if is_ssl:
204 if proxy_auth:
205 proxy_auth = 'Proxy-authorization: %s' % proxy_auth
206 proxy_connect = 'CONNECT %s:%s HTTP/1.0\r\n' % req_addr
207
208 user_agent = request.headers.iget('user_agent')
209 if not user_agent:
210 user_agent = "User-Agent: restkit/%s\r\n" % __version__
211
212 proxy_pieces = '%s%s%s\r\n' % (proxy_connect, proxy_auth,
213 user_agent)
214
215
216 conn = self._pool.get(host=addr[0], port=addr[1],
217 pool=self._pool, is_ssl=is_ssl,
218 extra_headers=[], **self.ssl_args)
219
220
221 conn.send(proxy_pieces)
222 p = HttpStream(SocketReader(conn.socket()), kind=1,
223 decompress=True)
224
225 if p.status_code != 200:
226 raise ProxyError("Tunnel connection failed: %d %s" %
227 (resp.status_int, body))
228
229 _ = p.body_string()
230
231 else:
232 headers = []
233 if proxy_auth:
234 headers = [('Proxy-authorization', proxy_auth)]
235
236 conn = self._pool.get(host=addr[0], port=addr[1],
237 pool=self._pool, is_ssl=False,
238 extra_headers=[], **self.ssl_args)
239 return conn
240
241 return
242
244 """ create final header string """
245 headers = request.headers.copy()
246 if extra_headers is not None:
247 for k, v in extra_headers:
248 headers[k] = v
249
250 if not request.body and request.method in ('POST', 'PUT',):
251 headers['Content-Length'] = 0
252
253 if self.version == (1,1):
254 httpver = "HTTP/1.1"
255 else:
256 httpver = "HTTP/1.0"
257
258 ua = headers.iget('user_agent')
259 if not ua:
260 ua = USER_AGENT
261 host = request.host
262
263 accept_encoding = headers.iget('accept-encoding')
264 if not accept_encoding:
265 accept_encoding = 'identity'
266
267 if request.is_proxied:
268 full_path = ("https://" if request.is_ssl() else "http://") + request.host + request.path
269 else:
270 full_path = request.path
271
272 lheaders = [
273 "%s %s %s\r\n" % (request.method, full_path, httpver),
274 "Host: %s\r\n" % host,
275 "User-Agent: %s\r\n" % ua,
276 "Accept-Encoding: %s\r\n" % accept_encoding
277 ]
278
279 lheaders.extend(["%s: %s\r\n" % (k, str(v)) for k, v in \
280 headers.items() if k.lower() not in \
281 ('user-agent', 'host', 'accept-encoding',)])
282 if log.isEnabledFor(logging.DEBUG):
283 log.debug("Send headers: %s" % lheaders)
284 return "%s\r\n" % "".join(lheaders)
285
394
395 - def request(self, url, method='GET', body=None, headers=None):
413
415 """ reset request, set new url of request and perform it """
416 if self._nb_redirections <= 0:
417 raise RedirectLimit("Redirection limit is reached")
418
419 if request.initial_url is None:
420 request.initial_url = self.url
421
422
423 location = rewrite_location(request.url, location)
424
425 if log.isEnabledFor(logging.DEBUG):
426 log.debug("Redirect to %s" % location)
427
428
429 request.url = location
430
431 self._nb_redirections -= 1
432
433
434 return self.perform(request)
435
437 """ return final respons, it is only accessible via peform
438 method """
439 if log.isEnabledFor(logging.DEBUG):
440 log.debug("Start to parse response")
441
442 p = HttpStream(SocketReader(connection.socket()), kind=1,
443 decompress=self.decompress)
444
445 if log.isEnabledFor(logging.DEBUG):
446 log.debug("Got response: %s %s" % (p.version(), p.status()))
447 log.debug("headers: [%s]" % p.headers())
448
449 location = p.headers().get('location')
450
451 if self.follow_redirect:
452 if p.status_code() in (301, 302, 307,):
453 connection.close()
454 if request.method in ('GET', 'HEAD',) or \
455 self.force_follow_redirect:
456 if hasattr(self.body, 'read'):
457 try:
458 self.body.seek(0)
459 except AttributeError:
460 raise RequestError("Can't redirect %s to %s "
461 "because body has already been read"
462 % (self.url, location))
463 return self.redirect(location, request)
464
465 elif p.status_code() == 303 and self.method == "POST":
466 connection.close()
467 request.method = "GET"
468 request.body = None
469 return self.redirect(location, request)
470
471
472 resp = self.response_class(connection, request, p)
473
474
475 for f in self.response_filters:
476 f.on_response(resp, request)
477
478 if log.isEnabledFor(logging.DEBUG):
479 log.debug("return response class")
480
481
482 return resp
483
484
486 proxy_username = os.environ.get('proxy-username')
487 if not proxy_username:
488 proxy_username = os.environ.get('proxy_username')
489 proxy_password = os.environ.get('proxy-password')
490 if not proxy_password:
491 proxy_password = os.environ.get('proxy_password')
492
493 proxy_password = proxy_password or ""
494
495 if not proxy_username:
496 u = urlparse.urlparse(proxy_settings)
497 if u.username:
498 proxy_password = u.password or proxy_password
499 proxy_settings = urlparse.urlunparse((u.scheme,
500 u.netloc.split("@")[-1], u.path, u.params, u.query,
501 u.fragment))
502
503 if proxy_username:
504 user_auth = base64.encodestring('%s:%s' % (proxy_username,
505 proxy_password))
506 return proxy_settings, 'Basic %s\r\n' % (user_auth.strip())
507 else:
508 return proxy_settings, ''
509