1
2
3
4
5
6 import os
7 import re
8 import sys
9 import urlparse
10 import zlib
11
12 try:
13 from cStringIO import StringIO
14 except ImportError:
15 from StringIO import StringIO
16
17 from .datastructures import MultiDict
18 from .errors import *
19
20
22 - def __init__(self, sock, max_chunk=8192):
23 self.buf = StringIO()
24 self.sock = sock
25 self.max_chunk = max_chunk
26
28 return self.sock.recv(self.max_chunk)
29
30 - def read(self, size=None):
31 if size is not None and not isinstance(size, (int, long)):
32 raise TypeError("size parameter must be an int or long.")
33 if size == 0:
34 return ""
35 if size < 0:
36 size = None
37
38 self.buf.seek(0, os.SEEK_END)
39
40 if size is None and self.buf.tell():
41 ret = self.buf.getvalue()
42 self.buf.truncate(0)
43 return ret
44 if size is None:
45 return self._data()
46
47 while self.buf.tell() < size:
48 data = self._data()
49 if not len(data):
50 ret = self.buf.getvalue()
51 self.buf.truncate(0)
52 return ret
53 self.buf.write(data)
54
55 data = self.buf.getvalue()
56 self.buf.truncate(0)
57 self.buf.write(data[size:])
58 return data[:size]
59
61 self.buf.seek(0, os.SEEK_END)
62 self.buf.write(data)
63
66 self.unreader = unreader
67 self.req = req
68 self.parser = self.parse_chunked(unreader)
69 self.buf = StringIO()
70
71 - def read(self, size):
72 if not isinstance(size, (int, long)):
73 raise TypeError("size must be an integral type")
74 if size <= 0:
75 raise ValueError("Size must be positive.")
76 if size == 0:
77 return ""
78
79 if self.parser:
80 while self.buf.tell() < size:
81 try:
82 self.buf.write(self.parser.next())
83 except StopIteration:
84 self.parser = None
85 break
86
87 data = self.buf.getvalue()
88 ret, rest = data[:size], data[size:]
89 self.buf.truncate(0)
90 self.buf.write(rest)
91 return ret
92
94 buf = StringIO()
95 buf.write(data)
96
97 idx = buf.getvalue().find("\r\n\r\n")
98 done = buf.getvalue()[:2] == "\r\n"
99
100 while idx < 0 and not done:
101 self.get_data(unreader, buf)
102 idx = buf.getvalue().find("\r\n\r\n")
103 done = buf.getvalue()[:2] == "\r\n"
104 if done:
105 unreader.unread(buf.getvalue()[2:])
106 return ""
107 self.req.trailers = self.req.parse_headers(buf.getvalue()[:idx])
108 unreader.unread(buf.getvalue()[idx+4:])
109
111 (size, rest) = self.parse_chunk_size(unreader)
112 while size > 0:
113 while size > len(rest):
114 size -= len(rest)
115 yield rest
116 rest = unreader.read()
117 if not rest:
118 raise NoMoreData()
119 yield rest[:size]
120
121 rest = rest[size:]
122 while len(rest) < 2:
123 rest += unreader.read()
124 if rest[:2] != '\r\n':
125 raise ChunkMissingTerminator(rest[:2])
126 (size, rest) = self.parse_chunk_size(unreader, data=rest[2:])
127
129 buf = StringIO()
130 if data is not None:
131 buf.write(data)
132
133 idx = buf.getvalue().find("\r\n")
134 while idx < 0:
135 self.get_data(unreader, buf)
136 idx = buf.getvalue().find("\r\n")
137
138 data = buf.getvalue()
139 line, rest_chunk = data[:idx], data[idx+2:]
140
141 chunk_size = line.split(";", 1)[0].strip()
142 try:
143 chunk_size = int(chunk_size, 16)
144 except ValueError:
145 raise InvalidChunkSize(chunk_size)
146
147 if chunk_size == 0:
148 try:
149 self.parse_trailers(unreader, rest_chunk)
150 except NoMoreData:
151 pass
152 return (0, None)
153 return (chunk_size, rest_chunk)
154
156 data = unreader.read()
157 if not data:
158 raise NoMoreData()
159 buf.write(data)
160
162 - def __init__(self, req, unreader, length):
163 self.req = req
164 self.unreader = unreader
165 self.length = length
166
167 - def read(self, size):
168 if not isinstance(size, (int, long)):
169 raise TypeError("size must be an integral type")
170
171 size = min(self.length, size)
172 if size < 0:
173 raise ValueError("Size must be positive.")
174 if size == 0:
175 return ""
176
177 buf = StringIO()
178 data = self.unreader.read()
179 while data:
180 buf.write(data)
181 if buf.tell() >= size:
182 break
183 data = self.unreader.read()
184
185
186 buf = buf.getvalue()
187 ret, rest = buf[:size], buf[size:]
188 self.unreader.unread(rest)
189 self.length -= size
190 return ret
191
194 self.req = req
195 self.unreader = unreader
196 self.buf = StringIO()
197 self.finished = False
198
199 - def read(self, size):
200 if not isinstance(size, (int, long)):
201 raise TypeError("size must be an integral type")
202 if size < 0:
203 raise ValueError("Size must be positive.")
204 if size == 0:
205 return ""
206
207 if self.finished:
208 data = self.buf.getvalue()
209 ret, rest = data[:size], data[size:]
210 self.buf.truncate(0)
211 self.buf.write(rest)
212 return ret
213
214 data = self.unreader.read()
215 while data:
216 self.buf.write(data)
217 if self.buf.tell() > size:
218 break
219 data = self.unreader.read()
220
221 if not data:
222 self.finished = True
223
224 data = self.buf.getvalue()
225 ret, rest = data[:size], data[size:]
226 self.buf.truncate(0)
227 self.buf.write(rest)
228 return ret
229
231 - def __init__(self, reader):
232 self.reader = reader
233 self.buf = StringIO()
234 self.closed = False
235
236 - def __iter__(self):
238
240 ret = self.readline()
241 if not ret:
242 raise StopIteration()
243 return ret
244
246 data = self.read(8192)
247 while data:
248 data = self.read()
249
250 - def getsize(self, size):
251 if size is None:
252 return sys.maxint
253 elif not isinstance(size, (int, long)):
254 raise TypeError("size must be an integral type")
255 elif size < 0:
256 return sys.maxint
257 return size
258
259 - def read(self, size=None):
260 size = self.getsize(size)
261 if size == 0:
262 return ""
263
264 if size < self.buf.tell():
265 data = self.buf.getvalue()
266 ret, rest = data[:size], data[size:]
267 self.buf.truncate(0)
268 self.buf.write(rest)
269 return ret
270
271 while size > self.buf.tell():
272 data = self.reader.read(1024)
273 if not len(data):
274 self.closed = True
275 break
276 self.buf.write(data)
277
278 data = self.buf.getvalue()
279 ret, rest = data[:size], data[size:]
280 self.buf.truncate(0)
281 self.buf.write(rest)
282 return ret
283
284 - def readline(self, size=None):
285 size = self.getsize(size)
286 if size == 0:
287 return ""
288
289 line = self.buf.getvalue()
290 idx = line.find("\n")
291 if idx >= 0:
292 ret = line[:idx+1]
293 self.buf.truncate(0)
294 self.buf.write(line[idx+1:])
295 return ret
296
297 self.buf.truncate(0)
298 ch = ""
299 buf = [line]
300 lsize = len(line)
301 while lsize < size and ch != "\n":
302 ch = self.reader.read(1)
303 if not len(ch):
304 self.closed = True
305 break
306 lsize += 1
307 buf.append(ch)
308 return "".join(buf)
309
310 - def readlines(self, size=None):
311 ret = []
312 data = self.read()
313 while len(data):
314 pos = data.find("\n")
315 if pos < 0:
316 ret.append(data)
317 data = ""
318 else:
319 line, data = data[:pos+1], data[pos+1:]
320 ret.append(line)
321 return ret
322
323
324 -class GzipBody(Body):
325 - def __init__(self, reader):
326 super(GzipBody, self).__init__(reader)
327 self._d = zlib.decompressobj(16+zlib.MAX_WBITS)
328
329 - def _decompress(self, data):
330 return self._d.decompress(data)
331
332 - def read(self, size=None):
333 size = self.getsize(size)
334 if size == 0:
335 return ""
336
337 if size < self.buf.tell():
338 data = self.buf.getvalue()
339 ret, rest = data[:size], data[size:]
340 self.buf.truncate(0)
341 self.buf.write(rest)
342 return self._decompress(ret)
343
344 while size > self.buf.tell():
345 data = self.reader.read(1024)
346 if not len(data):
347 break
348 self.buf.write(data)
349
350 data = self.buf.getvalue()
351 ret, rest = data[:size], data[size:]
352 self.buf.truncate(0)
353 self.buf.write(rest)
354 return self._decompress(ret)
355
356 - def readline(self, size=None):
357 size = self.getsize(size)
358 if size == 0:
359 return ""
360
361 idx = self.buf.getvalue().find("\n")
362 while idx < 0:
363 data = self.reader.read(1024)
364 if not len(data):
365 break
366 self.buf.write(self._decompress(data))
367 idx = self.buf.getvalue().find("\n")
368 if size < self.buf.tell():
369 break
370
371
372
373 if idx < 0:
374 rlen = min(size, self.buf.tell())
375 else:
376 rlen = idx + 1
377
378
379 if rlen > size:
380 rlen = size
381
382 data = self.buf.getvalue()
383 ret, rest = data[:rlen], data[rlen:]
384
385 self.buf.truncate(0)
386 self.buf.write(rest)
387 return ret
388
389
390 -class DeflateBody(GzipBody):
391 - def __init__(self, reader):
392 super(DeflateBody, self).__init__(reader)
393 self._d = zlib.decompressobj()
394
395
397 - def __init__(self, unreader, decompress=True):
398 self.unreader = unreader
399 self.version = None
400 self.headers = MultiDict()
401 self.trailers = []
402 self.body = None
403 self.encoding = None
404 self.status = None
405 self.reason = None
406 self.status_int = None
407 self.decompress = decompress
408
409 self.versre = re.compile("HTTP/(\d+).(\d+)")
410 self.stare = re.compile("(\d{3})\s*(\w*)")
411 self.hdrre = re.compile("[\x00-\x1F\x7F()<>@,;:\[\]={} \t\\\\\"]")
412
413 unused = self.parse(self.unreader)
414 self.unreader.unread(unused)
415 self.set_body_reader()
416
417 - def get_data(self, unreader, buf, stop=False):
418 data = unreader.read()
419 if not data:
420 if stop:
421 raise StopIteration()
422 raise NoMoreData(buf.getvalue())
423 buf.write(data)
424
425 - def parse(self, unreader):
426 buf = StringIO()
427
428 self.get_data(unreader, buf, stop=True)
429
430
431 idx = buf.getvalue().find("\r\n")
432 while idx < 0:
433 self.get_data(unreader, buf)
434 idx = buf.getvalue().find("\r\n")
435 self.parse_first_line(buf.getvalue()[:idx])
436 rest = buf.getvalue()[idx+2:]
437 buf.truncate(0)
438 buf.write(rest)
439
440
441 idx = buf.getvalue().find("\r\n\r\n")
442 done = buf.getvalue()[:2] == "\r\n"
443 while idx < 0 and not done:
444 self.get_data(unreader, buf)
445 idx = buf.getvalue().find("\r\n\r\n")
446 done = buf.getvalue()[:2] == "\r\n"
447 if done:
448 self.unreader.unread(buf.getvalue()[2:])
449 return ""
450
451 self.headers = self.parse_headers(buf.getvalue()[:idx])
452
453 ret = buf.getvalue()[idx+4:]
454 buf.truncate(0)
455 return ret
456
458 bits = line.split(None, 1)
459 if len(bits) != 2:
460 raise InvalidRequestLine(line)
461
462
463 matchv = self.versre.match(bits[0])
464 if matchv is None:
465 raise InvalidHTTPVersion(bits[0])
466 self.version = (int(matchv.group(1)), int(matchv.group(2)))
467
468
469 matchs = self.stare.match(bits[1])
470 if matchs is None:
471 raise InvalidHTTPStatus(bits[1])
472
473 self.status = bits[1]
474 self.status_int = int(matchs.group(1))
475 self.reason = matchs.group(2)
476
478 headers = MultiDict()
479
480
481 lines = [line + "\r\n" for line in data.split("\r\n")]
482
483
484
485 while len(lines):
486
487 curr = lines.pop(0)
488 if curr.find(":") < 0:
489 raise InvalidHeader(curr.strip())
490 name, value = curr.split(":", 1)
491 name = name.rstrip(" \t")
492 if self.hdrre.search(name.upper()):
493 raise InvalidHeaderName(name)
494 name, value = name.strip(), [value.lstrip()]
495
496
497 while len(lines) and lines[0].startswith((" ", "\t")):
498 value.append(lines.pop(0))
499 value = ''.join(value).rstrip()
500
501 headers.add(name, value)
502 return headers
503
504 - def set_body_reader(self):
505 clen = self.headers.iget('content-length')
506 te = self.headers.iget('transfer-encoding')
507 encoding = self.headers.iget('content-encoding')
508
509 chunked = False
510 clength = None
511 if clen is not None:
512 try:
513 clength = int(clen)
514 except ValueError:
515 pass
516 elif te is not None:
517 chunked = te.lower() == "chunked"
518
519 if encoding:
520 self.encoding = encoding.lower()
521
522 if chunked:
523 reader = ChunkedReader(self, self.unreader)
524 elif clength is not None:
525 reader = LengthReader(self, self.unreader, clength)
526 else:
527 reader = EOFReader(self, self.unreader)
528
529 if self.decompress and self.encoding in ('gzip', 'deflate',):
530 if self.encoding == "gzip":
531 self.body = GzipBody(reader)
532 else:
533 self.body = DeflateBody(reader)
534 else:
535 self.body = Body(reader)
536
538 connection = self.headers.iget("connection")
539
540 if connection is not None:
541 if connection.lower().strip() == "close":
542 return True
543 elif connection.lower().strip() == "keep-alive":
544 return False
545 return self.version <= (1, 0)
546