Package restkit :: Module http
[hide private]
[frames] | no frames]

Source Code for Module restkit.http

  1  # -*- coding: utf-8 - 
  2  # 
  3  # This file is part of restkit released under the MIT license.  
  4  # See the NOTICE for more information. 
  5   
  6  import os 
  7  import re 
  8  import sys 
  9  import urlparse 
 10  import zlib 
 11   
 12  try: 
 13      from cStringIO import StringIO 
 14  except ImportError: 
 15      from StringIO import StringIO 
 16   
 17  from .datastructures import MultiDict 
 18  from .errors import * 
 19   
 20   
21 -class Unreader(object):
22 - def __init__(self, sock, max_chunk=8192):
23 self.buf = StringIO() 24 self.sock = sock 25 self.max_chunk = max_chunk
26
27 - def _data(self):
28 return self.sock.recv(self.max_chunk)
29
30 - def read(self, size=None):
31 if size is not None and not isinstance(size, (int, long)): 32 raise TypeError("size parameter must be an int or long.") 33 if size == 0: 34 return "" 35 if size < 0: 36 size = None 37 38 self.buf.seek(0, os.SEEK_END) 39 40 if size is None and self.buf.tell(): 41 ret = self.buf.getvalue() 42 self.buf.truncate(0) 43 return ret 44 if size is None: 45 return self._data() 46 47 while self.buf.tell() < size: 48 data = self._data() 49 if not len(data): 50 ret = self.buf.getvalue() 51 self.buf.truncate(0) 52 return ret 53 self.buf.write(data) 54 55 data = self.buf.getvalue() 56 self.buf.truncate(0) 57 self.buf.write(data[size:]) 58 return data[:size]
59
60 - def unread(self, data):
61 self.buf.seek(0, os.SEEK_END) 62 self.buf.write(data)
63
64 -class ChunkedReader(object):
65 - def __init__(self, req, unreader):
66 self.unreader = unreader 67 self.req = req 68 self.parser = self.parse_chunked(unreader) 69 self.buf = StringIO()
70
71 - def read(self, size):
72 if not isinstance(size, (int, long)): 73 raise TypeError("size must be an integral type") 74 if size <= 0: 75 raise ValueError("Size must be positive.") 76 if size == 0: 77 return "" 78 79 if self.parser: 80 while self.buf.tell() < size: 81 try: 82 self.buf.write(self.parser.next()) 83 except StopIteration: 84 self.parser = None 85 break 86 87 data = self.buf.getvalue() 88 ret, rest = data[:size], data[size:] 89 self.buf.truncate(0) 90 self.buf.write(rest) 91 return ret
92
93 - def parse_trailers(self, unreader, data, eof=False):
94 buf = StringIO() 95 buf.write(data) 96 97 idx = buf.getvalue().find("\r\n\r\n") 98 done = buf.getvalue()[:2] == "\r\n" 99 100 while idx < 0 and not done: 101 self.get_data(unreader, buf) 102 idx = buf.getvalue().find("\r\n\r\n") 103 done = buf.getvalue()[:2] == "\r\n" 104 if done: 105 unreader.unread(buf.getvalue()[2:]) 106 return "" 107 self.req.trailers = self.req.parse_headers(buf.getvalue()[:idx]) 108 unreader.unread(buf.getvalue()[idx+4:])
109
110 - def parse_chunked(self, unreader):
111 (size, rest) = self.parse_chunk_size(unreader) 112 while size > 0: 113 while size > len(rest): 114 size -= len(rest) 115 yield rest 116 rest = unreader.read() 117 if not rest: 118 raise NoMoreData() 119 yield rest[:size] 120 # Remove \r\n after chunk 121 rest = rest[size:] 122 while len(rest) < 2: 123 rest += unreader.read() 124 if rest[:2] != '\r\n': 125 raise ChunkMissingTerminator(rest[:2]) 126 (size, rest) = self.parse_chunk_size(unreader, data=rest[2:])
127
128 - def parse_chunk_size(self, unreader, data=None):
129 buf = StringIO() 130 if data is not None: 131 buf.write(data) 132 133 idx = buf.getvalue().find("\r\n") 134 while idx < 0: 135 self.get_data(unreader, buf) 136 idx = buf.getvalue().find("\r\n") 137 138 data = buf.getvalue() 139 line, rest_chunk = data[:idx], data[idx+2:] 140 141 chunk_size = line.split(";", 1)[0].strip() 142 try: 143 chunk_size = int(chunk_size, 16) 144 except ValueError: 145 raise InvalidChunkSize(chunk_size) 146 147 if chunk_size == 0: 148 try: 149 self.parse_trailers(unreader, rest_chunk) 150 except NoMoreData: 151 pass 152 return (0, None) 153 return (chunk_size, rest_chunk)
154
155 - def get_data(self, unreader, buf):
156 data = unreader.read() 157 if not data: 158 raise NoMoreData() 159 buf.write(data)
160
161 -class LengthReader(object):
162 - def __init__(self, req, unreader, length):
163 self.req = req 164 self.unreader = unreader 165 self.length = length
166
167 - def read(self, size):
168 if not isinstance(size, (int, long)): 169 raise TypeError("size must be an integral type") 170 171 size = min(self.length, size) 172 if size < 0: 173 raise ValueError("Size must be positive.") 174 if size == 0: 175 return "" 176 177 buf = StringIO() 178 data = self.unreader.read() 179 while data: 180 buf.write(data) 181 if buf.tell() >= size: 182 break 183 data = self.unreader.read() 184 185 186 buf = buf.getvalue() 187 ret, rest = buf[:size], buf[size:] 188 self.unreader.unread(rest) 189 self.length -= size 190 return ret
191
192 -class EOFReader(object):
193 - def __init__(self, req, unreader):
194 self.req = req 195 self.unreader = unreader 196 self.buf = StringIO() 197 self.finished = False
198
199 - def read(self, size):
200 if not isinstance(size, (int, long)): 201 raise TypeError("size must be an integral type") 202 if size < 0: 203 raise ValueError("Size must be positive.") 204 if size == 0: 205 return "" 206 207 if self.finished: 208 data = self.buf.getvalue() 209 ret, rest = data[:size], data[size:] 210 self.buf.truncate(0) 211 self.buf.write(rest) 212 return ret 213 214 data = self.unreader.read() 215 while data: 216 self.buf.write(data) 217 if self.buf.tell() > size: 218 break 219 data = self.unreader.read() 220 221 if not data: 222 self.finished = True 223 224 data = self.buf.getvalue() 225 ret, rest = data[:size], data[size:] 226 self.buf.truncate(0) 227 self.buf.write(rest) 228 return ret
229
230 -class Body(object):
231 - def __init__(self, reader):
232 self.reader = reader 233 self.buf = StringIO() 234 self.closed = False
235
236 - def __iter__(self):
237 return self
238
239 - def next(self):
240 ret = self.readline() 241 if not ret: 242 raise StopIteration() 243 return ret
244
245 - def discard(self):
246 data = self.read(8192) 247 while data: 248 data = self.read()
249
250 - def getsize(self, size):
251 if size is None: 252 return sys.maxint 253 elif not isinstance(size, (int, long)): 254 raise TypeError("size must be an integral type") 255 elif size < 0: 256 return sys.maxint 257 return size
258
259 - def read(self, size=None):
260 size = self.getsize(size) 261 if size == 0: 262 return "" 263 264 if size < self.buf.tell(): 265 data = self.buf.getvalue() 266 ret, rest = data[:size], data[size:] 267 self.buf.truncate(0) 268 self.buf.write(rest) 269 return ret 270 271 while size > self.buf.tell(): 272 data = self.reader.read(1024) 273 if not len(data): 274 self.closed = True 275 break 276 self.buf.write(data) 277 278 data = self.buf.getvalue() 279 ret, rest = data[:size], data[size:] 280 self.buf.truncate(0) 281 self.buf.write(rest) 282 return ret
283
284 - def readline(self, size=None):
285 size = self.getsize(size) 286 if size == 0: 287 return "" 288 289 line = self.buf.getvalue() 290 idx = line.find("\n") 291 if idx >= 0: 292 ret = line[:idx+1] 293 self.buf.truncate(0) 294 self.buf.write(line[idx+1:]) 295 return ret 296 297 self.buf.truncate(0) 298 ch = "" 299 buf = [line] 300 lsize = len(line) 301 while lsize < size and ch != "\n": 302 ch = self.reader.read(1) 303 if not len(ch): 304 self.closed = True 305 break 306 lsize += 1 307 buf.append(ch) 308 return "".join(buf)
309
310 - def readlines(self, size=None):
311 ret = [] 312 data = self.read() 313 while len(data): 314 pos = data.find("\n") 315 if pos < 0: 316 ret.append(data) 317 data = "" 318 else: 319 line, data = data[:pos+1], data[pos+1:] 320 ret.append(line) 321 return ret
322 323
324 -class GzipBody(Body):
325 - def __init__(self, reader):
326 super(GzipBody, self).__init__(reader) 327 self._d = zlib.decompressobj(16+zlib.MAX_WBITS)
328
329 - def _decompress(self, data):
330 return self._d.decompress(data)
331
332 - def read(self, size=None):
333 size = self.getsize(size) 334 if size == 0: 335 return "" 336 337 if size < self.buf.tell(): 338 data = self.buf.getvalue() 339 ret, rest = data[:size], data[size:] 340 self.buf.truncate(0) 341 self.buf.write(rest) 342 return self._decompress(ret) 343 344 while size > self.buf.tell(): 345 data = self.reader.read(1024) 346 if not len(data): 347 break 348 self.buf.write(data) 349 350 data = self.buf.getvalue() 351 ret, rest = data[:size], data[size:] 352 self.buf.truncate(0) 353 self.buf.write(rest) 354 return self._decompress(ret)
355
356 - def readline(self, size=None):
357 size = self.getsize(size) 358 if size == 0: 359 return "" 360 361 idx = self.buf.getvalue().find("\n") 362 while idx < 0: 363 data = self.reader.read(1024) 364 if not len(data): 365 break 366 self.buf.write(self._decompress(data)) 367 idx = self.buf.getvalue().find("\n") 368 if size < self.buf.tell(): 369 break 370 371 # If we didn't find it, and we got here, we've 372 # exceeded size or run out of data. 373 if idx < 0: 374 rlen = min(size, self.buf.tell()) 375 else: 376 rlen = idx + 1 377 378 # If rlen is beyond our size threshold, trim back 379 if rlen > size: 380 rlen = size 381 382 data = self.buf.getvalue() 383 ret, rest = data[:rlen], data[rlen:] 384 385 self.buf.truncate(0) 386 self.buf.write(rest) 387 return ret
388 389
390 -class DeflateBody(GzipBody):
391 - def __init__(self, reader):
392 super(DeflateBody, self).__init__(reader) 393 self._d = zlib.decompressobj()
394 395
396 -class Request(object):
397 - def __init__(self, unreader, decompress=True):
398 self.unreader = unreader 399 self.version = None 400 self.headers = MultiDict() 401 self.trailers = [] 402 self.body = None 403 self.encoding = None 404 self.status = None 405 self.reason = None 406 self.status_int = None 407 self.decompress = decompress 408 409 self.versre = re.compile("HTTP/(\d+).(\d+)") 410 self.stare = re.compile("(\d{3})\s*(\w*)") 411 self.hdrre = re.compile("[\x00-\x1F\x7F()<>@,;:\[\]={} \t\\\\\"]") 412 413 unused = self.parse(self.unreader) 414 self.unreader.unread(unused) 415 self.set_body_reader()
416
417 - def get_data(self, unreader, buf, stop=False):
418 data = unreader.read() 419 if not data: 420 if stop: 421 raise StopIteration() 422 raise NoMoreData(buf.getvalue()) 423 buf.write(data)
424
425 - def parse(self, unreader):
426 buf = StringIO() 427 428 self.get_data(unreader, buf, stop=True) 429 430 # Request line 431 idx = buf.getvalue().find("\r\n") 432 while idx < 0: 433 self.get_data(unreader, buf) 434 idx = buf.getvalue().find("\r\n") 435 self.parse_first_line(buf.getvalue()[:idx]) 436 rest = buf.getvalue()[idx+2:] # Skip \r\n 437 buf.truncate(0) 438 buf.write(rest) 439 440 # Headers 441 idx = buf.getvalue().find("\r\n\r\n") 442 done = buf.getvalue()[:2] == "\r\n" 443 while idx < 0 and not done: 444 self.get_data(unreader, buf) 445 idx = buf.getvalue().find("\r\n\r\n") 446 done = buf.getvalue()[:2] == "\r\n" 447 if done: 448 self.unreader.unread(buf.getvalue()[2:]) 449 return "" 450 451 self.headers = self.parse_headers(buf.getvalue()[:idx]) 452 453 ret = buf.getvalue()[idx+4:] 454 buf.truncate(0) 455 return ret
456
457 - def parse_first_line(self, line):
458 bits = line.split(None, 1) 459 if len(bits) != 2: 460 raise InvalidRequestLine(line) 461 462 # version 463 matchv = self.versre.match(bits[0]) 464 if matchv is None: 465 raise InvalidHTTPVersion(bits[0]) 466 self.version = (int(matchv.group(1)), int(matchv.group(2))) 467 468 # status 469 matchs = self.stare.match(bits[1]) 470 if matchs is None: 471 raise InvalidHTTPStatus(bits[1]) 472 473 self.status = bits[1] 474 self.status_int = int(matchs.group(1)) 475 self.reason = matchs.group(2)
476
477 - def parse_headers(self, data):
478 headers = MultiDict() 479 480 # Split lines on \r\n keeping the \r\n on each line 481 lines = [line + "\r\n" for line in data.split("\r\n")] 482 483 # Parse headers into key/value pairs paying attention 484 # to continuation lines. 485 while len(lines): 486 # Parse initial header name : value pair. 487 curr = lines.pop(0) 488 if curr.find(":") < 0: 489 raise InvalidHeader(curr.strip()) 490 name, value = curr.split(":", 1) 491 name = name.rstrip(" \t") 492 if self.hdrre.search(name.upper()): 493 raise InvalidHeaderName(name) 494 name, value = name.strip(), [value.lstrip()] 495 496 # Consume value continuation lines 497 while len(lines) and lines[0].startswith((" ", "\t")): 498 value.append(lines.pop(0)) 499 value = ''.join(value).rstrip() 500 501 headers.add(name, value) 502 return headers
503
504 - def set_body_reader(self):
505 clen = self.headers.iget('content-length') 506 te = self.headers.iget('transfer-encoding') 507 encoding = self.headers.iget('content-encoding') 508 509 chunked = False 510 clength = None 511 if clen is not None: 512 try: 513 clength = int(clen) 514 except ValueError: 515 pass 516 elif te is not None: 517 chunked = te.lower() == "chunked" 518 519 if encoding: 520 self.encoding = encoding.lower() 521 522 if chunked: 523 reader = ChunkedReader(self, self.unreader) 524 elif clength is not None: 525 reader = LengthReader(self, self.unreader, clength) 526 else: 527 reader = EOFReader(self, self.unreader) 528 529 if self.decompress and self.encoding in ('gzip', 'deflate',): 530 if self.encoding == "gzip": 531 self.body = GzipBody(reader) 532 else: 533 self.body = DeflateBody(reader) 534 else: 535 self.body = Body(reader)
536
537 - def should_close(self):
538 connection = self.headers.iget("connection") 539 540 if connection is not None: 541 if connection.lower().strip() == "close": 542 return True 543 elif connection.lower().strip() == "keep-alive": 544 return False 545 return self.version <= (1, 0)
546