wtf.impl.http.

1 # -*- coding: ascii -*- 2 # 3 # Copyright 2006-2012 4 # Andr\xe9 Malo or his licensors, as applicable 5 # 6 # Licensed under the Apache License, Version 2.0 (the "License"); 7 # you may not use this file except in compliance with the License. 8 # You may obtain a copy of the License at 9 # 10 # http://www.apache.org/licenses/LICENSE-2.0 11 # 12 # Unless required by applicable law or agreed to in writing, software 13 # distributed under the License is distributed on an "AS IS" BASIS, 14 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 15 # See the License for the specific language governing permissions and 16 # limitations under the License. 17 """ 18 HTTP utilities 19 ============== 20 21 This module implements various tools for HTTP request/response handling. 22 23 :Variables: 24 - `CR`: ASCII CR byte (\\r) 25 - `LF`: ASCII LF byte (\\n) 26 - `CRLF`: ASCII CRLF sequence (\\r\\n) 27 28 :Types: 29 - `CR`: ``str`` 30 - `LF`: ``str`` 31 - `CRLF`: ``str`` 32 """ 33 __author__ = u"Andr\xe9 Malo" 34 __docformat__ = "restructuredtext en" 35 36 import errno as _errno 37 import socket as _socket 38 39 from wtf import Error 40 from wtf import httputil as _httputil 41 from wtf import stream as _stream 42 43 CR, LF, CRLF = _httputil.CR, _httputil.LF, _httputil.CRLF 44 make_date = _httputil.make_date 45 46

47 -class ParseError(Error):

48 """ 49 Request parse error 50 51 :CVariables: 52 - `status`: Statuscode and message 53 54 :Types: 55 - `status`: ``str`` 56 """ 57 status = "500 Internal Server Error" 58

59 - def __init__(self, message):

60 """ 61 Initialization 62 63 :Parameters: 64 - `message`: Message to emit 65 66 :Types: 67 - `message`: ``str`` 68 """ 69 Error.__init__(self, message) 70 self.msg = message

71 72

73 -class BadRequest(ParseError):

74 """ Bad Request error class """ 75 status = "400 Bad Request"

76

77 -class InvalidHeaderLine(BadRequest):

78 """ A header line is invalid """

79

80 -class IncompleteHeaders(BadRequest):

81 """ The headers are incomplete """

82

83 -class InvalidChunkHeader(BadRequest):

84 """ Chunk header was bogus """

85

86 -class IncompleteChunk(BadRequest):

87 """ Chunk did not fully arrive """

88 89

90 -def read_headers(stream):

91 """ 92 Read MIME headers from stream 93 94 :Parameters: 95 - `stream`: The stream to read from 96 97 :Types: 98 - `stream`: ``file`` 99 100 :return: Dictionary of comma folded headers 101 :rtype: ``dict`` 102 103 :Exceptions: 104 - `IncompleteHeaders`: The stream ended before the final empty line 105 - `InvalidHeaderLine`: Unparsable header line 106 """ 107 try: 108 return dict((name, ", ".join(values)) 109 for name, values in _httputil.read_headers(stream).iteritems()) 110 except _httputil.IncompleteHeaders, e: 111 raise IncompleteHeaders(str(e)) 112 except _httputil.InvalidHeaderLine, e: 113 raise InvalidHeaderLine(str(e))

114 115

116 -class ChunkedWriter(object):

117 """ 118 Chunked transfer encoding encoder 119 120 :Ivariables: 121 - `_stream`: The stream to write the chunks to 122 123 :Types: 124 - `_stream`: ``file`` 125 """ 126

127 - def __init__(self, stream):

128 """ 129 Initialization 130 131 :Parameters: 132 - `stream`: The stream to write the chunks to 133 134 :Types: 135 - `stream`: ``file`` 136 """ 137 self._stream = stream

138

139 - def __getattr__(self, name):

140 """ 141 Delegate undefined attribute requests to the underlying stream 142 143 :Parameters: 144 - `name`: Attribute name 145 146 :Types: 147 - `name`: ``str`` 148 149 :return: The requested attribute 150 :rtype: any 151 152 :Exceptions: 153 - `AttributeError`: Attribute not found 154 """ 155 return getattr(self._stream, name)

156

157 - def write(self, data, _force_empty=False):

158 """ 159 Write a chunk of data 160 161 :Parameters: 162 - `data`: The chunk of data to write 163 - `_force_empty`: Write the chunk even if data is empty 164 (this will be - by definition - the last chunk) 165 166 :Types: 167 - `data`: ``str`` 168 - `_force_empty`: ``bool`` 169 """ 170 data = str(data) 171 if data or _force_empty: 172 self._stream.write('%X' % len(data) + CRLF) 173 self._stream.write(data + CRLF)

174

175 - def close(self):

176 """ 177 Finish chunked writing 178 179 This writes the last (empty) chunk and closes the stream afterwards. 180 """ 181 self.write("", True) 182 try: 183 self._stream.close() 184 except _socket.error, e: 185 if e[0] != _errno.EPIPE: 186 raise

187 188

189 -class ChunkedReader(object):

190 """ 191 Chunked transfer encoding decoder 192 193 :IVariables: 194 - `_stream`: The stream to decode 195 - `_state`: Current read function 196 - `_left`: Bytes left of the current chunk 197 198 :Types: 199 - `_stream`: ``file`` 200 - `_state`: ``callable`` 201 - `_left`: ``int`` 202 """ 203

204 - def __init__(self, stream):

205 """ 206 Initialization 207 208 :Parameters: 209 - `stream`: The stream to decode 210 211 :Types: 212 - `stream`: ``file`` 213 """ 214 self._stream = stream 215 self._state = self._read_header 216 self._left = 0

217

218 - def read(self, size):

219 """ 220 Read (at max) ``size`` bytes from the stream 221 222 This just calls the current state reader and returns its result. 223 224 :Parameters: 225 - `size`: The maximum number of bytes to read (>0) 226 227 :Types: 228 - `size`: ``int`` 229 230 :return: Bytes read (empty on EOF) 231 :rtype: ``str`` 232 233 :Exceptions: 234 - `InvalidChunkHeader`: A chunk header was unparsable 235 - `InvalidHeaderLine`: A header line (of the trailer) could not be 236 parsed 237 - `IncompleteHeaders`: The stream ended unexpectedly while parsing 238 headers (of the trailer) 239 - `IncompleteChunk`: The stream ended unexpectedly in the middle of 240 a chunk 241 """ 242 assert size > 0, "size must be > 0" 243 return self._state(size)

244

245 - def _done(self, size):

246 """ 247 Final state: Hit EOF (or EOS, rather) 248 249 This function always returns an empty string, so `size` is ignored. 250 251 :Parameters: 252 - `size`: The maximum number of bytes to read (>0) 253 254 :Types: 255 - `size`: ``int`` 256 257 :return: Bytes read (empty on EOF) 258 :rtype: ``str`` 259 """ 260 # pylint: disable = W0613 261 262 return ""

263

264 - def _read_header(self, size):

265 """ 266 Initial State: Read chunk header and follow up the stream 267 268 :Parameters: 269 - `size`: The maximum number of bytes to read (>0) 270 271 :Types: 272 - `size`: ``int`` 273 274 :return: Bytes read (empty on EOF) 275 :rtype: ``str`` 276 277 :Exceptions: 278 - `InvalidChunkHeader`: The chunk header was unparsable 279 """ 280 line = self._stream.readline().split(';', 1)[0].strip() 281 try: 282 chunksize = int(line, 16) 283 if chunksize < 0: 284 raise ValueError() 285 except (TypeError, ValueError): 286 raise InvalidChunkHeader("Unparsable chunksize") 287 if chunksize == 0: 288 self._state = self._read_trailer 289 else: 290 self._left, self._state = chunksize, self._read_data 291 return self.read(size)

292

293 - def _read_trailer(self, size):

294 """ 295 After last chunk state: Read trailer and finish 296 297 :Parameters: 298 - `size`: The maximum number of bytes to read (>0) 299 300 :Types: 301 - `size`: ``int`` 302 303 :return: Bytes read (empty on EOF) 304 :rtype: ``str`` 305 306 :Exceptions: 307 - `InvalidHeaderLine`: A header line could not be parsed 308 - `IncompleteHeaders`: The stream ended while parsing headers 309 """ 310 # We could add the headers to the request object, but 311 # nobody reads them, after the body was fetched. So, why bother? 312 read_headers(self._stream) 313 self._state = self._done 314 return self.read(size)

315

316 - def _read_data(self, size):

317 """ 318 Read actual chunk data 319 320 :Parameters: 321 - `size`: The maximum number of bytes to read (>0) 322 323 :Types: 324 - `size`: ``int`` 325 326 :return: Bytes read (empty on EOF) 327 :rtype: ``str`` 328 329 :Exceptions: 330 - `IncompleteChunk`: The stream ended unexpectedly in the middle of 331 the chunk 332 """ 333 rsize = min(self._left, size) 334 if rsize <= 0: 335 self._state = self._read_suffix 336 return self.read(size) 337 result = _stream.read_exact(self._stream, rsize) 338 if not result: 339 raise IncompleteChunk("Missing at least %d bytes" % self._left) 340 self._left -= len(result) 341 if self._left <= 0: 342 self._state = self._read_suffix 343 return result

344

345 - def _read_suffix(self, size):

346 """ 347 Read trailing CRLF after chunk data 348 349 After that it switches back to `_read_header` and follows up on the 350 stream. 351 352 :Parameters: 353 - `size`: The maximum number of bytes to read (>0) 354 355 :Types: 356 - `size`: ``int`` 357 358 :return: Bytes read (empty on EOF) 359 :rtype: ``str`` 360 361 :Exceptions: 362 - `IncompleteChunk`: Trailing CRLF was missing (or something else) 363 """ 364 eol = self._stream.read(1) 365 if eol == CR: 366 eol = self._stream.read(1) 367 if eol != LF: 368 raise IncompleteChunk("Missing trailing CRLF") 369 self._state = self._read_header 370 return self.read(size)

371 372

373 -class ExpectationReader(object):

374 """ 375 Before actual reading send a 100 continue intermediate response 376 377 :IVariables: 378 - `_stream`: The stream to read from 379 - `_request`: Request instance 380 381 :Types: 382 - `_stream`: ``file`` 383 - `_request`: `HTTPRequest` 384 """ 385

386 - def __init__(self, stream, request):

387 """ 388 Initialization 389 390 :Parameters: 391 - `stream`: The stream to actually read data from 392 - `request`: Request instance 393 394 :Types: 395 - `stream`: ``file`` 396 - `request`: `HTTPRequest` 397 """ 398 self._stream, self._request = stream, request 399 self.read = self._send_continue

400

401 - def _send_continue(self, size):

402 """ 403 Send continue before reading 404 405 :Parameters: 406 - `size`: Read at max `size` bytes 407 408 :Types: 409 - `size`: ``int`` 410 411 :return: The read bytes, or empty on EOF 412 :rtype: ``str`` 413 """ 414 if self.read == self._send_continue: 415 self.read = self._stream.read 416 self._request.send_continue() 417 return self.read(size)

418 419

420 -class HTTPConnection(object):

421 """ 422 HTTP connection abstraction 423 424 :CVariables: 425 - `DROPPERS`: List of status codes which drop the connection 426 - `KEEPERS`: List of status codes which keep the connection without 427 content length 428 429 :IVariables: 430 - `_request`: The request instance 431 - `persist`: Does the connection persist? This property may be queried 432 on a higher level in orer to determine whether to close a connection 433 or not. 434 - `reader`: Reading stream for this connection 435 - `writer`: Writing stream for this connection 436 - `headers`: Header dictionary to add to the outgoing headers. This 437 MAY contain ``Connection`` and ``Transfer-Encoding`` headers. 438 439 :Types: 440 - `DROPPERS`: ``dict`` 441 - `KEEPERS`: ``dict`` 442 - `_request`: `HTTPRequest` 443 - `persist`: ``bool`` 444 - `reader`: `wtf.stream.GenericStream` 445 - `writer`: `wtf.stream.GenericStream` 446 - `headers`: ``dict`` 447 """ 448 reader, writer, persist = None, None, False 449 DROPPERS = dict.fromkeys((400, 405, 408, 411, 413, 414, 500, 501, 503)) 450 KEEPERS = dict.fromkeys((204, 205, 304)) 451

452 - def __init__(self, request, connection):

453 """ 454 Initialization 455 456 :Parameters: 457 - `request`: The request instance 458 - `connection`: The connection instance 459 460 :Types: 461 - `request`: `HTTPRequest` 462 - `connection`: `wtf.impl.http.Connection` 463 """ 464 self._request = request 465 self.reader = connection.reader() 466 self.writer = connection.writer() 467 self.headers = {'connection': 'close'} 468 self.settimeout = connection.settimeout

469

470 - def __del__(self):

471 self.close()

472

473 - def close(self):

474 """ Close the HTTP streams """ 475 try: 476 try: 477 reader, self.reader = self.reader, None 478 if reader is not None: 479 reader.close() 480 finally: 481 writer, self.writer = self.writer, None 482 if writer is not None: 483 writer.close() 484 except _socket.error, e: 485 if e[0] != _errno.EPIPE: 486 raise

487

488 - def compute_status(self):

489 """ 490 Compute the connection persistance status 491 492 This function "only" has side effects. It sets ``headers`` and 493 ``persist`` according to request and response parameters. 494 """ 495 # pylint: disable = R0912 496 497 request = self._request 498 protocol = request.protocol 499 if protocol < (1, 0): 500 self.persist = False 501 self.headers = {} 502 else: 503 headers_out = request.response_headers 504 if 'keep-alive' in headers_out: 505 del headers_out['keep-alive'] 506 if 'connection' in headers_out: 507 rtokens = [item.strip().lower() 508 for item in headers_out['connection'].split(',')] 509 for token in rtokens: 510 if token in headers_out: 511 del headers_out[token] 512 del headers_out['connection'] 513 if 'transfer-encoding' in headers_out: 514 del headers_out['transfer-encoding'] 515 516 if 'connection' in request.headers: 517 tokens = [item.strip().lower() 518 for item in request.headers['connection'].split(',')] 519 else: 520 tokens = () 521 status = request.response_status 522 if not self._request.keep_alive or \ 523 (status is not None and status in self.DROPPERS): 524 self.persist = False 525 if protocol >= (1, 1) or ( 526 protocol == (1, 0) and 'keep-alive' in tokens): 527 self.headers = {'connection': 'close'} 528 else: 529 if 'close' in tokens: 530 self.persist = False 531 if protocol >= (1, 1): 532 self.headers = {'connection': 'close'} 533 elif protocol == (1, 0): 534 if 'keep-alive' in tokens: 535 self.persist = True 536 self.headers = {'connection': 'Keep-Alive'} 537 else: 538 self.persist = False 539 self.headers = {} 540 elif protocol >= (1, 1): 541 self.persist = True 542 self.headers = {} 543 544 if 'content-length' not in headers_out and \ 545 status not in self.KEEPERS and request.method != 'HEAD': 546 if protocol == (1, 0): 547 self.persist = False 548 if 'keep-alive' in tokens: 549 self.headers = {'connection': 'close'} 550 else: 551 self.headers = {} 552 else: 553 self.headers['transfer-encoding'] = 'chunked'

554

555 - def settimeout(self, timeout): # pylint: disable = E0202

556 """ 557 Set a socket timeout for next operations 558 559 :Parameters: 560 - `timeout`: Socket timeout to set 561 562 :Types: 563 - `timeout`: ``float`` 564 """ 565 # pylint: disable = W0613 566 567 raise AssertionError("Object not initialized properly")

568

Source Code for Module wtf.impl.http._util