Hide keyboard shortcuts

Hot-keys on this page

r m x p   toggle line displays

j k   next/prev highlighted chunk

0   (zero) top of page

1   (one) first highlighted chunk

1from __future__ import absolute_import 

2 

3import email 

4import logging 

5import re 

6import time 

7import warnings 

8from collections import namedtuple 

9from itertools import takewhile 

10 

11from ..exceptions import ( 

12 ConnectTimeoutError, 

13 InvalidHeader, 

14 MaxRetryError, 

15 ProtocolError, 

16 ProxyError, 

17 ReadTimeoutError, 

18 ResponseError, 

19) 

20from ..packages import six 

21 

22log = logging.getLogger(__name__) 

23 

24 

25# Data structure for representing the metadata of requests that result in a retry. 

26RequestHistory = namedtuple( 

27 "RequestHistory", ["method", "url", "error", "status", "redirect_location"] 

28) 

29 

30 

31# TODO: In v2 we can remove this sentinel and metaclass with deprecated options. 

32_Default = object() 

33 

34 

35class _RetryMeta(type): 

36 @property 

37 def DEFAULT_METHOD_WHITELIST(cls): 

38 warnings.warn( 

39 "Using 'Retry.DEFAULT_METHOD_WHITELIST' is deprecated and " 

40 "will be removed in v2.0. Use 'Retry.DEFAULT_METHODS_ALLOWED' instead", 

41 DeprecationWarning, 

42 ) 

43 return cls.DEFAULT_ALLOWED_METHODS 

44 

45 @DEFAULT_METHOD_WHITELIST.setter 

46 def DEFAULT_METHOD_WHITELIST(cls, value): 

47 warnings.warn( 

48 "Using 'Retry.DEFAULT_METHOD_WHITELIST' is deprecated and " 

49 "will be removed in v2.0. Use 'Retry.DEFAULT_ALLOWED_METHODS' instead", 

50 DeprecationWarning, 

51 ) 

52 cls.DEFAULT_ALLOWED_METHODS = value 

53 

54 @property 

55 def DEFAULT_REDIRECT_HEADERS_BLACKLIST(cls): 

56 warnings.warn( 

57 "Using 'Retry.DEFAULT_REDIRECT_HEADERS_BLACKLIST' is deprecated and " 

58 "will be removed in v2.0. Use 'Retry.DEFAULT_REMOVE_HEADERS_ON_REDIRECT' instead", 

59 DeprecationWarning, 

60 ) 

61 return cls.DEFAULT_REMOVE_HEADERS_ON_REDIRECT 

62 

63 @DEFAULT_REDIRECT_HEADERS_BLACKLIST.setter 

64 def DEFAULT_REDIRECT_HEADERS_BLACKLIST(cls, value): 

65 warnings.warn( 

66 "Using 'Retry.DEFAULT_REDIRECT_HEADERS_BLACKLIST' is deprecated and " 

67 "will be removed in v2.0. Use 'Retry.DEFAULT_REMOVE_HEADERS_ON_REDIRECT' instead", 

68 DeprecationWarning, 

69 ) 

70 cls.DEFAULT_REMOVE_HEADERS_ON_REDIRECT = value 

71 

72 

73@six.add_metaclass(_RetryMeta) 

74class Retry(object): 

75 """Retry configuration. 

76 

77 Each retry attempt will create a new Retry object with updated values, so 

78 they can be safely reused. 

79 

80 Retries can be defined as a default for a pool:: 

81 

82 retries = Retry(connect=5, read=2, redirect=5) 

83 http = PoolManager(retries=retries) 

84 response = http.request('GET', 'http://example.com/') 

85 

86 Or per-request (which overrides the default for the pool):: 

87 

88 response = http.request('GET', 'http://example.com/', retries=Retry(10)) 

89 

90 Retries can be disabled by passing ``False``:: 

91 

92 response = http.request('GET', 'http://example.com/', retries=False) 

93 

94 Errors will be wrapped in :class:`~urllib3.exceptions.MaxRetryError` unless 

95 retries are disabled, in which case the causing exception will be raised. 

96 

97 :param int total: 

98 Total number of retries to allow. Takes precedence over other counts. 

99 

100 Set to ``None`` to remove this constraint and fall back on other 

101 counts. 

102 

103 Set to ``0`` to fail on the first retry. 

104 

105 Set to ``False`` to disable and imply ``raise_on_redirect=False``. 

106 

107 :param int connect: 

108 How many connection-related errors to retry on. 

109 

110 These are errors raised before the request is sent to the remote server, 

111 which we assume has not triggered the server to process the request. 

112 

113 Set to ``0`` to fail on the first retry of this type. 

114 

115 :param int read: 

116 How many times to retry on read errors. 

117 

118 These errors are raised after the request was sent to the server, so the 

119 request may have side-effects. 

120 

121 Set to ``0`` to fail on the first retry of this type. 

122 

123 :param int redirect: 

124 How many redirects to perform. Limit this to avoid infinite redirect 

125 loops. 

126 

127 A redirect is a HTTP response with a status code 301, 302, 303, 307 or 

128 308. 

129 

130 Set to ``0`` to fail on the first retry of this type. 

131 

132 Set to ``False`` to disable and imply ``raise_on_redirect=False``. 

133 

134 :param int status: 

135 How many times to retry on bad status codes. 

136 

137 These are retries made on responses, where status code matches 

138 ``status_forcelist``. 

139 

140 Set to ``0`` to fail on the first retry of this type. 

141 

142 :param int other: 

143 How many times to retry on other errors. 

144 

145 Other errors are errors that are not connect, read, redirect or status errors. 

146 These errors might be raised after the request was sent to the server, so the 

147 request might have side-effects. 

148 

149 Set to ``0`` to fail on the first retry of this type. 

150 

151 If ``total`` is not set, it's a good idea to set this to 0 to account 

152 for unexpected edge cases and avoid infinite retry loops. 

153 

154 :param iterable allowed_methods: 

155 Set of uppercased HTTP method verbs that we should retry on. 

156 

157 By default, we only retry on methods which are considered to be 

158 idempotent (multiple requests with the same parameters end with the 

159 same state). See :attr:`Retry.DEFAULT_ALLOWED_METHODS`. 

160 

161 Set to a ``False`` value to retry on any verb. 

162 

163 .. warning:: 

164 

165 Previously this parameter was named ``method_whitelist``, that 

166 usage is deprecated in v1.26.0 and will be removed in v2.0. 

167 

168 :param iterable status_forcelist: 

169 A set of integer HTTP status codes that we should force a retry on. 

170 A retry is initiated if the request method is in ``allowed_methods`` 

171 and the response status code is in ``status_forcelist``. 

172 

173 By default, this is disabled with ``None``. 

174 

175 :param float backoff_factor: 

176 A backoff factor to apply between attempts after the second try 

177 (most errors are resolved immediately by a second try without a 

178 delay). urllib3 will sleep for:: 

179 

180 {backoff factor} * (2 ** ({number of total retries} - 1)) 

181 

182 seconds. If the backoff_factor is 0.1, then :func:`.sleep` will sleep 

183 for [0.0s, 0.2s, 0.4s, ...] between retries. It will never be longer 

184 than :attr:`Retry.BACKOFF_MAX`. 

185 

186 By default, backoff is disabled (set to 0). 

187 

188 :param bool raise_on_redirect: Whether, if the number of redirects is 

189 exhausted, to raise a MaxRetryError, or to return a response with a 

190 response code in the 3xx range. 

191 

192 :param bool raise_on_status: Similar meaning to ``raise_on_redirect``: 

193 whether we should raise an exception, or return a response, 

194 if status falls in ``status_forcelist`` range and retries have 

195 been exhausted. 

196 

197 :param tuple history: The history of the request encountered during 

198 each call to :meth:`~Retry.increment`. The list is in the order 

199 the requests occurred. Each list item is of class :class:`RequestHistory`. 

200 

201 :param bool respect_retry_after_header: 

202 Whether to respect Retry-After header on status codes defined as 

203 :attr:`Retry.RETRY_AFTER_STATUS_CODES` or not. 

204 

205 :param iterable remove_headers_on_redirect: 

206 Sequence of headers to remove from the request when a response 

207 indicating a redirect is returned before firing off the redirected 

208 request. 

209 """ 

210 

211 #: Default methods to be used for ``allowed_methods`` 

212 DEFAULT_ALLOWED_METHODS = frozenset( 

213 ["HEAD", "GET", "PUT", "DELETE", "OPTIONS", "TRACE"] 

214 ) 

215 

216 #: Default status codes to be used for ``status_forcelist`` 

217 RETRY_AFTER_STATUS_CODES = frozenset([413, 429, 503]) 

218 

219 #: Default headers to be used for ``remove_headers_on_redirect`` 

220 DEFAULT_REMOVE_HEADERS_ON_REDIRECT = frozenset(["Authorization"]) 

221 

222 #: Maximum backoff time. 

223 BACKOFF_MAX = 120 

224 

225 def __init__( 

226 self, 

227 total=10, 

228 connect=None, 

229 read=None, 

230 redirect=None, 

231 status=None, 

232 other=None, 

233 allowed_methods=_Default, 

234 status_forcelist=None, 

235 backoff_factor=0, 

236 raise_on_redirect=True, 

237 raise_on_status=True, 

238 history=None, 

239 respect_retry_after_header=True, 

240 remove_headers_on_redirect=_Default, 

241 # TODO: Deprecated, remove in v2.0 

242 method_whitelist=_Default, 

243 ): 

244 

245 if method_whitelist is not _Default: 

246 if allowed_methods is not _Default: 

247 raise ValueError( 

248 "Using both 'allowed_methods' and " 

249 "'method_whitelist' together is not allowed. " 

250 "Instead only use 'allowed_methods'" 

251 ) 

252 warnings.warn( 

253 "Using 'method_whitelist' with Retry is deprecated and " 

254 "will be removed in v2.0. Use 'allowed_methods' instead", 

255 DeprecationWarning, 

256 stacklevel=2, 

257 ) 

258 allowed_methods = method_whitelist 

259 if allowed_methods is _Default: 

260 allowed_methods = self.DEFAULT_ALLOWED_METHODS 

261 if remove_headers_on_redirect is _Default: 

262 remove_headers_on_redirect = self.DEFAULT_REMOVE_HEADERS_ON_REDIRECT 

263 

264 self.total = total 

265 self.connect = connect 

266 self.read = read 

267 self.status = status 

268 self.other = other 

269 

270 if redirect is False or total is False: 

271 redirect = 0 

272 raise_on_redirect = False 

273 

274 self.redirect = redirect 

275 self.status_forcelist = status_forcelist or set() 

276 self.allowed_methods = allowed_methods 

277 self.backoff_factor = backoff_factor 

278 self.raise_on_redirect = raise_on_redirect 

279 self.raise_on_status = raise_on_status 

280 self.history = history or tuple() 

281 self.respect_retry_after_header = respect_retry_after_header 

282 self.remove_headers_on_redirect = frozenset( 

283 [h.lower() for h in remove_headers_on_redirect] 

284 ) 

285 

286 def new(self, **kw): 

287 params = dict( 

288 total=self.total, 

289 connect=self.connect, 

290 read=self.read, 

291 redirect=self.redirect, 

292 status=self.status, 

293 other=self.other, 

294 status_forcelist=self.status_forcelist, 

295 backoff_factor=self.backoff_factor, 

296 raise_on_redirect=self.raise_on_redirect, 

297 raise_on_status=self.raise_on_status, 

298 history=self.history, 

299 remove_headers_on_redirect=self.remove_headers_on_redirect, 

300 respect_retry_after_header=self.respect_retry_after_header, 

301 ) 

302 

303 # TODO: If already given in **kw we use what's given to us 

304 # If not given we need to figure out what to pass. We decide 

305 # based on whether our class has the 'method_whitelist' property 

306 # and if so we pass the deprecated 'method_whitelist' otherwise 

307 # we use 'allowed_methods'. Remove in v2.0 

308 if "method_whitelist" not in kw and "allowed_methods" not in kw: 

309 if "method_whitelist" in self.__dict__: 

310 warnings.warn( 

311 "Using 'method_whitelist' with Retry is deprecated and " 

312 "will be removed in v2.0. Use 'allowed_methods' instead", 

313 DeprecationWarning, 

314 ) 

315 params["method_whitelist"] = self.allowed_methods 

316 else: 

317 params["allowed_methods"] = self.allowed_methods 

318 

319 params.update(kw) 

320 return type(self)(**params) 

321 

322 @classmethod 

323 def from_int(cls, retries, redirect=True, default=None): 

324 """ Backwards-compatibility for the old retries format.""" 

325 if retries is None: 

326 retries = default if default is not None else cls.DEFAULT 

327 

328 if isinstance(retries, Retry): 

329 return retries 

330 

331 redirect = bool(redirect) and None 

332 new_retries = cls(retries, redirect=redirect) 

333 log.debug("Converted retries value: %r -> %r", retries, new_retries) 

334 return new_retries 

335 

336 def get_backoff_time(self): 

337 """Formula for computing the current backoff 

338 

339 :rtype: float 

340 """ 

341 # We want to consider only the last consecutive errors sequence (Ignore redirects). 

342 consecutive_errors_len = len( 

343 list( 

344 takewhile(lambda x: x.redirect_location is None, reversed(self.history)) 

345 ) 

346 ) 

347 if consecutive_errors_len <= 1: 

348 return 0 

349 

350 backoff_value = self.backoff_factor * (2 ** (consecutive_errors_len - 1)) 

351 return min(self.BACKOFF_MAX, backoff_value) 

352 

353 def parse_retry_after(self, retry_after): 

354 # Whitespace: https://tools.ietf.org/html/rfc7230#section-3.2.4 

355 if re.match(r"^\s*[0-9]+\s*$", retry_after): 

356 seconds = int(retry_after) 

357 else: 

358 retry_date_tuple = email.utils.parsedate_tz(retry_after) 

359 if retry_date_tuple is None: 

360 raise InvalidHeader("Invalid Retry-After header: %s" % retry_after) 

361 if retry_date_tuple[9] is None: # Python 2 

362 # Assume UTC if no timezone was specified 

363 # On Python2.7, parsedate_tz returns None for a timezone offset 

364 # instead of 0 if no timezone is given, where mktime_tz treats 

365 # a None timezone offset as local time. 

366 retry_date_tuple = retry_date_tuple[:9] + (0,) + retry_date_tuple[10:] 

367 

368 retry_date = email.utils.mktime_tz(retry_date_tuple) 

369 seconds = retry_date - time.time() 

370 

371 if seconds < 0: 

372 seconds = 0 

373 

374 return seconds 

375 

376 def get_retry_after(self, response): 

377 """ Get the value of Retry-After in seconds. """ 

378 

379 retry_after = response.getheader("Retry-After") 

380 

381 if retry_after is None: 

382 return None 

383 

384 return self.parse_retry_after(retry_after) 

385 

386 def sleep_for_retry(self, response=None): 

387 retry_after = self.get_retry_after(response) 

388 if retry_after: 

389 time.sleep(retry_after) 

390 return True 

391 

392 return False 

393 

394 def _sleep_backoff(self): 

395 backoff = self.get_backoff_time() 

396 if backoff <= 0: 

397 return 

398 time.sleep(backoff) 

399 

400 def sleep(self, response=None): 

401 """Sleep between retry attempts. 

402 

403 This method will respect a server's ``Retry-After`` response header 

404 and sleep the duration of the time requested. If that is not present, it 

405 will use an exponential backoff. By default, the backoff factor is 0 and 

406 this method will return immediately. 

407 """ 

408 

409 if self.respect_retry_after_header and response: 

410 slept = self.sleep_for_retry(response) 

411 if slept: 

412 return 

413 

414 self._sleep_backoff() 

415 

416 def _is_connection_error(self, err): 

417 """Errors when we're fairly sure that the server did not receive the 

418 request, so it should be safe to retry. 

419 """ 

420 if isinstance(err, ProxyError): 

421 err = err.original_error 

422 return isinstance(err, ConnectTimeoutError) 

423 

424 def _is_read_error(self, err): 

425 """Errors that occur after the request has been started, so we should 

426 assume that the server began processing it. 

427 """ 

428 return isinstance(err, (ReadTimeoutError, ProtocolError)) 

429 

430 def _is_method_retryable(self, method): 

431 """Checks if a given HTTP method should be retried upon, depending if 

432 it is included in the allowed_methods 

433 """ 

434 # TODO: For now favor if the Retry implementation sets its own method_whitelist 

435 # property outside of our constructor to avoid breaking custom implementations. 

436 if "method_whitelist" in self.__dict__: 

437 warnings.warn( 

438 "Using 'method_whitelist' with Retry is deprecated and " 

439 "will be removed in v2.0. Use 'allowed_methods' instead", 

440 DeprecationWarning, 

441 ) 

442 allowed_methods = self.method_whitelist 

443 else: 

444 allowed_methods = self.allowed_methods 

445 

446 if allowed_methods and method.upper() not in allowed_methods: 

447 return False 

448 return True 

449 

450 def is_retry(self, method, status_code, has_retry_after=False): 

451 """Is this method/status code retryable? (Based on allowlists and control 

452 variables such as the number of total retries to allow, whether to 

453 respect the Retry-After header, whether this header is present, and 

454 whether the returned status code is on the list of status codes to 

455 be retried upon on the presence of the aforementioned header) 

456 """ 

457 if not self._is_method_retryable(method): 

458 return False 

459 

460 if self.status_forcelist and status_code in self.status_forcelist: 

461 return True 

462 

463 return ( 

464 self.total 

465 and self.respect_retry_after_header 

466 and has_retry_after 

467 and (status_code in self.RETRY_AFTER_STATUS_CODES) 

468 ) 

469 

470 def is_exhausted(self): 

471 """ Are we out of retries? """ 

472 retry_counts = ( 

473 self.total, 

474 self.connect, 

475 self.read, 

476 self.redirect, 

477 self.status, 

478 self.other, 

479 ) 

480 retry_counts = list(filter(None, retry_counts)) 

481 if not retry_counts: 

482 return False 

483 

484 return min(retry_counts) < 0 

485 

486 def increment( 

487 self, 

488 method=None, 

489 url=None, 

490 response=None, 

491 error=None, 

492 _pool=None, 

493 _stacktrace=None, 

494 ): 

495 """Return a new Retry object with incremented retry counters. 

496 

497 :param response: A response object, or None, if the server did not 

498 return a response. 

499 :type response: :class:`~urllib3.response.HTTPResponse` 

500 :param Exception error: An error encountered during the request, or 

501 None if the response was received successfully. 

502 

503 :return: A new ``Retry`` object. 

504 """ 

505 if self.total is False and error: 

506 # Disabled, indicate to re-raise the error. 

507 raise six.reraise(type(error), error, _stacktrace) 

508 

509 total = self.total 

510 if total is not None: 

511 total -= 1 

512 

513 connect = self.connect 

514 read = self.read 

515 redirect = self.redirect 

516 status_count = self.status 

517 other = self.other 

518 cause = "unknown" 

519 status = None 

520 redirect_location = None 

521 

522 if error and self._is_connection_error(error): 

523 # Connect retry? 

524 if connect is False: 

525 raise six.reraise(type(error), error, _stacktrace) 

526 elif connect is not None: 

527 connect -= 1 

528 

529 elif error and self._is_read_error(error): 

530 # Read retry? 

531 if read is False or not self._is_method_retryable(method): 

532 raise six.reraise(type(error), error, _stacktrace) 

533 elif read is not None: 

534 read -= 1 

535 

536 elif error: 

537 # Other retry? 

538 if other is not None: 

539 other -= 1 

540 

541 elif response and response.get_redirect_location(): 

542 # Redirect retry? 

543 if redirect is not None: 

544 redirect -= 1 

545 cause = "too many redirects" 

546 redirect_location = response.get_redirect_location() 

547 status = response.status 

548 

549 else: 

550 # Incrementing because of a server error like a 500 in 

551 # status_forcelist and the given method is in the allowed_methods 

552 cause = ResponseError.GENERIC_ERROR 

553 if response and response.status: 

554 if status_count is not None: 

555 status_count -= 1 

556 cause = ResponseError.SPECIFIC_ERROR.format(status_code=response.status) 

557 status = response.status 

558 

559 history = self.history + ( 

560 RequestHistory(method, url, error, status, redirect_location), 

561 ) 

562 

563 new_retry = self.new( 

564 total=total, 

565 connect=connect, 

566 read=read, 

567 redirect=redirect, 

568 status=status_count, 

569 other=other, 

570 history=history, 

571 ) 

572 

573 if new_retry.is_exhausted(): 

574 raise MaxRetryError(_pool, url, error or ResponseError(cause)) 

575 

576 log.debug("Incremented Retry for (url='%s'): %r", url, new_retry) 

577 

578 return new_retry 

579 

580 def __repr__(self): 

581 return ( 

582 "{cls.__name__}(total={self.total}, connect={self.connect}, " 

583 "read={self.read}, redirect={self.redirect}, status={self.status})" 

584 ).format(cls=type(self), self=self) 

585 

586 def __getattr__(self, item): 

587 if item == "method_whitelist": 

588 # TODO: Remove this deprecated alias in v2.0 

589 warnings.warn( 

590 "Using 'method_whitelist' with Retry is deprecated and " 

591 "will be removed in v2.0. Use 'allowed_methods' instead", 

592 DeprecationWarning, 

593 ) 

594 return self.allowed_methods 

595 try: 

596 return getattr(super(Retry, self), item) 

597 except AttributeError: 

598 return getattr(Retry, item) 

599 

600 

601# For backwards compatibility (equivalent to pre-v1.9): 

602Retry.DEFAULT = Retry(3)