# Licensed to the Apache Software Foundation (ASF) under one or more
# contributor license agreements. See the NOTICE file distributed with
# this work for additional information regarding copyright ownership.
# The ASF licenses this file to You under the Apache License, Version 2.0
# (the "License"); you may not use this file except in compliance with
# the License. You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
import ssl
import time
import socket
import logging
from datetime import datetime, timedelta
from functools import wraps
from libcloud.utils.py3 import httplib
from libcloud.common.exceptions import RateLimitReachedError
__all__ = [
"Retry",
"RetryForeverOnRateLimitError",
]
_logger = logging.getLogger(__name__)
# Error message which indicates a transient SSL error upon which request
# can be retried
TRANSIENT_SSL_ERROR = "The read operation timed out"
class TransientSSLError(ssl.SSLError):
"""Represent transient SSL errors, e.g. timeouts"""
pass
# Constants used by the ``retry`` class
# All the time values (timeout, delay, backoff) are in seconds
DEFAULT_TIMEOUT = 30 # default retry timeout
DEFAULT_DELAY = 1 # default sleep delay used in each iterator
DEFAULT_BACKOFF = 1 # retry backup multiplier
RETRY_EXCEPTIONS = (
RateLimitReachedError,
socket.error,
socket.gaierror,
httplib.NotConnected,
httplib.ImproperConnectionState,
TransientSSLError,
)
class MinimalRetry:
def __init__(
self,
retry_delay=DEFAULT_DELAY,
timeout=DEFAULT_TIMEOUT,
backoff=DEFAULT_BACKOFF,
):
"""
Wrapper around retrying that helps to handle common transient
exceptions.
This minimalistic version only retries SSL errors and rate limiting.
:param retry_delay: retry delay between the attempts.
:param timeout: maximum time to wait.
:param backoff: multiplier added to delay between attempts.
:Example:
retry_request = MinimalRetry(timeout=1, retry_delay=1, backoff=1)
retry_request(self.connection.request)()
"""
if retry_delay is None:
retry_delay = DEFAULT_DELAY
if timeout is None:
timeout = DEFAULT_TIMEOUT
if backoff is None:
backoff = DEFAULT_BACKOFF
timeout = max(timeout, 0)
self.retry_delay = retry_delay
self.timeout = timeout
self.backoff = backoff
def __call__(self, func):
def transform_ssl_error(function, *args, **kwargs):
try:
return function(*args, **kwargs)
except ssl.SSLError as exc:
if TRANSIENT_SSL_ERROR in str(exc):
raise TransientSSLError(*exc.args)
raise exc
@wraps(func)
def retry_loop(*args, **kwargs):
current_delay = self.retry_delay
end = datetime.now() + timedelta(seconds=self.timeout)
last_exc = None
while datetime.now() < end:
try:
return transform_ssl_error(func, *args, **kwargs)
except Exception as exc:
last_exc = exc
if isinstance(exc, RateLimitReachedError):
_logger.debug("You are being rate limited, backing " "off...")
# NOTE: Retry after defaults to 0 in the
# RateLimitReachedError class so we a use more
# reasonable default in case that attribute is not
# present. This way we prevent busy waiting, etc.
retry_after = exc.retry_after if exc.retry_after else 2
time.sleep(retry_after)
# Reset delay if we're told to wait due to rate
# limiting
current_delay = self.retry_delay
elif self.should_retry(exc):
time.sleep(current_delay)
current_delay *= self.backoff
else:
raise
raise last_exc
return retry_loop
def should_retry(self, exception):
return False
[docs]class Retry(MinimalRetry):
def __init__(
self,
retry_exceptions=RETRY_EXCEPTIONS,
retry_delay=DEFAULT_DELAY,
timeout=DEFAULT_TIMEOUT,
backoff=DEFAULT_BACKOFF,
):
"""
Wrapper around retrying that helps to handle common transient
exceptions.
This version retries the errors that
`libcloud.utils.retry:MinimalRetry` retries and all errors of the
exception types that are given.
:param retry_exceptions: types of exceptions to retry on.
:param retry_delay: retry delay between the attempts.
:param timeout: maximum time to wait.
:param backoff: multiplier added to delay between attempts.
:Example:
retry_request = Retry(retry_exceptions=(httplib.NotConnected,),
timeout=1, retry_delay=1, backoff=1)
retry_request(self.connection.request)()
"""
super().__init__(retry_delay=retry_delay, timeout=timeout, backoff=backoff)
if retry_exceptions is None:
retry_exceptions = RETRY_EXCEPTIONS
self.retry_exceptions = retry_exceptions
[docs] def should_retry(self, exception):
return isinstance(exception, tuple(self.retry_exceptions))
[docs]class RetryForeverOnRateLimitError(Retry):
"""
This class is only here for backward compatibility reasons with
pre-Libcloud v3.3.2.
If works by ignoring timeout argument and retrying forever until API
is returning 429 RateLimitReached errors.
In most cases using this class is not a good idea since it can cause code
to hang and retry for ever in case API continues to return retry limit
reached.
"""
def __call__(self, func):
def transform_ssl_error(function, *args, **kwargs):
try:
return function(*args, **kwargs)
except ssl.SSLError as exc:
if TRANSIENT_SSL_ERROR in str(exc):
raise TransientSSLError(*exc.args)
raise exc
@wraps(func)
def retry_loop(*args, **kwargs):
current_delay = self.retry_delay
end = datetime.now() + timedelta(seconds=self.timeout)
while True:
try:
return transform_ssl_error(func, *args, **kwargs)
except Exception as exc:
if isinstance(exc, RateLimitReachedError):
time.sleep(exc.retry_after)
# Reset retries if we're told to wait due to rate
# limiting
current_delay = self.retry_delay
end = datetime.now() + timedelta(seconds=exc.retry_after + self.timeout)
elif datetime.now() >= end:
raise
elif self.should_retry(exc):
time.sleep(current_delay)
current_delay *= self.backoff
else:
raise
return retry_loop