Skip to content

base_provider

BaseProvider Base Class

BaseProvider #

Bases: ABC

Base Provider Class

Source code in camply/providers/base_provider.py 
 33
 34
 35
 36
 37
 38
 39
 40
 41
 42
 43
 44
 45
 46
 47
 48
 49
 50
 51
 52
 53
 54
 55
 56
 57
 58
 59
 60
 61
 62
 63
 64
 65
 66
 67
 68
 69
 70
 71
 72
 73
 74
 75
 76
 77
 78
 79
 80
 81
 82
 83
 84
 85
 86
 87
 88
 89
 90
 91
 92
 93
 94
 95
 96
 97
 98
 99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
class BaseProvider(ABC):
    """
    Base Provider Class
    """

    RETRY_CONFIG: Type[APIConfig] = APIConfig
    FIVE_HUNDRED_STATUS_CODES = [
        # Official Server Errors
        500,  # Internal Server Error
        501,  # Not Implemented
        502,  # Bad Gateway
        503,  # Service Unavailable
        504,  # Gateway Timeout
        505,  # HTTP Version Not Supported
        506,  # Variant Also Negotiates
        507,  # Insufficient Storage
        508,  # Loop Detected
        510,  # Not Extended
        511,  # Network Authentication Required
        # Unofficial Server Errors
        509,  # Bandwidth Limit Exceeded
        529,  # Site is overloaded
        530,  # Site is frozen
        598,  # Network read timeout error
        599,  # Network connect timeout error
        # Vendor Errors
        520,  # Unknown Error
        521,  # Web Server Is Down
        522,  # Connection Timed Out
        523,  # Origin Is Unreachable
        524,  # A Timeout Occurred
        525,  # SSL Handshake Failed
        526,  # Invalid SSL Certificate
        527,  # Railgun Error
        530,  # Origin DNS Error
        561,  # Unauthorized
    ]

    def __repr__(self):
        """
        String Representation

        Returns
        -------
        str
        """
        return f"<{self.__class__.__name__}>"

    def __init__(self):
        """
        Initialize with a session
        """
        _user_agent = UserAgent(browsers=["chrome"]).random
        self.session = requests.Session()
        self.headers = {"User-Agent": _user_agent}
        self.session.headers = self.headers
        self.json_headers = self.headers.copy()
        self.json_headers.update({"Content-Type": "application/json"})

    @classmethod
    def get_search_months(cls, search_days) -> List[datetime]:
        """
        Get the Unique Months that need to be Searched

        Returns
        -------
        search_months: List[datetime]
            Datetime Months to search for reservations
        """
        truncated_months = {day.replace(day=1) for day in search_days}
        if len(truncated_months) > 1:
            logger.info(
                f"{len(truncated_months)} different months selected for search, "
                f"ranging from {min(search_days)} to {max(search_days)}"
            )
            return sorted(truncated_months)
        elif len(truncated_months) == 0:
            logger.error(SearchConfig.ERROR_MESSAGE)
            raise RuntimeError(SearchConfig.ERROR_MESSAGE)
        else:
            return sorted(truncated_months)

    @abstractmethod
    def find_campgrounds(self) -> List[CampgroundFacility]:
        """
        List Recreation Areas for the provider
        """

    def make_http_request(
        self,
        url: str,
        method: str = "GET",
        data: Optional[Union[Dict[str, Any], str]] = None,
        headers: Optional[Dict[str, Any]] = None,
        retry_response_codes: Optional[List[int]] = None,
    ) -> requests.Response:
        """
        Make an HTTP Request

        Parameters
        ----------
        url: str
            URL to make the request to
        method: str
            HTTP Method to use. Defaults to GET
        data: Optional[Union[Dict[str, Any], str]]
            Data to send with the request
        headers: Optional[Dict[str, Any]]
            Headers to send with the request
        retry_response_codes: Optional[List[int]]
            List of response codes to raise a ProviderError. on. Defaults to 500 range

        Returns
        -------
        response: requests.Response

        Raises
        ------
        ProviderError
            If the response code is in the retry_response_codes list
        HTTPError
            If the response code is not in the retry_response_codes list and the request fails
        """
        if retry_response_codes is None:
            retry_response_codes = self.FIVE_HUNDRED_STATUS_CODES
        response = self.session.request(
            method=method, url=url, data=data, headers=headers
        )
        if response.status_code not in retry_response_codes:
            response.raise_for_status()
        else:
            error_message = f"HTTP Error - {self.__class__.__name__} - {response.url} - {response.status_code}"
            logger.warning(error_message)
            error_message += f": {response.text}"
            raise ProviderError(error_message)
        return response

    def make_http_request_retry(
        self,
        url: str,
        method: str = "GET",
        data: Optional[Union[Dict[str, Any], str]] = None,
        headers: Optional[Dict[str, Any]] = None,
        retry_response_codes: Optional[List[int]] = None,
    ) -> requests.Response:
        """
        Make an HTTP Request with Exponential Backoff

        This method will retry the request with exponential backoff if the request fails.
        By default, it will only ignore 500 range status codes, but this can be overridden.

        Parameters
        ----------
        url: str
            URL to make the request to
        method: str
            HTTP Method to use. Defaults to GET
        data: Optional[Union[Dict[str, Any], str]]
            Data to send with the request
        headers: Optional[Dict[str, Any]]
            Headers to send with the request
        retry_response_codes: Optional[List[int]]
            List of response codes to retry on. Defaults to 500 range

        Returns
        -------
        response: requests.Response
        """
        retryer = tenacity.Retrying(
            wait=tenacity.wait_random_exponential(
                multiplier=self.RETRY_CONFIG.RETRY_API_MULTIPLIER,
                max=self.RETRY_CONFIG.RETRY_MAX_API_ATTEMPTS,
            ),
            stop=tenacity.stop.stop_after_delay(
                self.RETRY_CONFIG.RETRY_MAX_API_TIMEOUT
            ),
            retry=tenacity.retry_if_exception_type(ProviderError),
        )
        response: requests.Response = retryer.__call__(
            fn=self.make_http_request,
            url=url,
            method=method,
            data=data,
            headers=headers,
            retry_response_codes=retry_response_codes,
        )
        return response

__init__() #

Initialize with a session

Source code in camply/providers/base_provider.py 
81
82
83
84
85
86
87
88
89
90
def __init__(self):
    """
    Initialize with a session
    """
    _user_agent = UserAgent(browsers=["chrome"]).random
    self.session = requests.Session()
    self.headers = {"User-Agent": _user_agent}
    self.session.headers = self.headers
    self.json_headers = self.headers.copy()
    self.json_headers.update({"Content-Type": "application/json"})

__repr__() #

String Representation

Returns:

Type Description
str
Source code in camply/providers/base_provider.py 
71
72
73
74
75
76
77
78
79
def __repr__(self):
    """
    String Representation

    Returns
    -------
    str
    """
    return f"<{self.__class__.__name__}>"

find_campgrounds() abstractmethod #

List Recreation Areas for the provider

Source code in camply/providers/base_provider.py 
115
116
117
118
119
@abstractmethod
def find_campgrounds(self) -> List[CampgroundFacility]:
    """
    List Recreation Areas for the provider
    """

get_search_months(search_days) classmethod #

Get the Unique Months that need to be Searched

Returns:

Name Type Description
search_months List[datetime]

Datetime Months to search for reservations
Source code in camply/providers/base_provider.py 
 92
 93
 94
 95
 96
 97
 98
 99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
@classmethod
def get_search_months(cls, search_days) -> List[datetime]:
    """
    Get the Unique Months that need to be Searched

    Returns
    -------
    search_months: List[datetime]
        Datetime Months to search for reservations
    """
    truncated_months = {day.replace(day=1) for day in search_days}
    if len(truncated_months) > 1:
        logger.info(
            f"{len(truncated_months)} different months selected for search, "
            f"ranging from {min(search_days)} to {max(search_days)}"
        )
        return sorted(truncated_months)
    elif len(truncated_months) == 0:
        logger.error(SearchConfig.ERROR_MESSAGE)
        raise RuntimeError(SearchConfig.ERROR_MESSAGE)
    else:
        return sorted(truncated_months)

make_http_request(url, method='GET', data=None, headers=None, retry_response_codes=None) #

Make an HTTP Request

Parameters:

Name Type Description Default
url str

URL to make the request to

 required
method str

HTTP Method to use. Defaults to GET

 'GET'
data Optional[Union[Dict[str, Any], str]]

Data to send with the request

 None
headers Optional[Dict[str, Any]]

Headers to send with the request

 None
retry_response_codes Optional[List[int]]

List of response codes to raise a ProviderError. on. Defaults to 500 range

 None

Returns:

Name Type Description
response Response

Raises:

Type Description
ProviderError

If the response code is in the retry_response_codes list
HTTPError

If the response code is not in the retry_response_codes list and the request fails
Source code in camply/providers/base_provider.py 
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
def make_http_request(
    self,
    url: str,
    method: str = "GET",
    data: Optional[Union[Dict[str, Any], str]] = None,
    headers: Optional[Dict[str, Any]] = None,
    retry_response_codes: Optional[List[int]] = None,
) -> requests.Response:
    """
    Make an HTTP Request

    Parameters
    ----------
    url: str
        URL to make the request to
    method: str
        HTTP Method to use. Defaults to GET
    data: Optional[Union[Dict[str, Any], str]]
        Data to send with the request
    headers: Optional[Dict[str, Any]]
        Headers to send with the request
    retry_response_codes: Optional[List[int]]
        List of response codes to raise a ProviderError. on. Defaults to 500 range

    Returns
    -------
    response: requests.Response

    Raises
    ------
    ProviderError
        If the response code is in the retry_response_codes list
    HTTPError
        If the response code is not in the retry_response_codes list and the request fails
    """
    if retry_response_codes is None:
        retry_response_codes = self.FIVE_HUNDRED_STATUS_CODES
    response = self.session.request(
        method=method, url=url, data=data, headers=headers
    )
    if response.status_code not in retry_response_codes:
        response.raise_for_status()
    else:
        error_message = f"HTTP Error - {self.__class__.__name__} - {response.url} - {response.status_code}"
        logger.warning(error_message)
        error_message += f": {response.text}"
        raise ProviderError(error_message)
    return response

make_http_request_retry(url, method='GET', data=None, headers=None, retry_response_codes=None) #

Make an HTTP Request with Exponential Backoff

This method will retry the request with exponential backoff if the request fails. By default, it will only ignore 500 range status codes, but this can be overridden.

Parameters:

Name Type Description Default
url str

URL to make the request to

 required
method str

HTTP Method to use. Defaults to GET

 'GET'
data Optional[Union[Dict[str, Any], str]]

Data to send with the request

 None
headers Optional[Dict[str, Any]]

Headers to send with the request

 None
retry_response_codes Optional[List[int]]

List of response codes to retry on. Defaults to 500 range

 None

Returns:

Name Type Description
response Response
Source code in camply/providers/base_provider.py 
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
def make_http_request_retry(
    self,
    url: str,
    method: str = "GET",
    data: Optional[Union[Dict[str, Any], str]] = None,
    headers: Optional[Dict[str, Any]] = None,
    retry_response_codes: Optional[List[int]] = None,
) -> requests.Response:
    """
    Make an HTTP Request with Exponential Backoff

    This method will retry the request with exponential backoff if the request fails.
    By default, it will only ignore 500 range status codes, but this can be overridden.

    Parameters
    ----------
    url: str
        URL to make the request to
    method: str
        HTTP Method to use. Defaults to GET
    data: Optional[Union[Dict[str, Any], str]]
        Data to send with the request
    headers: Optional[Dict[str, Any]]
        Headers to send with the request
    retry_response_codes: Optional[List[int]]
        List of response codes to retry on. Defaults to 500 range

    Returns
    -------
    response: requests.Response
    """
    retryer = tenacity.Retrying(
        wait=tenacity.wait_random_exponential(
            multiplier=self.RETRY_CONFIG.RETRY_API_MULTIPLIER,
            max=self.RETRY_CONFIG.RETRY_MAX_API_ATTEMPTS,
        ),
        stop=tenacity.stop.stop_after_delay(
            self.RETRY_CONFIG.RETRY_MAX_API_TIMEOUT
        ),
        retry=tenacity.retry_if_exception_type(ProviderError),
    )
    response: requests.Response = retryer.__call__(
        fn=self.make_http_request,
        url=url,
        method=method,
        data=data,
        headers=headers,
        retry_response_codes=retry_response_codes,
    )
    return response

ProviderError #

Bases: Exception

General Provider Error

Source code in camply/providers/base_provider.py 
21
22
23
24
class ProviderError(Exception):
    """
    General Provider Error
    """

ProviderSearchError #

Bases: ProviderError

Searching Error

Source code in camply/providers/base_provider.py 
27
28
29
30
class ProviderSearchError(ProviderError):
    """
    Searching Error
    """