airbyte_cdk.test.mock_http

 26class HttpMocker(contextlib.ContextDecorator):
 27    """
 28    WARNING 1: This implementation only works if the lib used to perform HTTP requests is `requests`.
 29
 30    WARNING 2: Given multiple requests that are not mutually exclusive, the request will match the first one. This can happen in scenarios
 31    where the same request is added twice (in which case there will always be an exception because we will never match the second
 32    request) or in a case like this:
 33    ```
 34    http_mocker.get(HttpRequest(_A_URL, headers={"less_granular": "1", "more_granular": "2"}), <...>)
 35    http_mocker.get(HttpRequest(_A_URL, headers={"less_granular": "1"}), <...>)
 36    requests.get(_A_URL, headers={"less_granular": "1", "more_granular": "2"})
 37    ```
 38    In the example above, the matcher would match the second mock as requests_mock iterate over the matcher in reverse order (see
 39    https://github.com/jamielennox/requests-mock/blob/c06f124a33f56e9f03840518e19669ba41b93202/requests_mock/adapter.py#L246) even
 40    though the request sent is a better match for the first `http_mocker.get`.
 41    """
 42
 43    def __init__(self) -> None:
 44        self._mocker = requests_mock.Mocker()
 45        self._matchers: Dict[SupportedHttpMethods, List[HttpRequestMatcher]] = defaultdict(list)
 46
 47    def __enter__(self) -> "HttpMocker":
 48        self._mocker.__enter__()
 49        return self
 50
 51    def __exit__(
 52        self,
 53        exc_type: Optional[BaseException],
 54        exc_val: Optional[BaseException],
 55        exc_tb: Optional[TracebackType],
 56    ) -> None:
 57        self._mocker.__exit__(exc_type, exc_val, exc_tb)
 58
 59    def _validate_all_matchers_called(self) -> None:
 60        for matcher in self._get_matchers():
 61            if not matcher.has_expected_match_count():
 62                raise ValueError(f"Invalid number of matches for `{matcher}`")
 63
 64    def _mock_request_method(
 65        self,
 66        method: SupportedHttpMethods,
 67        request: HttpRequest,
 68        responses: Union[HttpResponse, List[HttpResponse]],
 69    ) -> None:
 70        if isinstance(responses, HttpResponse):
 71            responses = [responses]
 72
 73        matcher = HttpRequestMatcher(request, len(responses))
 74        if matcher in self._matchers[method]:
 75            raise ValueError(f"Request {matcher.request} already mocked")
 76        self._matchers[method].append(matcher)
 77
 78        getattr(self._mocker, method)(
 79            requests_mock.ANY,
 80            additional_matcher=self._matches_wrapper(matcher),
 81            response_list=[
 82                {
 83                    self._get_body_field(response): response.body,
 84                    "status_code": response.status_code,
 85                    "headers": response.headers,
 86                }
 87                for response in responses
 88            ],
 89        )
 90
 91    @staticmethod
 92    def _get_body_field(response: HttpResponse) -> str:
 93        return "text" if isinstance(response.body, str) else "content"
 94
 95    def get(self, request: HttpRequest, responses: Union[HttpResponse, List[HttpResponse]]) -> None:
 96        self._mock_request_method(SupportedHttpMethods.GET, request, responses)
 97
 98    def patch(
 99        self, request: HttpRequest, responses: Union[HttpResponse, List[HttpResponse]]
100    ) -> None:
101        self._mock_request_method(SupportedHttpMethods.PATCH, request, responses)
102
103    def post(
104        self, request: HttpRequest, responses: Union[HttpResponse, List[HttpResponse]]
105    ) -> None:
106        self._mock_request_method(SupportedHttpMethods.POST, request, responses)
107
108    def put(self, request: HttpRequest, responses: Union[HttpResponse, List[HttpResponse]]) -> None:
109        self._mock_request_method(SupportedHttpMethods.PUT, request, responses)
110
111    def delete(
112        self, request: HttpRequest, responses: Union[HttpResponse, List[HttpResponse]]
113    ) -> None:
114        self._mock_request_method(SupportedHttpMethods.DELETE, request, responses)
115
116    @staticmethod
117    def _matches_wrapper(
118        matcher: HttpRequestMatcher,
119    ) -> Callable[[requests_mock.request._RequestObjectProxy], bool]:
120        def matches(requests_mock_request: requests_mock.request._RequestObjectProxy) -> bool:
121            # query_params are provided as part of `requests_mock_request.url`
122            http_request = HttpRequest(
123                requests_mock_request.url,
124                query_params={},
125                headers=requests_mock_request.headers,
126                body=requests_mock_request.body,
127            )
128            return matcher.matches(http_request)
129
130        return matches
131
132    def assert_number_of_calls(self, request: HttpRequest, number_of_calls: int) -> None:
133        corresponding_matchers = list(
134            filter(lambda matcher: matcher.request is request, self._get_matchers())
135        )
136        if len(corresponding_matchers) != 1:
137            raise ValueError(
138                f"Was expecting only one matcher to match the request but got `{corresponding_matchers}`"
139            )
140
141        assert corresponding_matchers[0].actual_number_of_matches == number_of_calls
142
143    # trying to type that using callables provides the error `incompatible with return type "_F" in supertype "ContextDecorator"`
144    def __call__(self, f):  # type: ignore
145        @functools.wraps(f)
146        def wrapper(*args, **kwargs):  # type: ignore  # this is a very generic wrapper that does not need to be typed
147            with self:
148                assertion_error = None
149
150                kwargs["http_mocker"] = self
151                try:
152                    result = f(*args, **kwargs)
153                except requests_mock.NoMockAddress as no_mock_exception:
154                    matchers_as_string = "\n\t".join(
155                        map(lambda matcher: str(matcher.request), self._get_matchers())
156                    )
157                    raise ValueError(
158                        f"No matcher matches {no_mock_exception.args[0]} with headers `{no_mock_exception.request.headers}` "
159                        f"and body `{no_mock_exception.request.body}`. "
160                        f"Matchers currently configured are:\n\t{matchers_as_string}."
161                    ) from no_mock_exception
162                except AssertionError as test_assertion:
163                    assertion_error = test_assertion
164
165                # We validate the matchers before raising the assertion error because we want to show the tester if an HTTP request wasn't
166                # mocked correctly
167                try:
168                    self._validate_all_matchers_called()
169                except ValueError as http_mocker_exception:
170                    # This seems useless as it catches ValueError and raises ValueError but without this, the prevailing error message in
171                    # the output is the function call that failed the assertion, whereas raising `ValueError(http_mocker_exception)`
172                    # like we do here provides additional context for the exception.
173                    raise ValueError(http_mocker_exception) from None
174                if assertion_error:
175                    raise assertion_error
176                return result
177
178        return wrapper
179
180    def _get_matchers(self) -> Iterable[HttpRequestMatcher]:
181        for matchers in self._matchers.values():
182            yield from matchers
183
184    def clear_all_matchers(self) -> None:
185        """Clears all stored matchers by resetting the _matchers list to an empty state."""
186        self._matchers = defaultdict(list)

 15class HttpRequest:
 16    def __init__(
 17        self,
 18        url: str,
 19        query_params: Optional[Union[str, Mapping[str, Union[str, List[str]]]]] = None,
 20        headers: Optional[Mapping[str, str]] = None,
 21        body: Optional[Union[str, bytes, Mapping[str, Any]]] = None,
 22    ) -> None:
 23        self._parsed_url = urlparse(url)
 24        self._query_params = query_params
 25        if not self._parsed_url.query and query_params:
 26            self._parsed_url = urlparse(f"{url}?{self._encode_qs(query_params)}")
 27        elif self._parsed_url.query and query_params:
 28            raise ValueError(
 29                "If query params are provided as part of the url, `query_params` should be empty"
 30            )
 31
 32        self._headers = headers or {}
 33        self._body = body
 34
 35    @staticmethod
 36    def _encode_qs(query_params: Union[str, Mapping[str, Union[str, List[str]]]]) -> str:
 37        if isinstance(query_params, str):
 38            return query_params
 39        return urlencode(query_params, doseq=True)
 40
 41    def matches(self, other: Any) -> bool:
 42        """
 43        If the body of any request is a Mapping, we compare as Mappings which means that the order is not important.
 44        If the body is a string, encoding ISO-8859-1 will be assumed
 45        Headers only need to be a subset of `other` in order to match
 46        """
 47        if isinstance(other, HttpRequest):
 48            # if `other` is a mapping, we match as an object and formatting is not considers
 49            if isinstance(self._body, Mapping) or isinstance(other._body, Mapping):
 50                body_match = self._to_mapping(self._body) == self._to_mapping(other._body)
 51            else:
 52                body_match = self._to_bytes(self._body) == self._to_bytes(other._body)
 53
 54            return (
 55                self._parsed_url.scheme == other._parsed_url.scheme
 56                and self._parsed_url.hostname == other._parsed_url.hostname
 57                and self._parsed_url.path == other._parsed_url.path
 58                and (
 59                    ANY_QUERY_PARAMS in (self._query_params, other._query_params)
 60                    or parse_qs(self._parsed_url.query) == parse_qs(other._parsed_url.query)
 61                )
 62                and _is_subdict(other._headers, self._headers)
 63                and body_match
 64            )
 65        return False
 66
 67    @staticmethod
 68    def _to_mapping(
 69        body: Optional[Union[str, bytes, Mapping[str, Any]]],
 70    ) -> Optional[Mapping[str, Any]]:
 71        if isinstance(body, Mapping):
 72            return body
 73        elif isinstance(body, bytes):
 74            return json.loads(body.decode())  # type: ignore  # assumes return type of Mapping[str, Any]
 75        elif isinstance(body, str):
 76            return json.loads(body)  # type: ignore  # assumes return type of Mapping[str, Any]
 77        return None
 78
 79    @staticmethod
 80    def _to_bytes(body: Optional[Union[str, bytes]]) -> bytes:
 81        if isinstance(body, bytes):
 82            return body
 83        elif isinstance(body, str):
 84            # `ISO-8859-1` is the default encoding used by requests
 85            return body.encode("ISO-8859-1")
 86        return b""
 87
 88    def __str__(self) -> str:
 89        return f"{self._parsed_url} with headers {self._headers} and body {self._body!r})"
 90
 91    def __repr__(self) -> str:
 92        return (
 93            f"HttpRequest(request={self._parsed_url}, headers={self._headers}, body={self._body!r})"
 94        )
 95
 96    def __eq__(self, other: Any) -> bool:
 97        if isinstance(other, HttpRequest):
 98            return (
 99                self._parsed_url == other._parsed_url
100                and self._query_params == other._query_params
101                and self._headers == other._headers
102                and self._body == other._body
103            )
104        return False

 8class HttpRequestMatcher:
 9    def __init__(self, request: HttpRequest, minimum_number_of_expected_match: int):
10        self._request_to_match = request
11        self._minimum_number_of_expected_match = minimum_number_of_expected_match
12        self._actual_number_of_matches = 0
13
14    def matches(self, request: HttpRequest) -> bool:
15        hit = request.matches(self._request_to_match)
16        if hit:
17            self._actual_number_of_matches += 1
18        return hit
19
20    def has_expected_match_count(self) -> bool:
21        return self._actual_number_of_matches >= self._minimum_number_of_expected_match
22
23    @property
24    def actual_number_of_matches(self) -> int:
25        return self._actual_number_of_matches
26
27    @property
28    def request(self) -> HttpRequest:
29        return self._request_to_match
30
31    def __str__(self) -> str:
32        return (
33            f"HttpRequestMatcher("
34            f"request_to_match={self._request_to_match}, "
35            f"minimum_number_of_expected_match={self._minimum_number_of_expected_match}, "
36            f"actual_number_of_matches={self._actual_number_of_matches})"
37        )
38
39    def __eq__(self, other: Any) -> bool:
40        if isinstance(other, HttpRequestMatcher):
41            return self._request_to_match == other._request_to_match
42        return False

 8class HttpResponse:
 9    def __init__(
10        self,
11        body: Union[str, bytes],
12        status_code: int = 200,
13        headers: Mapping[str, str] = MappingProxyType({}),
14    ):
15        self._body = body
16        self._status_code = status_code
17        self._headers = headers
18
19    @property
20    def body(self) -> Union[str, bytes]:
21        return self._body
22
23    @property
24    def status_code(self) -> int:
25        return self._status_code
26
27    @property
28    def headers(self) -> Mapping[str, str]:
29        return self._headers