airbyte_cdk.sources.declarative.requesters.paginators.strategies.pagination_strategy 

 1#
 2# Copyright (c) 2023 Airbyte, Inc., all rights reserved.
 3#
 4
 5from abc import abstractmethod
 6from dataclasses import dataclass
 7from typing import Any, Optional
 8
 9import requests
10
11from airbyte_cdk.sources.types import Record, StreamSlice
12
13
14@dataclass
15class PaginationStrategy:
16    """
17    Defines how to get the next page token
18
19    One instance paginates every partition of the stream, several at once, so `next_page_token` must derive the
20    next token from its arguments, such as `last_page_token_value`, rather than from state kept on the instance.
21    """
22
23    @property
24    @abstractmethod
25    def initial_token(self) -> Optional[Any]:
26        """
27        Return the initial value of the token
28        """
29
30    @abstractmethod
31    def next_page_token(
32        self,
33        response: requests.Response,
34        last_page_size: int,
35        last_record: Optional[Record],
36        last_page_token_value: Optional[Any],
37        page_size_override: Optional[int] = None,
38        stream_slice: Optional[StreamSlice] = None,
39    ) -> Optional[Any]:
40        """
41        :param response: response to process
42        :param last_page_size: the number of records read from the response
43        :param last_record: the last record extracted from the response
44        :param last_page_token_value: The current value of the page token made on the last request
45        :param page_size_override: the page size that was actually requested, when it differs from the configured
46            one because of a `REDUCE_PAGE_SIZE` response action. Strategies that use their configured page size as
47            a stop condition must honor this value, else they end the pagination early and skip records.
48        :param stream_slice: the slice the page was read for. Only passed to strategies that declare it, so existing
49            strategies defined outside of the CDK keep working unchanged.
50        :return: next page token. Returns None if there are no more pages to fetch
51        """
52        pass
53
54    @abstractmethod
55    def get_page_size(self) -> Optional[int]:
56        """
57        :return: page size: The number of records to fetch in a page. Returns None if unspecified
58        """
@dataclass
class PaginationStrategy: 
15@dataclass
16class PaginationStrategy:
17    """
18    Defines how to get the next page token
19
20    One instance paginates every partition of the stream, several at once, so `next_page_token` must derive the
21    next token from its arguments, such as `last_page_token_value`, rather than from state kept on the instance.
22    """
23
24    @property
25    @abstractmethod
26    def initial_token(self) -> Optional[Any]:
27        """
28        Return the initial value of the token
29        """
30
31    @abstractmethod
32    def next_page_token(
33        self,
34        response: requests.Response,
35        last_page_size: int,
36        last_record: Optional[Record],
37        last_page_token_value: Optional[Any],
38        page_size_override: Optional[int] = None,
39        stream_slice: Optional[StreamSlice] = None,
40    ) -> Optional[Any]:
41        """
42        :param response: response to process
43        :param last_page_size: the number of records read from the response
44        :param last_record: the last record extracted from the response
45        :param last_page_token_value: The current value of the page token made on the last request
46        :param page_size_override: the page size that was actually requested, when it differs from the configured
47            one because of a `REDUCE_PAGE_SIZE` response action. Strategies that use their configured page size as
48            a stop condition must honor this value, else they end the pagination early and skip records.
49        :param stream_slice: the slice the page was read for. Only passed to strategies that declare it, so existing
50            strategies defined outside of the CDK keep working unchanged.
51        :return: next page token. Returns None if there are no more pages to fetch
52        """
53        pass
54
55    @abstractmethod
56    def get_page_size(self) -> Optional[int]:
57        """
58        :return: page size: The number of records to fetch in a page. Returns None if unspecified
59        """

Defines how to get the next page token

One instance paginates every partition of the stream, several at once, so next_page_token must derive the next token from its arguments, such as last_page_token_value, rather than from state kept on the instance.

initial_token: Optional[Any] 
24    @property
25    @abstractmethod
26    def initial_token(self) -> Optional[Any]:
27        """
28        Return the initial value of the token
29        """

Return the initial value of the token

@abstractmethod
def next_page_token( self, response: requests.models.Response, last_page_size: int, last_record: Optional[airbyte_cdk.Record], last_page_token_value: Optional[Any], page_size_override: Optional[int] = None, stream_slice: Optional[airbyte_cdk.StreamSlice] = None) -> Optional[Any]: 
31    @abstractmethod
32    def next_page_token(
33        self,
34        response: requests.Response,
35        last_page_size: int,
36        last_record: Optional[Record],
37        last_page_token_value: Optional[Any],
38        page_size_override: Optional[int] = None,
39        stream_slice: Optional[StreamSlice] = None,
40    ) -> Optional[Any]:
41        """
42        :param response: response to process
43        :param last_page_size: the number of records read from the response
44        :param last_record: the last record extracted from the response
45        :param last_page_token_value: The current value of the page token made on the last request
46        :param page_size_override: the page size that was actually requested, when it differs from the configured
47            one because of a `REDUCE_PAGE_SIZE` response action. Strategies that use their configured page size as
48            a stop condition must honor this value, else they end the pagination early and skip records.
49        :param stream_slice: the slice the page was read for. Only passed to strategies that declare it, so existing
50            strategies defined outside of the CDK keep working unchanged.
51        :return: next page token. Returns None if there are no more pages to fetch
52        """
53        pass
Parameters
  • response: response to process
  • last_page_size: the number of records read from the response
  • last_record: the last record extracted from the response
  • last_page_token_value: The current value of the page token made on the last request
  • page_size_override: the page size that was actually requested, when it differs from the configured one because of a REDUCE_PAGE_SIZE response action. Strategies that use their configured page size as a stop condition must honor this value, else they end the pagination early and skip records.
  • stream_slice: the slice the page was read for. Only passed to strategies that declare it, so existing strategies defined outside of the CDK keep working unchanged.
Returns

next page token. Returns None if there are no more pages to fetch

@abstractmethod
def get_page_size(self) -> Optional[int]: 
55    @abstractmethod
56    def get_page_size(self) -> Optional[int]:
57        """
58        :return: page size: The number of records to fetch in a page. Returns None if unspecified
59        """
Returns

page size: The number of records to fetch in a page. Returns None if unspecified