airbyte_cdk.sources.declarative.stream_slicers
1# 2# Copyright (c) 2023 Airbyte, Inc., all rights reserved. 3# 4 5from airbyte_cdk.sources.declarative.stream_slicers.stream_slicer import StreamSlicer 6from airbyte_cdk.sources.declarative.stream_slicers.stream_slicer_test_read_decorator import ( 7 StreamSlicerTestReadDecorator, 8) 9 10__all__ = ["StreamSlicer", "StreamSlicerTestReadDecorator"]
class
StreamSlicer(airbyte_cdk.sources.streams.concurrent.partitions.stream_slicer.StreamSlicer, airbyte_cdk.sources.declarative.requesters.request_options.request_options_provider.RequestOptionsProvider, abc.ABC):
16class StreamSlicer(ConcurrentStreamSlicer, RequestOptionsProvider, ABC): 17 """ 18 Slices the stream into a subset of records. 19 Slices enable state checkpointing and data retrieval parallelization. 20 21 The stream slicer keeps track of the cursor state as a dict of cursor_field -> cursor_value 22 23 See the stream slicing section of the docs for more information. 24 """ 25 26 pass
Slices the stream into a subset of records. Slices enable state checkpointing and data retrieval parallelization.
The stream slicer keeps track of the cursor state as a dict of cursor_field -> cursor_value
See the stream slicing section of the docs for more information.
@dataclass
class
StreamSlicerTestReadDecorator14@dataclass 15class StreamSlicerTestReadDecorator(StreamSlicer): 16 """ 17 In some cases, we want to limit the number of requests that are made to the backend source. This class allows for limiting the number of 18 slices that are queried throughout a read command. 19 """ 20 21 wrapped_slicer: StreamSlicer 22 maximum_number_of_slices: int = 5 23 24 def stream_slices(self) -> Iterable[StreamSlice]: 25 return islice(self.wrapped_slicer.stream_slices(), self.maximum_number_of_slices) 26 27 def __getattr__(self, name: str) -> Any: 28 # Delegate everything else to the wrapped object 29 return getattr(self.wrapped_slicer, name)
In some cases, we want to limit the number of requests that are made to the backend source. This class allows for limiting the number of slices that are queried throughout a read command.
StreamSlicerTestReadDecorator( wrapped_slicer: airbyte_cdk.sources.streams.concurrent.partitions.stream_slicer.StreamSlicer, maximum_number_of_slices: int = 5)