airbyte_cdk.sources.declarative.retrievers.retriever
1# 2# Copyright (c) 2023 Airbyte, Inc., all rights reserved. 3# 4 5from abc import abstractmethod 6from typing import Any, Iterable, Mapping, Optional 7 8from typing_extensions import deprecated 9 10from airbyte_cdk.sources.streams.core import StreamData 11from airbyte_cdk.sources.types import StreamSlice, StreamState 12 13 14class Retriever: 15 """ 16 Responsible for fetching a stream's records from an HTTP API source. 17 18 A stream creates one retriever and reads all its partitions through it: `read_records` is called from several 19 threads at once, one call per stream slice. Implementations, custom ones included, must keep per-read state such 20 as page tokens, counters or recursion depth in local variables rather than on the instance or its components. 21 """ 22 23 @abstractmethod 24 def read_records( 25 self, 26 records_schema: Mapping[str, Any], 27 stream_slice: Optional[StreamSlice] = None, 28 ) -> Iterable[StreamData]: 29 """ 30 Fetch a stream's records from an HTTP API source 31 32 :param records_schema: json schema to describe record 33 :param stream_slice: The stream slice to read data for 34 :return: The records read from the API source 35 """ 36 37 @deprecated("Stream slicing is being moved to the stream level.") 38 def stream_slices(self) -> Iterable[Optional[StreamSlice]]: 39 """Does nothing as this method is deprecated, so underlying Retriever implementations 40 do not need to implement this. 41 """ 42 yield from [] 43 44 @property 45 @deprecated("State management is being moved to the stream level.") 46 def state(self) -> StreamState: 47 """ 48 Does nothing as this method is deprecated, so underlying Retriever implementations 49 do not need to implement this. 50 """ 51 return {} 52 53 @state.setter 54 @deprecated("State management is being moved to the stream level.") 55 def state(self, value: StreamState) -> None: 56 """ 57 Does nothing as this method is deprecated, so underlying Retriever implementations 58 do not need to implement this. 59 """ 60 pass
class
Retriever:
15class Retriever: 16 """ 17 Responsible for fetching a stream's records from an HTTP API source. 18 19 A stream creates one retriever and reads all its partitions through it: `read_records` is called from several 20 threads at once, one call per stream slice. Implementations, custom ones included, must keep per-read state such 21 as page tokens, counters or recursion depth in local variables rather than on the instance or its components. 22 """ 23 24 @abstractmethod 25 def read_records( 26 self, 27 records_schema: Mapping[str, Any], 28 stream_slice: Optional[StreamSlice] = None, 29 ) -> Iterable[StreamData]: 30 """ 31 Fetch a stream's records from an HTTP API source 32 33 :param records_schema: json schema to describe record 34 :param stream_slice: The stream slice to read data for 35 :return: The records read from the API source 36 """ 37 38 @deprecated("Stream slicing is being moved to the stream level.") 39 def stream_slices(self) -> Iterable[Optional[StreamSlice]]: 40 """Does nothing as this method is deprecated, so underlying Retriever implementations 41 do not need to implement this. 42 """ 43 yield from [] 44 45 @property 46 @deprecated("State management is being moved to the stream level.") 47 def state(self) -> StreamState: 48 """ 49 Does nothing as this method is deprecated, so underlying Retriever implementations 50 do not need to implement this. 51 """ 52 return {} 53 54 @state.setter 55 @deprecated("State management is being moved to the stream level.") 56 def state(self, value: StreamState) -> None: 57 """ 58 Does nothing as this method is deprecated, so underlying Retriever implementations 59 do not need to implement this. 60 """ 61 pass
Responsible for fetching a stream's records from an HTTP API source.
A stream creates one retriever and reads all its partitions through it: read_records is called from several
threads at once, one call per stream slice. Implementations, custom ones included, must keep per-read state such
as page tokens, counters or recursion depth in local variables rather than on the instance or its components.
@abstractmethod
def
read_records( self, records_schema: Mapping[str, Any], stream_slice: Optional[airbyte_cdk.StreamSlice] = None) -> Iterable[Union[Mapping[str, Any], airbyte_cdk.AirbyteMessage]]:
24 @abstractmethod 25 def read_records( 26 self, 27 records_schema: Mapping[str, Any], 28 stream_slice: Optional[StreamSlice] = None, 29 ) -> Iterable[StreamData]: 30 """ 31 Fetch a stream's records from an HTTP API source 32 33 :param records_schema: json schema to describe record 34 :param stream_slice: The stream slice to read data for 35 :return: The records read from the API source 36 """
Fetch a stream's records from an HTTP API source
Parameters
- records_schema: json schema to describe record
- stream_slice: The stream slice to read data for
Returns
The records read from the API source
@deprecated('Stream slicing is being moved to the stream level.')
def
stream_slices(self) -> Iterable[Optional[airbyte_cdk.StreamSlice]]:
38 @deprecated("Stream slicing is being moved to the stream level.") 39 def stream_slices(self) -> Iterable[Optional[StreamSlice]]: 40 """Does nothing as this method is deprecated, so underlying Retriever implementations 41 do not need to implement this. 42 """ 43 yield from []
Does nothing as this method is deprecated, so underlying Retriever implementations do not need to implement this.
state: Mapping[str, Any]
45 @property 46 @deprecated("State management is being moved to the stream level.") 47 def state(self) -> StreamState: 48 """ 49 Does nothing as this method is deprecated, so underlying Retriever implementations 50 do not need to implement this. 51 """ 52 return {}
Does nothing as this method is deprecated, so underlying Retriever implementations do not need to implement this.