airbyte_cdk.sources.declarative.parsers.model_to_component_factory
1# 2# Copyright (c) 2025 Airbyte, Inc., all rights reserved. 3# 4 5from __future__ import annotations 6 7import datetime 8import importlib 9import inspect 10import json 11import logging 12import math 13import re 14from functools import partial 15from typing import ( 16 TYPE_CHECKING, 17 Any, 18 Callable, 19 Dict, 20 List, 21 Mapping, 22 MutableMapping, 23 Optional, 24 Tuple, 25 Type, 26 Union, 27 cast, 28 get_args, 29 get_origin, 30 get_type_hints, 31) 32 33if TYPE_CHECKING: 34 from airbyte_cdk.legacy.sources.declarative.incremental.datetime_based_cursor import ( 35 DatetimeBasedCursor, 36 ) 37 38from airbyte_protocol_dataclasses.models import ConfiguredAirbyteStream 39from isodate import parse_duration 40from pydantic.v1 import BaseModel 41from requests import Response 42 43from airbyte_cdk.connector_builder.models import ( 44 LogMessage as ConnectorBuilderLogMessage, 45) 46from airbyte_cdk.models import ( 47 AirbyteStateBlob, 48 AirbyteStateMessage, 49 AirbyteStateType, 50 AirbyteStreamState, 51 ConfiguredAirbyteCatalog, 52 FailureType, 53 Level, 54 StreamDescriptor, 55) 56from airbyte_cdk.sources.connector_state_manager import ConnectorStateManager 57from airbyte_cdk.sources.declarative.async_job.job_orchestrator import AsyncJobOrchestrator 58from airbyte_cdk.sources.declarative.async_job.job_tracker import JobTracker 59from airbyte_cdk.sources.declarative.async_job.repository import AsyncJobRepository 60from airbyte_cdk.sources.declarative.async_job.status import AsyncJobStatus 61from airbyte_cdk.sources.declarative.auth import DeclarativeOauth2Authenticator, JwtAuthenticator 62from airbyte_cdk.sources.declarative.auth.declarative_authenticator import ( 63 DeclarativeAuthenticator, 64 NoAuth, 65) 66from airbyte_cdk.sources.declarative.auth.jwt import JwtAlgorithm 67from airbyte_cdk.sources.declarative.auth.oauth import ( 68 DeclarativeSingleUseRefreshTokenOauth2Authenticator, 69) 70from airbyte_cdk.sources.declarative.auth.rate_limited_multiple_token import ( 71 RateLimitedMultipleTokenAuthenticator, 72 TokenQuota, 73) 74from airbyte_cdk.sources.declarative.auth.selective_authenticator import SelectiveAuthenticator 75from airbyte_cdk.sources.declarative.auth.token import ( 76 ApiKeyAuthenticator, 77 BasicHttpAuthenticator, 78 BearerAuthenticator, 79 LegacySessionTokenAuthenticator, 80) 81from airbyte_cdk.sources.declarative.auth.token_provider import ( 82 InterpolatedSessionTokenProvider, 83 InterpolatedStringTokenProvider, 84 SessionTokenProvider, 85 TokenProvider, 86) 87from airbyte_cdk.sources.declarative.checks import ( 88 CheckDynamicStream, 89 CheckStream, 90 DynamicStreamCheckConfig, 91) 92from airbyte_cdk.sources.declarative.concurrency_level import ConcurrencyLevel 93from airbyte_cdk.sources.declarative.datetime.min_max_datetime import MinMaxDatetime 94from airbyte_cdk.sources.declarative.decoders import ( 95 Decoder, 96 IterableDecoder, 97 JsonDecoder, 98 PaginationDecoderDecorator, 99 XmlDecoder, 100 ZipfileDecoder, 101) 102from airbyte_cdk.sources.declarative.decoders.composite_raw_decoder import ( 103 CompositeRawDecoder, 104 CsvParser, 105 GzipParser, 106 JsonItemsParser, 107 JsonLineParser, 108 JsonParser, 109 Parser, 110) 111from airbyte_cdk.sources.declarative.expanders.record_expander import ( 112 OnNoRecords, 113 ParentFieldPath, 114 RecordExpander, 115) 116from airbyte_cdk.sources.declarative.extractors import ( 117 CombinedExtractor, 118 CombineMode, 119 DpathExtractor, 120 RecordFilter, 121 RecordSelector, 122 ResponseToFileExtractor, 123) 124from airbyte_cdk.sources.declarative.extractors.record_extractor import RecordExtractor 125from airbyte_cdk.sources.declarative.extractors.record_filter import ( 126 ClientSideIncrementalRecordFilterDecorator, 127) 128from airbyte_cdk.sources.declarative.incremental import ( 129 ConcurrentCursorFactory, 130 ConcurrentPerPartitionCursor, 131) 132from airbyte_cdk.sources.declarative.interpolation import InterpolatedString 133from airbyte_cdk.sources.declarative.interpolation.interpolated_mapping import InterpolatedMapping 134from airbyte_cdk.sources.declarative.migrations.legacy_to_per_partition_state_migration import ( 135 LegacyToPerPartitionStateMigration, 136) 137from airbyte_cdk.sources.declarative.models import ( 138 CustomStateMigration, 139 PaginationResetLimits, 140) 141from airbyte_cdk.sources.declarative.models.base_model_with_deprecations import ( 142 DEPRECATION_LOGS_TAG, 143 BaseModelWithDeprecations, 144) 145from airbyte_cdk.sources.declarative.models.declarative_component_schema import ( 146 Action as HttpResponseFilterActionModel, 147) 148from airbyte_cdk.sources.declarative.models.declarative_component_schema import ( 149 Action1 as PaginationResetActionModel, 150) 151from airbyte_cdk.sources.declarative.models.declarative_component_schema import ( 152 AddedFieldDefinition as AddedFieldDefinitionModel, 153) 154from airbyte_cdk.sources.declarative.models.declarative_component_schema import ( 155 AddFields as AddFieldsModel, 156) 157from airbyte_cdk.sources.declarative.models.declarative_component_schema import ( 158 ApiKeyAuthenticator as ApiKeyAuthenticatorModel, 159) 160from airbyte_cdk.sources.declarative.models.declarative_component_schema import ( 161 AsyncJobStatusMap as AsyncJobStatusMapModel, 162) 163from airbyte_cdk.sources.declarative.models.declarative_component_schema import ( 164 AsyncRetriever as AsyncRetrieverModel, 165) 166from airbyte_cdk.sources.declarative.models.declarative_component_schema import ( 167 BasicHttpAuthenticator as BasicHttpAuthenticatorModel, 168) 169from airbyte_cdk.sources.declarative.models.declarative_component_schema import ( 170 BearerAuthenticator as BearerAuthenticatorModel, 171) 172from airbyte_cdk.sources.declarative.models.declarative_component_schema import ( 173 CheckDynamicStream as CheckDynamicStreamModel, 174) 175from airbyte_cdk.sources.declarative.models.declarative_component_schema import ( 176 CheckStream as CheckStreamModel, 177) 178from airbyte_cdk.sources.declarative.models.declarative_component_schema import ( 179 CombinedExtractor as CombinedExtractorModel, 180) 181from airbyte_cdk.sources.declarative.models.declarative_component_schema import ( 182 ComplexFieldType as ComplexFieldTypeModel, 183) 184from airbyte_cdk.sources.declarative.models.declarative_component_schema import ( 185 ComponentMappingDefinition as ComponentMappingDefinitionModel, 186) 187from airbyte_cdk.sources.declarative.models.declarative_component_schema import ( 188 CompositeErrorHandler as CompositeErrorHandlerModel, 189) 190from airbyte_cdk.sources.declarative.models.declarative_component_schema import ( 191 ConcurrencyLevel as ConcurrencyLevelModel, 192) 193from airbyte_cdk.sources.declarative.models.declarative_component_schema import ( 194 ConfigAddFields as ConfigAddFieldsModel, 195) 196from airbyte_cdk.sources.declarative.models.declarative_component_schema import ( 197 ConfigComponentsResolver as ConfigComponentsResolverModel, 198) 199from airbyte_cdk.sources.declarative.models.declarative_component_schema import ( 200 ConfigMigration as ConfigMigrationModel, 201) 202from airbyte_cdk.sources.declarative.models.declarative_component_schema import ( 203 ConfigRemapField as ConfigRemapFieldModel, 204) 205from airbyte_cdk.sources.declarative.models.declarative_component_schema import ( 206 ConfigRemoveFields as ConfigRemoveFieldsModel, 207) 208from airbyte_cdk.sources.declarative.models.declarative_component_schema import ( 209 ConstantBackoffStrategy as ConstantBackoffStrategyModel, 210) 211from airbyte_cdk.sources.declarative.models.declarative_component_schema import ( 212 CsvDecoder as CsvDecoderModel, 213) 214from airbyte_cdk.sources.declarative.models.declarative_component_schema import ( 215 CursorPagination as CursorPaginationModel, 216) 217from airbyte_cdk.sources.declarative.models.declarative_component_schema import ( 218 CustomAuthenticator as CustomAuthenticatorModel, 219) 220from airbyte_cdk.sources.declarative.models.declarative_component_schema import ( 221 CustomBackoffStrategy as CustomBackoffStrategyModel, 222) 223from airbyte_cdk.sources.declarative.models.declarative_component_schema import ( 224 CustomConfigTransformation as CustomConfigTransformationModel, 225) 226from airbyte_cdk.sources.declarative.models.declarative_component_schema import ( 227 CustomDecoder as CustomDecoderModel, 228) 229from airbyte_cdk.sources.declarative.models.declarative_component_schema import ( 230 CustomErrorHandler as CustomErrorHandlerModel, 231) 232from airbyte_cdk.sources.declarative.models.declarative_component_schema import ( 233 CustomPaginationStrategy as CustomPaginationStrategyModel, 234) 235from airbyte_cdk.sources.declarative.models.declarative_component_schema import ( 236 CustomPartitionRouter as CustomPartitionRouterModel, 237) 238from airbyte_cdk.sources.declarative.models.declarative_component_schema import ( 239 CustomRecordExtractor as CustomRecordExtractorModel, 240) 241from airbyte_cdk.sources.declarative.models.declarative_component_schema import ( 242 CustomRecordFilter as CustomRecordFilterModel, 243) 244from airbyte_cdk.sources.declarative.models.declarative_component_schema import ( 245 CustomRequester as CustomRequesterModel, 246) 247from airbyte_cdk.sources.declarative.models.declarative_component_schema import ( 248 CustomRetriever as CustomRetrieverModel, 249) 250from airbyte_cdk.sources.declarative.models.declarative_component_schema import ( 251 CustomSchemaLoader as CustomSchemaLoader, 252) 253from airbyte_cdk.sources.declarative.models.declarative_component_schema import ( 254 CustomSchemaNormalization as CustomSchemaNormalizationModel, 255) 256from airbyte_cdk.sources.declarative.models.declarative_component_schema import ( 257 CustomTransformation as CustomTransformationModel, 258) 259from airbyte_cdk.sources.declarative.models.declarative_component_schema import ( 260 CustomValidationStrategy as CustomValidationStrategyModel, 261) 262from airbyte_cdk.sources.declarative.models.declarative_component_schema import ( 263 DatetimeBasedCursor as DatetimeBasedCursorModel, 264) 265from airbyte_cdk.sources.declarative.models.declarative_component_schema import ( 266 DeclarativeStream as DeclarativeStreamModel, 267) 268from airbyte_cdk.sources.declarative.models.declarative_component_schema import ( 269 DefaultErrorHandler as DefaultErrorHandlerModel, 270) 271from airbyte_cdk.sources.declarative.models.declarative_component_schema import ( 272 DefaultPaginator as DefaultPaginatorModel, 273) 274from airbyte_cdk.sources.declarative.models.declarative_component_schema import ( 275 DpathExtractor as DpathExtractorModel, 276) 277from airbyte_cdk.sources.declarative.models.declarative_component_schema import ( 278 DpathFlattenFields as DpathFlattenFieldsModel, 279) 280from airbyte_cdk.sources.declarative.models.declarative_component_schema import ( 281 DpathValidator as DpathValidatorModel, 282) 283from airbyte_cdk.sources.declarative.models.declarative_component_schema import ( 284 DynamicSchemaLoader as DynamicSchemaLoaderModel, 285) 286from airbyte_cdk.sources.declarative.models.declarative_component_schema import ( 287 DynamicStreamCheckConfig as DynamicStreamCheckConfigModel, 288) 289from airbyte_cdk.sources.declarative.models.declarative_component_schema import ( 290 ExponentialBackoffStrategy as ExponentialBackoffStrategyModel, 291) 292from airbyte_cdk.sources.declarative.models.declarative_component_schema import ( 293 FileUploader as FileUploaderModel, 294) 295from airbyte_cdk.sources.declarative.models.declarative_component_schema import ( 296 FixedWindowCallRatePolicy as FixedWindowCallRatePolicyModel, 297) 298from airbyte_cdk.sources.declarative.models.declarative_component_schema import ( 299 FlattenFields as FlattenFieldsModel, 300) 301from airbyte_cdk.sources.declarative.models.declarative_component_schema import ( 302 GroupByKeyMergeStrategy as GroupByKeyMergeStrategyModel, 303) 304from airbyte_cdk.sources.declarative.models.declarative_component_schema import ( 305 GroupingPartitionRouter as GroupingPartitionRouterModel, 306) 307from airbyte_cdk.sources.declarative.models.declarative_component_schema import ( 308 GzipDecoder as GzipDecoderModel, 309) 310from airbyte_cdk.sources.declarative.models.declarative_component_schema import ( 311 HTTPAPIBudget as HTTPAPIBudgetModel, 312) 313from airbyte_cdk.sources.declarative.models.declarative_component_schema import ( 314 HttpComponentsResolver as HttpComponentsResolverModel, 315) 316from airbyte_cdk.sources.declarative.models.declarative_component_schema import ( 317 HttpRequester as HttpRequesterModel, 318) 319from airbyte_cdk.sources.declarative.models.declarative_component_schema import ( 320 HttpRequestRegexMatcher as HttpRequestRegexMatcherModel, 321) 322from airbyte_cdk.sources.declarative.models.declarative_component_schema import ( 323 HttpResponseFilter as HttpResponseFilterModel, 324) 325from airbyte_cdk.sources.declarative.models.declarative_component_schema import ( 326 IncrementingCountCursor as IncrementingCountCursorModel, 327) 328from airbyte_cdk.sources.declarative.models.declarative_component_schema import ( 329 InlineSchemaLoader as InlineSchemaLoaderModel, 330) 331from airbyte_cdk.sources.declarative.models.declarative_component_schema import ( 332 IterableDecoder as IterableDecoderModel, 333) 334from airbyte_cdk.sources.declarative.models.declarative_component_schema import ( 335 JsonDecoder as JsonDecoderModel, 336) 337from airbyte_cdk.sources.declarative.models.declarative_component_schema import ( 338 JsonFileSchemaLoader as JsonFileSchemaLoaderModel, 339) 340from airbyte_cdk.sources.declarative.models.declarative_component_schema import ( 341 JsonItemsDecoder as JsonItemsDecoderModel, 342) 343from airbyte_cdk.sources.declarative.models.declarative_component_schema import ( 344 JsonlDecoder as JsonlDecoderModel, 345) 346from airbyte_cdk.sources.declarative.models.declarative_component_schema import ( 347 JsonSchemaPropertySelector as JsonSchemaPropertySelectorModel, 348) 349from airbyte_cdk.sources.declarative.models.declarative_component_schema import ( 350 JwtAuthenticator as JwtAuthenticatorModel, 351) 352from airbyte_cdk.sources.declarative.models.declarative_component_schema import ( 353 JwtHeaders as JwtHeadersModel, 354) 355from airbyte_cdk.sources.declarative.models.declarative_component_schema import ( 356 JwtPayload as JwtPayloadModel, 357) 358from airbyte_cdk.sources.declarative.models.declarative_component_schema import ( 359 KeysReplace as KeysReplaceModel, 360) 361from airbyte_cdk.sources.declarative.models.declarative_component_schema import ( 362 KeysToLower as KeysToLowerModel, 363) 364from airbyte_cdk.sources.declarative.models.declarative_component_schema import ( 365 KeysToSnakeCase as KeysToSnakeCaseModel, 366) 367from airbyte_cdk.sources.declarative.models.declarative_component_schema import ( 368 LegacySessionTokenAuthenticator as LegacySessionTokenAuthenticatorModel, 369) 370from airbyte_cdk.sources.declarative.models.declarative_component_schema import ( 371 LegacyToPerPartitionStateMigration as LegacyToPerPartitionStateMigrationModel, 372) 373from airbyte_cdk.sources.declarative.models.declarative_component_schema import ( 374 ListPartitionRouter as ListPartitionRouterModel, 375) 376from airbyte_cdk.sources.declarative.models.declarative_component_schema import ( 377 MinMaxDatetime as MinMaxDatetimeModel, 378) 379from airbyte_cdk.sources.declarative.models.declarative_component_schema import ( 380 MovingWindowCallRatePolicy as MovingWindowCallRatePolicyModel, 381) 382from airbyte_cdk.sources.declarative.models.declarative_component_schema import ( 383 NoAuth as NoAuthModel, 384) 385from airbyte_cdk.sources.declarative.models.declarative_component_schema import ( 386 NoPagination as NoPaginationModel, 387) 388from airbyte_cdk.sources.declarative.models.declarative_component_schema import ( 389 OAuthAuthenticator as OAuthAuthenticatorModel, 390) 391from airbyte_cdk.sources.declarative.models.declarative_component_schema import ( 392 OffsetIncrement as OffsetIncrementModel, 393) 394from airbyte_cdk.sources.declarative.models.declarative_component_schema import ( 395 PageIncrement as PageIncrementModel, 396) 397from airbyte_cdk.sources.declarative.models.declarative_component_schema import ( 398 PageSizeReduction as PageSizeReductionModel, 399) 400from airbyte_cdk.sources.declarative.models.declarative_component_schema import ( 401 PaginationReset as PaginationResetModel, 402) 403from airbyte_cdk.sources.declarative.models.declarative_component_schema import ( 404 ParametrizedComponentsResolver as ParametrizedComponentsResolverModel, 405) 406from airbyte_cdk.sources.declarative.models.declarative_component_schema import ( 407 ParentFieldPath as ParentFieldPathModel, 408) 409from airbyte_cdk.sources.declarative.models.declarative_component_schema import ( 410 ParentStreamConfig as ParentStreamConfigModel, 411) 412from airbyte_cdk.sources.declarative.models.declarative_component_schema import ( 413 PredicateValidator as PredicateValidatorModel, 414) 415from airbyte_cdk.sources.declarative.models.declarative_component_schema import ( 416 PropertiesFromEndpoint as PropertiesFromEndpointModel, 417) 418from airbyte_cdk.sources.declarative.models.declarative_component_schema import ( 419 PropertyChunking as PropertyChunkingModel, 420) 421from airbyte_cdk.sources.declarative.models.declarative_component_schema import ( 422 PropertyLimitType as PropertyLimitTypeModel, 423) 424from airbyte_cdk.sources.declarative.models.declarative_component_schema import ( 425 QueryProperties as QueryPropertiesModel, 426) 427from airbyte_cdk.sources.declarative.models.declarative_component_schema import ( 428 Rate as RateModel, 429) 430from airbyte_cdk.sources.declarative.models.declarative_component_schema import ( 431 RateLimitedMultipleTokenAuthenticator as RateLimitedMultipleTokenAuthenticatorModel, 432) 433from airbyte_cdk.sources.declarative.models.declarative_component_schema import ( 434 RecordExpander as RecordExpanderModel, 435) 436from airbyte_cdk.sources.declarative.models.declarative_component_schema import ( 437 RecordFilter as RecordFilterModel, 438) 439from airbyte_cdk.sources.declarative.models.declarative_component_schema import ( 440 RecordSelector as RecordSelectorModel, 441) 442from airbyte_cdk.sources.declarative.models.declarative_component_schema import ( 443 RefreshTokenUpdater as RefreshTokenUpdaterModel, 444) 445from airbyte_cdk.sources.declarative.models.declarative_component_schema import ( 446 RemoveFields as RemoveFieldsModel, 447) 448from airbyte_cdk.sources.declarative.models.declarative_component_schema import ( 449 RequestOption as RequestOptionModel, 450) 451from airbyte_cdk.sources.declarative.models.declarative_component_schema import ( 452 RequestPath as RequestPathModel, 453) 454from airbyte_cdk.sources.declarative.models.declarative_component_schema import ( 455 ResponseToFileExtractor as ResponseToFileExtractorModel, 456) 457from airbyte_cdk.sources.declarative.models.declarative_component_schema import ( 458 SchemaNormalization as SchemaNormalizationModel, 459) 460from airbyte_cdk.sources.declarative.models.declarative_component_schema import ( 461 SchemaTypeIdentifier as SchemaTypeIdentifierModel, 462) 463from airbyte_cdk.sources.declarative.models.declarative_component_schema import ( 464 SelectiveAuthenticator as SelectiveAuthenticatorModel, 465) 466from airbyte_cdk.sources.declarative.models.declarative_component_schema import ( 467 SessionTokenAuthenticator as SessionTokenAuthenticatorModel, 468) 469from airbyte_cdk.sources.declarative.models.declarative_component_schema import ( 470 SimpleRetriever as SimpleRetrieverModel, 471) 472from airbyte_cdk.sources.declarative.models.declarative_component_schema import Spec as SpecModel 473from airbyte_cdk.sources.declarative.models.declarative_component_schema import ( 474 StateDelegatingStream as StateDelegatingStreamModel, 475) 476from airbyte_cdk.sources.declarative.models.declarative_component_schema import ( 477 StreamConfig as StreamConfigModel, 478) 479from airbyte_cdk.sources.declarative.models.declarative_component_schema import ( 480 SubstreamPartitionRouter as SubstreamPartitionRouterModel, 481) 482from airbyte_cdk.sources.declarative.models.declarative_component_schema import ( 483 TypesMap as TypesMapModel, 484) 485from airbyte_cdk.sources.declarative.models.declarative_component_schema import ( 486 UnionPartitionRouter as UnionPartitionRouterModel, 487) 488from airbyte_cdk.sources.declarative.models.declarative_component_schema import ( 489 UnlimitedCallRatePolicy as UnlimitedCallRatePolicyModel, 490) 491from airbyte_cdk.sources.declarative.models.declarative_component_schema import ( 492 ValidateAdheresToSchema as ValidateAdheresToSchemaModel, 493) 494from airbyte_cdk.sources.declarative.models.declarative_component_schema import ValueType 495from airbyte_cdk.sources.declarative.models.declarative_component_schema import ( 496 WaitTimeFromHeader as WaitTimeFromHeaderModel, 497) 498from airbyte_cdk.sources.declarative.models.declarative_component_schema import ( 499 WaitUntilTimeFromHeader as WaitUntilTimeFromHeaderModel, 500) 501from airbyte_cdk.sources.declarative.models.declarative_component_schema import ( 502 XmlDecoder as XmlDecoderModel, 503) 504from airbyte_cdk.sources.declarative.models.declarative_component_schema import ( 505 ZipfileDecoder as ZipfileDecoderModel, 506) 507from airbyte_cdk.sources.declarative.parsers.custom_code_compiler import ( 508 INJECTED_MANIFEST, 509 AirbyteCustomCodeNotPermittedError, 510 custom_code_execution_permitted, 511) 512from airbyte_cdk.sources.declarative.parsers.stop_condition_safety import ( 513 StopConditionSafety, 514 classify_stop_condition, 515) 516from airbyte_cdk.sources.declarative.partition_routers import ( 517 CartesianProductStreamSlicer, 518 GroupingPartitionRouter, 519 ListPartitionRouter, 520 PartitionRouter, 521 SinglePartitionRouter, 522 SubstreamPartitionRouter, 523 UnionPartitionRouter, 524) 525from airbyte_cdk.sources.declarative.partition_routers.async_job_partition_router import ( 526 AsyncJobPartitionRouter, 527) 528from airbyte_cdk.sources.declarative.partition_routers.substream_partition_router import ( 529 ParentStreamConfig, 530) 531from airbyte_cdk.sources.declarative.requesters import HttpRequester, RequestOption 532from airbyte_cdk.sources.declarative.requesters.error_handlers import ( 533 CompositeErrorHandler, 534 DefaultErrorHandler, 535 HttpResponseFilter, 536) 537from airbyte_cdk.sources.declarative.requesters.error_handlers.backoff_strategies import ( 538 ConstantBackoffStrategy, 539 ExponentialBackoffStrategy, 540 WaitTimeFromHeaderBackoffStrategy, 541 WaitUntilTimeFromHeaderBackoffStrategy, 542) 543from airbyte_cdk.sources.declarative.requesters.http_job_repository import AsyncHttpJobRepository 544from airbyte_cdk.sources.declarative.requesters.paginators import ( 545 DefaultPaginator, 546 NoPagination, 547 PaginatorTestReadDecorator, 548) 549from airbyte_cdk.sources.declarative.requesters.paginators.strategies import ( 550 CursorPaginationStrategy, 551 CursorStopCondition, 552 OffsetIncrement, 553 PageIncrement, 554 StopConditionPaginationStrategyDecorator, 555) 556from airbyte_cdk.sources.declarative.requesters.query_properties import ( 557 PropertiesFromEndpoint, 558 PropertyChunking, 559 QueryProperties, 560) 561from airbyte_cdk.sources.declarative.requesters.query_properties.property_chunking import ( 562 PropertyLimitType, 563) 564from airbyte_cdk.sources.declarative.requesters.query_properties.property_selector import ( 565 JsonSchemaPropertySelector, 566) 567from airbyte_cdk.sources.declarative.requesters.query_properties.strategies import ( 568 GroupByKey, 569) 570from airbyte_cdk.sources.declarative.requesters.request_option import RequestOptionType 571from airbyte_cdk.sources.declarative.requesters.request_options import ( 572 DatetimeBasedRequestOptionsProvider, 573 DefaultRequestOptionsProvider, 574 InterpolatedRequestOptionsProvider, 575 RequestOptionsProvider, 576) 577from airbyte_cdk.sources.declarative.requesters.request_options.per_partition_request_option_provider import ( 578 PerPartitionRequestOptionsProvider, 579) 580from airbyte_cdk.sources.declarative.requesters.request_path import RequestPath 581from airbyte_cdk.sources.declarative.requesters.requester import HttpMethod, Requester 582from airbyte_cdk.sources.declarative.resolvers import ( 583 ComponentMappingDefinition, 584 ConfigComponentsResolver, 585 HttpComponentsResolver, 586 ParametrizedComponentsResolver, 587 StreamConfig, 588 StreamParametersDefinition, 589) 590from airbyte_cdk.sources.declarative.retrievers import ( 591 AsyncRetriever, 592 LazySimpleRetriever, 593 SimpleRetriever, 594) 595from airbyte_cdk.sources.declarative.retrievers.file_uploader import ( 596 ConnectorBuilderFileUploader, 597 DefaultFileUploader, 598 FileUploader, 599 LocalFileSystemFileWriter, 600 NoopFileWriter, 601) 602from airbyte_cdk.sources.declarative.retrievers.page_size_reducer import ( 603 PageSizeReduction, 604 PageSizeResetPolicy, 605) 606from airbyte_cdk.sources.declarative.retrievers.pagination_tracker import PaginationTracker 607from airbyte_cdk.sources.declarative.retrievers.request_window_splitting import ( 608 RequestWindowSplitting, 609) 610from airbyte_cdk.sources.declarative.schema import ( 611 ComplexFieldType, 612 DefaultSchemaLoader, 613 DynamicSchemaLoader, 614 InlineSchemaLoader, 615 JsonFileSchemaLoader, 616 SchemaLoader, 617 SchemaTypeIdentifier, 618 TypesMap, 619) 620from airbyte_cdk.sources.declarative.schema.caching_schema_loader_decorator import ( 621 CachingSchemaLoaderDecorator, 622) 623from airbyte_cdk.sources.declarative.schema.composite_schema_loader import CompositeSchemaLoader 624from airbyte_cdk.sources.declarative.spec import ConfigMigration, Spec 625from airbyte_cdk.sources.declarative.stream_slicers import ( 626 StreamSlicer, 627 StreamSlicerTestReadDecorator, 628) 629from airbyte_cdk.sources.declarative.stream_slicers.declarative_partition_generator import ( 630 DeclarativePartitionFactory, 631 StreamSlicerPartitionGenerator, 632) 633from airbyte_cdk.sources.declarative.transformations import ( 634 AddFields, 635 RecordTransformation, 636 RemoveFields, 637) 638from airbyte_cdk.sources.declarative.transformations.add_fields import AddedFieldDefinition 639from airbyte_cdk.sources.declarative.transformations.config_transformations import ( 640 ConfigAddFields, 641 ConfigRemapField, 642 ConfigRemoveFields, 643) 644from airbyte_cdk.sources.declarative.transformations.config_transformations.config_transformation import ( 645 ConfigTransformation, 646) 647from airbyte_cdk.sources.declarative.transformations.dpath_flatten_fields import ( 648 DpathFlattenFields, 649 KeyTransformation, 650) 651from airbyte_cdk.sources.declarative.transformations.flatten_fields import ( 652 FlattenFields, 653) 654from airbyte_cdk.sources.declarative.transformations.keys_replace_transformation import ( 655 KeysReplaceTransformation, 656) 657from airbyte_cdk.sources.declarative.transformations.keys_to_lower_transformation import ( 658 KeysToLowerTransformation, 659) 660from airbyte_cdk.sources.declarative.transformations.keys_to_snake_transformation import ( 661 KeysToSnakeCaseTransformation, 662) 663from airbyte_cdk.sources.declarative.validators import ( 664 DpathValidator, 665 PredicateValidator, 666 ValidateAdheresToSchema, 667) 668from airbyte_cdk.sources.http_logger import format_http_message 669from airbyte_cdk.sources.message import ( 670 InMemoryMessageRepository, 671 LogAppenderMessageRepositoryDecorator, 672 MessageRepository, 673 NoopMessageRepository, 674) 675from airbyte_cdk.sources.message.repository import StateFilteringMessageRepository 676from airbyte_cdk.sources.streams import NO_CURSOR_STATE_KEY 677from airbyte_cdk.sources.streams.call_rate import ( 678 APIBudget, 679 FixedWindowCallRatePolicy, 680 HttpAPIBudget, 681 HttpRequestRegexMatcher, 682 MovingWindowCallRatePolicy, 683 Rate, 684 UnlimitedCallRatePolicy, 685) 686from airbyte_cdk.sources.streams.concurrent.abstract_stream import AbstractStream 687from airbyte_cdk.sources.streams.concurrent.clamping import ( 688 ClampingEndProvider, 689 ClampingStrategy, 690 DayClampingStrategy, 691 MonthClampingStrategy, 692 NoClamping, 693 WeekClampingStrategy, 694 Weekday, 695) 696from airbyte_cdk.sources.streams.concurrent.cursor import ( 697 ConcurrentCursor, 698 Cursor, 699 CursorField, 700 FinalStateCursor, 701) 702from airbyte_cdk.sources.streams.concurrent.default_stream import DefaultStream 703from airbyte_cdk.sources.streams.concurrent.helpers import get_primary_key_from_stream 704from airbyte_cdk.sources.streams.concurrent.partitions.stream_slicer import ( 705 StreamSlicer as ConcurrentStreamSlicer, 706) 707from airbyte_cdk.sources.streams.concurrent.state_converters.datetime_stream_state_converter import ( 708 CustomFormatConcurrentStreamStateConverter, 709 DateTimeStreamStateConverter, 710) 711from airbyte_cdk.sources.streams.concurrent.state_converters.incrementing_count_stream_state_converter import ( 712 IncrementingCountStreamStateConverter, 713) 714from airbyte_cdk.sources.streams.http.error_handlers.response_models import ResponseAction 715from airbyte_cdk.sources.types import Config 716from airbyte_cdk.sources.utils.transform import TransformConfig, TypeTransformer 717from airbyte_cdk.utils.traced_exception import AirbyteTracedException 718 719ComponentDefinition = Mapping[str, Any] 720 721SCHEMA_TRANSFORMER_TYPE_MAPPING = { 722 SchemaNormalizationModel.None_: TransformConfig.NoTransform, 723 SchemaNormalizationModel.Default: TransformConfig.DefaultSchemaNormalization, 724} 725_NO_STREAM_SLICING = SinglePartitionRouter(parameters={}) 726 727# Ideally this should use the value defined in ConcurrentDeclarativeSource, but 728# this would be a circular import 729MAX_SLICES = 5 730 731# Smallest duration each `DatetimeParser`/`strftime` directive can distinguish, used by 732# `_smallest_datetime_format_unit` to check `datetime_format` can represent `cursor_granularity`. A format with 733# no entry here (e.g. a bare `%Y`/`%m`) is treated as "cannot determine" rather than rejected. 734_DATETIME_FORMAT_DIRECTIVE_GRANULARITY: List[Tuple[str, datetime.timedelta]] = [ 735 ("%epoch_microseconds", datetime.timedelta(microseconds=1)), 736 ("%s_as_float", datetime.timedelta(microseconds=1)), 737 ("%f", datetime.timedelta(microseconds=1)), 738 ("%_ms", datetime.timedelta(milliseconds=1)), 739 ("%ms", datetime.timedelta(milliseconds=1)), 740 ("%S", datetime.timedelta(seconds=1)), 741 ("%s", datetime.timedelta(seconds=1)), 742 ("%M", datetime.timedelta(minutes=1)), 743 ("%H", datetime.timedelta(hours=1)), 744 ("%I", datetime.timedelta(hours=1)), 745 ("%d", datetime.timedelta(days=1)), 746] 747 748LOGGER = logging.getLogger(f"airbyte.model_to_component_factory") 749 750 751class ModelToComponentFactory: 752 EPOCH_DATETIME_FORMAT = "%s" 753 754 def __init__( 755 self, 756 limit_pages_fetched_per_slice: Optional[int] = None, 757 limit_slices_fetched: Optional[int] = None, 758 emit_connector_builder_messages: bool = False, 759 disable_retries: bool = False, 760 disable_cache: bool = False, 761 message_repository: Optional[MessageRepository] = None, 762 connector_state_manager: Optional[ConnectorStateManager] = None, 763 max_concurrent_async_job_count: Optional[int] = None, 764 configured_catalog: Optional[ConfiguredAirbyteCatalog] = None, 765 api_budget: Optional[APIBudget] = None, 766 rate_limited_authenticators: Optional[ 767 Dict[str, RateLimitedMultipleTokenAuthenticator] 768 ] = None, 769 custom_components_trusted: bool = True, 770 ): 771 self._init_mappings() 772 self._custom_components_trusted = custom_components_trusted 773 self._limit_pages_fetched_per_slice = limit_pages_fetched_per_slice 774 self._limit_slices_fetched = limit_slices_fetched 775 self._emit_connector_builder_messages = emit_connector_builder_messages 776 self._disable_retries = disable_retries 777 self._disable_cache = disable_cache 778 self._message_repository = message_repository or InMemoryMessageRepository( 779 self._evaluate_log_level(emit_connector_builder_messages) 780 ) 781 self._stream_name_to_configured_stream = self._create_stream_name_to_configured_stream( 782 configured_catalog 783 ) 784 self._connector_state_manager = connector_state_manager or ConnectorStateManager() 785 self._api_budget: Optional[Union[APIBudget]] = api_budget 786 # Shared instances so all streams see the same token quota counters (like api_budget) 787 self._rate_limited_authenticators: Dict[str, RateLimitedMultipleTokenAuthenticator] = ( 788 rate_limited_authenticators if rate_limited_authenticators is not None else {} 789 ) 790 self._job_tracker: JobTracker = JobTracker(max_concurrent_async_job_count or 1) 791 # placeholder for deprecation warnings 792 self._collected_deprecation_logs: List[ConnectorBuilderLogMessage] = [] 793 794 def _init_mappings(self) -> None: 795 self.PYDANTIC_MODEL_TO_CONSTRUCTOR: Mapping[Type[BaseModel], Callable[..., Any]] = { 796 AddedFieldDefinitionModel: self.create_added_field_definition, 797 AddFieldsModel: self.create_add_fields, 798 ApiKeyAuthenticatorModel: self.create_api_key_authenticator, 799 BasicHttpAuthenticatorModel: self.create_basic_http_authenticator, 800 BearerAuthenticatorModel: self.create_bearer_authenticator, 801 CheckStreamModel: self.create_check_stream, 802 DynamicStreamCheckConfigModel: self.create_dynamic_stream_check_config, 803 CheckDynamicStreamModel: self.create_check_dynamic_stream, 804 CompositeErrorHandlerModel: self.create_composite_error_handler, 805 ConcurrencyLevelModel: self.create_concurrency_level, 806 ConfigMigrationModel: self.create_config_migration, 807 ConfigAddFieldsModel: self.create_config_add_fields, 808 ConfigRemapFieldModel: self.create_config_remap_field, 809 ConfigRemoveFieldsModel: self.create_config_remove_fields, 810 ConstantBackoffStrategyModel: self.create_constant_backoff_strategy, 811 CsvDecoderModel: self.create_csv_decoder, 812 CursorPaginationModel: self.create_cursor_pagination, 813 CustomAuthenticatorModel: self.create_custom_component, 814 CustomBackoffStrategyModel: self.create_custom_component, 815 CustomDecoderModel: self.create_custom_component, 816 CustomErrorHandlerModel: self.create_custom_component, 817 CustomRecordExtractorModel: self.create_custom_component, 818 CustomRecordFilterModel: self.create_custom_component, 819 CustomRequesterModel: self.create_custom_component, 820 CustomRetrieverModel: self.create_custom_component, 821 CustomSchemaLoader: self.create_custom_component, 822 CustomSchemaNormalizationModel: self.create_custom_component, 823 CustomStateMigration: self.create_custom_component, 824 CustomPaginationStrategyModel: self.create_custom_component, 825 CustomPartitionRouterModel: self.create_custom_component, 826 CustomTransformationModel: self.create_custom_component, 827 CustomValidationStrategyModel: self.create_custom_component, 828 CustomConfigTransformationModel: self.create_custom_component, 829 DeclarativeStreamModel: self.create_default_stream, 830 DefaultErrorHandlerModel: self.create_default_error_handler, 831 DefaultPaginatorModel: self.create_default_paginator, 832 CombinedExtractorModel: self.create_combined_extractor, 833 DpathExtractorModel: self.create_dpath_extractor, 834 DpathValidatorModel: self.create_dpath_validator, 835 ResponseToFileExtractorModel: self.create_response_to_file_extractor, 836 ExponentialBackoffStrategyModel: self.create_exponential_backoff_strategy, 837 SessionTokenAuthenticatorModel: self.create_session_token_authenticator, 838 GroupByKeyMergeStrategyModel: self.create_group_by_key, 839 HttpRequesterModel: self.create_http_requester, 840 HttpResponseFilterModel: self.create_http_response_filter, 841 InlineSchemaLoaderModel: self.create_inline_schema_loader, 842 JsonDecoderModel: self.create_json_decoder, 843 JsonItemsDecoderModel: self.create_json_items_decoder, 844 JsonlDecoderModel: self.create_jsonl_decoder, 845 JsonSchemaPropertySelectorModel: self.create_json_schema_property_selector, 846 GzipDecoderModel: self.create_gzip_decoder, 847 KeysToLowerModel: self.create_keys_to_lower_transformation, 848 KeysToSnakeCaseModel: self.create_keys_to_snake_transformation, 849 KeysReplaceModel: self.create_keys_replace_transformation, 850 FlattenFieldsModel: self.create_flatten_fields, 851 DpathFlattenFieldsModel: self.create_dpath_flatten_fields, 852 IterableDecoderModel: self.create_iterable_decoder, 853 XmlDecoderModel: self.create_xml_decoder, 854 JsonFileSchemaLoaderModel: self.create_json_file_schema_loader, 855 DynamicSchemaLoaderModel: self.create_dynamic_schema_loader, 856 SchemaTypeIdentifierModel: self.create_schema_type_identifier, 857 TypesMapModel: self.create_types_map, 858 ComplexFieldTypeModel: self.create_complex_field_type, 859 JwtAuthenticatorModel: self.create_jwt_authenticator, 860 LegacyToPerPartitionStateMigrationModel: self.create_legacy_to_per_partition_state_migration, 861 ListPartitionRouterModel: self.create_list_partition_router, 862 MinMaxDatetimeModel: self.create_min_max_datetime, 863 NoAuthModel: self.create_no_auth, 864 NoPaginationModel: self.create_no_pagination, 865 OAuthAuthenticatorModel: self.create_oauth_authenticator, 866 OffsetIncrementModel: self.create_offset_increment, 867 PageIncrementModel: self.create_page_increment, 868 ParentStreamConfigModel: self.create_parent_stream_config_with_substream_wrapper, 869 PredicateValidatorModel: self.create_predicate_validator, 870 PropertiesFromEndpointModel: self.create_properties_from_endpoint, 871 PropertyChunkingModel: self.create_property_chunking, 872 QueryPropertiesModel: self.create_query_properties, 873 ParentFieldPathModel: self.create_parent_field_path, 874 RecordExpanderModel: self.create_record_expander, 875 RecordFilterModel: self.create_record_filter, 876 RecordSelectorModel: self.create_record_selector, 877 RemoveFieldsModel: self.create_remove_fields, 878 RequestPathModel: self.create_request_path, 879 RequestOptionModel: self.create_request_option, 880 LegacySessionTokenAuthenticatorModel: self.create_legacy_session_token_authenticator, 881 SelectiveAuthenticatorModel: self.create_selective_authenticator, 882 SimpleRetrieverModel: self.create_simple_retriever, 883 StateDelegatingStreamModel: self.create_state_delegating_stream, 884 SpecModel: self.create_spec, 885 SubstreamPartitionRouterModel: self.create_substream_partition_router, 886 ValidateAdheresToSchemaModel: self.create_validate_adheres_to_schema, 887 WaitTimeFromHeaderModel: self.create_wait_time_from_header, 888 WaitUntilTimeFromHeaderModel: self.create_wait_until_time_from_header, 889 AsyncRetrieverModel: self.create_async_retriever, 890 HttpComponentsResolverModel: self.create_http_components_resolver, 891 ConfigComponentsResolverModel: self.create_config_components_resolver, 892 ParametrizedComponentsResolverModel: self.create_parametrized_components_resolver, 893 StreamConfigModel: self.create_stream_config, 894 ComponentMappingDefinitionModel: self.create_components_mapping_definition, 895 ZipfileDecoderModel: self.create_zipfile_decoder, 896 HTTPAPIBudgetModel: self.create_http_api_budget, 897 FileUploaderModel: self.create_file_uploader, 898 FixedWindowCallRatePolicyModel: self.create_fixed_window_call_rate_policy, 899 MovingWindowCallRatePolicyModel: self.create_moving_window_call_rate_policy, 900 UnlimitedCallRatePolicyModel: self.create_unlimited_call_rate_policy, 901 RateModel: self.create_rate, 902 HttpRequestRegexMatcherModel: self.create_http_request_matcher, 903 RateLimitedMultipleTokenAuthenticatorModel: self.create_rate_limited_multiple_token_authenticator, 904 GroupingPartitionRouterModel: self.create_grouping_partition_router, 905 UnionPartitionRouterModel: self.create_union_partition_router, 906 } 907 908 # Needed for the case where we need to perform a second parse on the fields of a custom component 909 self.TYPE_NAME_TO_MODEL = {cls.__name__: cls for cls in self.PYDANTIC_MODEL_TO_CONSTRUCTOR} 910 911 @staticmethod 912 def _create_stream_name_to_configured_stream( 913 configured_catalog: Optional[ConfiguredAirbyteCatalog], 914 ) -> Mapping[str, ConfiguredAirbyteStream]: 915 return ( 916 {stream.stream.name: stream for stream in configured_catalog.streams} 917 if configured_catalog 918 else {} 919 ) 920 921 def create_component( 922 self, 923 model_type: Type[BaseModel], 924 component_definition: ComponentDefinition, 925 config: Config, 926 **kwargs: Any, 927 ) -> Any: 928 """ 929 Takes a given Pydantic model type and Mapping representing a component definition and creates a declarative component and 930 subcomponents which will be used at runtime. This is done by first parsing the mapping into a Pydantic model and then creating 931 creating declarative components from that model. 932 933 :param model_type: The type of declarative component that is being initialized 934 :param component_definition: The mapping that represents a declarative component 935 :param config: The connector config that is provided by the customer 936 :return: The declarative component to be used at runtime 937 """ 938 939 component_type = component_definition.get("type") 940 if component_definition.get("type") != model_type.__name__: 941 raise ValueError( 942 f"Expected manifest component of type {model_type.__name__}, but received {component_type} instead" 943 ) 944 945 declarative_component_model = model_type.parse_obj(component_definition) 946 947 if not isinstance(declarative_component_model, model_type): 948 raise ValueError( 949 f"Expected {model_type.__name__} component, but received {declarative_component_model.__class__.__name__}" 950 ) 951 952 return self._create_component_from_model( 953 model=declarative_component_model, config=config, **kwargs 954 ) 955 956 def _create_component_from_model(self, model: BaseModel, config: Config, **kwargs: Any) -> Any: 957 if model.__class__ not in self.PYDANTIC_MODEL_TO_CONSTRUCTOR: 958 raise ValueError( 959 f"{model.__class__} with attributes {model} is not a valid component type" 960 ) 961 component_constructor = self.PYDANTIC_MODEL_TO_CONSTRUCTOR.get(model.__class__) 962 if not component_constructor: 963 raise ValueError(f"Could not find constructor for {model.__class__}") 964 965 # collect deprecation warnings for supported models. 966 if isinstance(model, BaseModelWithDeprecations): 967 self._collect_model_deprecations(model) 968 969 return component_constructor(model=model, config=config, **kwargs) 970 971 def get_model_deprecations(self) -> List[ConnectorBuilderLogMessage]: 972 """ 973 Returns the deprecation warnings that were collected during the creation of components. 974 """ 975 return self._collected_deprecation_logs 976 977 def _collect_model_deprecations(self, model: BaseModelWithDeprecations) -> None: 978 """ 979 Collects deprecation logs from the given model and appends any new logs to the internal collection. 980 981 This method checks if the provided model has deprecation logs (identified by the presence of the DEPRECATION_LOGS_TAG attribute and a non-None `_deprecation_logs` property). It iterates through each deprecation log in the model and appends it to the `_collected_deprecation_logs` list if it has not already been collected, ensuring that duplicate logs are avoided. 982 983 Args: 984 model (BaseModelWithDeprecations): The model instance from which to collect deprecation logs. 985 """ 986 if hasattr(model, DEPRECATION_LOGS_TAG) and model._deprecation_logs is not None: 987 for log in model._deprecation_logs: 988 # avoid duplicates for deprecation logs observed. 989 if log not in self._collected_deprecation_logs: 990 self._collected_deprecation_logs.append(log) 991 992 def create_config_migration( 993 self, model: ConfigMigrationModel, config: Config 994 ) -> ConfigMigration: 995 transformations: List[ConfigTransformation] = [ 996 self._create_component_from_model(transformation, config) 997 for transformation in model.transformations 998 ] 999 1000 return ConfigMigration( 1001 description=model.description, 1002 transformations=transformations, 1003 ) 1004 1005 def create_config_add_fields( 1006 self, model: ConfigAddFieldsModel, config: Config, **kwargs: Any 1007 ) -> ConfigAddFields: 1008 fields = [self._create_component_from_model(field, config) for field in model.fields] 1009 return ConfigAddFields( 1010 fields=fields, 1011 condition=model.condition or "", 1012 ) 1013 1014 @staticmethod 1015 def create_config_remove_fields( 1016 model: ConfigRemoveFieldsModel, config: Config, **kwargs: Any 1017 ) -> ConfigRemoveFields: 1018 return ConfigRemoveFields( 1019 field_pointers=model.field_pointers, 1020 condition=model.condition or "", 1021 ) 1022 1023 @staticmethod 1024 def create_config_remap_field( 1025 model: ConfigRemapFieldModel, config: Config, **kwargs: Any 1026 ) -> ConfigRemapField: 1027 mapping = cast(Mapping[str, Any], model.map) 1028 return ConfigRemapField( 1029 map=mapping, 1030 field_path=model.field_path, 1031 config=config, 1032 ) 1033 1034 def create_dpath_validator(self, model: DpathValidatorModel, config: Config) -> DpathValidator: 1035 strategy = self._create_component_from_model(model.validation_strategy, config) 1036 1037 return DpathValidator( 1038 field_path=model.field_path, 1039 strategy=strategy, 1040 ) 1041 1042 def create_predicate_validator( 1043 self, model: PredicateValidatorModel, config: Config 1044 ) -> PredicateValidator: 1045 strategy = self._create_component_from_model(model.validation_strategy, config) 1046 1047 return PredicateValidator( 1048 value=model.value, 1049 strategy=strategy, 1050 ) 1051 1052 @staticmethod 1053 def create_validate_adheres_to_schema( 1054 model: ValidateAdheresToSchemaModel, config: Config, **kwargs: Any 1055 ) -> ValidateAdheresToSchema: 1056 base_schema = cast(Mapping[str, Any], model.base_schema) 1057 return ValidateAdheresToSchema( 1058 schema=base_schema, 1059 ) 1060 1061 @staticmethod 1062 def create_added_field_definition( 1063 model: AddedFieldDefinitionModel, config: Config, **kwargs: Any 1064 ) -> AddedFieldDefinition: 1065 interpolated_value = InterpolatedString.create( 1066 model.value, parameters=model.parameters or {} 1067 ) 1068 return AddedFieldDefinition( 1069 path=model.path, 1070 value=interpolated_value, 1071 value_type=ModelToComponentFactory._json_schema_type_name_to_type(model.value_type), 1072 parameters=model.parameters or {}, 1073 ) 1074 1075 def create_add_fields(self, model: AddFieldsModel, config: Config, **kwargs: Any) -> AddFields: 1076 added_field_definitions = [ 1077 self._create_component_from_model( 1078 model=added_field_definition_model, 1079 value_type=ModelToComponentFactory._json_schema_type_name_to_type( 1080 added_field_definition_model.value_type 1081 ), 1082 config=config, 1083 ) 1084 for added_field_definition_model in model.fields 1085 ] 1086 return AddFields( 1087 fields=added_field_definitions, 1088 condition=model.condition or "", 1089 parameters=model.parameters or {}, 1090 ) 1091 1092 def create_keys_to_lower_transformation( 1093 self, model: KeysToLowerModel, config: Config, **kwargs: Any 1094 ) -> KeysToLowerTransformation: 1095 return KeysToLowerTransformation() 1096 1097 def create_keys_to_snake_transformation( 1098 self, model: KeysToSnakeCaseModel, config: Config, **kwargs: Any 1099 ) -> KeysToSnakeCaseTransformation: 1100 return KeysToSnakeCaseTransformation() 1101 1102 def create_keys_replace_transformation( 1103 self, model: KeysReplaceModel, config: Config, **kwargs: Any 1104 ) -> KeysReplaceTransformation: 1105 return KeysReplaceTransformation( 1106 old=model.old, new=model.new, parameters=model.parameters or {} 1107 ) 1108 1109 def create_flatten_fields( 1110 self, model: FlattenFieldsModel, config: Config, **kwargs: Any 1111 ) -> FlattenFields: 1112 return FlattenFields( 1113 flatten_lists=model.flatten_lists if model.flatten_lists is not None else True 1114 ) 1115 1116 def create_dpath_flatten_fields( 1117 self, model: DpathFlattenFieldsModel, config: Config, **kwargs: Any 1118 ) -> DpathFlattenFields: 1119 model_field_path: List[Union[InterpolatedString, str]] = [x for x in model.field_path] 1120 key_transformation = ( 1121 KeyTransformation( 1122 config=config, 1123 prefix=model.key_transformation.prefix, 1124 suffix=model.key_transformation.suffix, 1125 parameters=model.parameters or {}, 1126 ) 1127 if model.key_transformation is not None 1128 else None 1129 ) 1130 return DpathFlattenFields( 1131 config=config, 1132 field_path=model_field_path, 1133 delete_origin_value=model.delete_origin_value 1134 if model.delete_origin_value is not None 1135 else False, 1136 replace_record=model.replace_record if model.replace_record is not None else False, 1137 key_transformation=key_transformation, 1138 parameters=model.parameters or {}, 1139 ) 1140 1141 @staticmethod 1142 def _json_schema_type_name_to_type(value_type: Optional[ValueType]) -> Optional[Type[Any]]: 1143 if not value_type: 1144 return None 1145 names_to_types = { 1146 ValueType.string: str, 1147 ValueType.number: float, 1148 ValueType.integer: int, 1149 ValueType.boolean: bool, 1150 } 1151 return names_to_types[value_type] 1152 1153 def create_api_key_authenticator( 1154 self, 1155 model: ApiKeyAuthenticatorModel, 1156 config: Config, 1157 token_provider: Optional[TokenProvider] = None, 1158 **kwargs: Any, 1159 ) -> ApiKeyAuthenticator: 1160 if model.inject_into is None and model.header is None: 1161 raise ValueError( 1162 "Expected either inject_into or header to be set for ApiKeyAuthenticator" 1163 ) 1164 1165 if model.inject_into is not None and model.header is not None: 1166 raise ValueError( 1167 "inject_into and header cannot be set both for ApiKeyAuthenticator - remove the deprecated header option" 1168 ) 1169 1170 if token_provider is not None and model.api_token != "": 1171 raise ValueError( 1172 "If token_provider is set, api_token is ignored and has to be set to empty string." 1173 ) 1174 1175 request_option = ( 1176 self._create_component_from_model( 1177 model.inject_into, config, parameters=model.parameters or {} 1178 ) 1179 if model.inject_into 1180 else RequestOption( 1181 inject_into=RequestOptionType.header, 1182 field_name=model.header or "", 1183 parameters=model.parameters or {}, 1184 ) 1185 ) 1186 1187 return ApiKeyAuthenticator( 1188 token_provider=( 1189 token_provider 1190 if token_provider is not None 1191 else InterpolatedStringTokenProvider( 1192 api_token=model.api_token or "", 1193 config=config, 1194 parameters=model.parameters or {}, 1195 ) 1196 ), 1197 request_option=request_option, 1198 config=config, 1199 parameters=model.parameters or {}, 1200 ) 1201 1202 def create_legacy_to_per_partition_state_migration( 1203 self, 1204 model: LegacyToPerPartitionStateMigrationModel, 1205 config: Mapping[str, Any], 1206 declarative_stream: DeclarativeStreamModel, 1207 ) -> LegacyToPerPartitionStateMigration: 1208 retriever = declarative_stream.retriever 1209 if not isinstance(retriever, (SimpleRetrieverModel, AsyncRetrieverModel)): 1210 raise ValueError( 1211 f"LegacyToPerPartitionStateMigrations can only be applied on a DeclarativeStream with a SimpleRetriever or AsyncRetriever. Got {type(retriever)}" 1212 ) 1213 partition_router = retriever.partition_router 1214 if not isinstance( 1215 partition_router, 1216 ( 1217 SubstreamPartitionRouterModel, 1218 CustomPartitionRouterModel, 1219 UnionPartitionRouterModel, 1220 ), 1221 ): 1222 raise ValueError( 1223 f"LegacyToPerPartitionStateMigrations can only be applied on a SimpleRetriever with a SubstreamPartitionRouter, UnionPartitionRouter or CustomPartitionRouter. Got {type(partition_router)}" 1224 ) 1225 if not isinstance(partition_router, UnionPartitionRouterModel) and not hasattr( 1226 partition_router, "parent_stream_configs" 1227 ): 1228 raise ValueError( 1229 "LegacyToPerPartitionStateMigrations can only be applied with a parent stream configuration." 1230 ) 1231 1232 if not hasattr(declarative_stream, "incremental_sync"): 1233 raise ValueError( 1234 "LegacyToPerPartitionStateMigrations can only be applied with an incremental_sync configuration." 1235 ) 1236 1237 return LegacyToPerPartitionStateMigration( 1238 partition_router, # type: ignore # was already checked above 1239 declarative_stream.incremental_sync, # type: ignore # was already checked. Migration can be applied only to incremental streams. 1240 config, 1241 declarative_stream.parameters, # type: ignore # different type is expected here Mapping[str, Any], got Dict[str, Any] 1242 ) 1243 1244 def create_session_token_authenticator( 1245 self, model: SessionTokenAuthenticatorModel, config: Config, *, name: str, **kwargs: Any 1246 ) -> Union[ApiKeyAuthenticator, BearerAuthenticator]: 1247 self._reject_reduce_page_size_action( 1248 model.login_requester, f"`login_requester` of the SessionTokenAuthenticator of {name}" 1249 ) 1250 self._reject_split_request_window_action( 1251 model.login_requester, f"`login_requester` of the SessionTokenAuthenticator of {name}" 1252 ) 1253 decoder = ( 1254 self._create_component_from_model(model=model.decoder, config=config) 1255 if model.decoder 1256 else JsonDecoder(parameters={}) 1257 ) 1258 login_requester = self._create_component_from_model( 1259 model=model.login_requester, 1260 config=config, 1261 name=f"{name}_login_requester", 1262 decoder=decoder, 1263 ) 1264 token_provider = SessionTokenProvider( 1265 login_requester=login_requester, 1266 session_token_path=model.session_token_path, 1267 expiration_duration=parse_duration(model.expiration_duration) 1268 if model.expiration_duration 1269 else None, 1270 parameters=model.parameters or {}, 1271 message_repository=self._message_repository, 1272 decoder=decoder, 1273 ) 1274 if model.request_authentication.type == "Bearer": 1275 return ModelToComponentFactory.create_bearer_authenticator( 1276 BearerAuthenticatorModel(type="BearerAuthenticator", api_token=""), # type: ignore # $parameters has a default value 1277 config, 1278 token_provider=token_provider, 1279 ) 1280 else: 1281 # Get the api_token template if specified, default to just the session token 1282 api_token_template = ( 1283 getattr(model.request_authentication, "api_token", None) or "{{ session_token }}" 1284 ) 1285 final_token_provider: TokenProvider = InterpolatedSessionTokenProvider( 1286 config=config, 1287 api_token=api_token_template, 1288 session_token_provider=token_provider, 1289 parameters=model.parameters or {}, 1290 ) 1291 return self.create_api_key_authenticator( 1292 ApiKeyAuthenticatorModel( 1293 type="ApiKeyAuthenticator", 1294 api_token="", 1295 inject_into=model.request_authentication.inject_into, 1296 ), # type: ignore # $parameters and headers default to None 1297 config=config, 1298 token_provider=final_token_provider, 1299 ) 1300 1301 @staticmethod 1302 def create_basic_http_authenticator( 1303 model: BasicHttpAuthenticatorModel, config: Config, **kwargs: Any 1304 ) -> BasicHttpAuthenticator: 1305 return BasicHttpAuthenticator( 1306 password=model.password or "", 1307 username=model.username, 1308 config=config, 1309 parameters=model.parameters or {}, 1310 ) 1311 1312 @staticmethod 1313 def create_bearer_authenticator( 1314 model: BearerAuthenticatorModel, 1315 config: Config, 1316 token_provider: Optional[TokenProvider] = None, 1317 **kwargs: Any, 1318 ) -> BearerAuthenticator: 1319 if token_provider is not None and model.api_token != "": 1320 raise ValueError( 1321 "If token_provider is set, api_token is ignored and has to be set to empty string." 1322 ) 1323 return BearerAuthenticator( 1324 token_provider=( 1325 token_provider 1326 if token_provider is not None 1327 else InterpolatedStringTokenProvider( 1328 api_token=model.api_token or "", 1329 config=config, 1330 parameters=model.parameters or {}, 1331 ) 1332 ), 1333 config=config, 1334 parameters=model.parameters or {}, 1335 ) 1336 1337 @staticmethod 1338 def create_dynamic_stream_check_config( 1339 model: DynamicStreamCheckConfigModel, config: Config, **kwargs: Any 1340 ) -> DynamicStreamCheckConfig: 1341 return DynamicStreamCheckConfig( 1342 dynamic_stream_name=model.dynamic_stream_name, 1343 stream_count=model.stream_count, 1344 ) 1345 1346 def create_check_stream( 1347 self, model: CheckStreamModel, config: Config, **kwargs: Any 1348 ) -> CheckStream: 1349 if model.dynamic_streams_check_configs is None and model.stream_names is None: 1350 raise ValueError( 1351 "Expected either stream_names or dynamic_streams_check_configs to be set for CheckStream" 1352 ) 1353 1354 dynamic_streams_check_configs = ( 1355 [ 1356 self._create_component_from_model(model=dynamic_stream_check_config, config=config) 1357 for dynamic_stream_check_config in model.dynamic_streams_check_configs 1358 ] 1359 if model.dynamic_streams_check_configs 1360 else [] 1361 ) 1362 1363 # `model.config_overrides` is deliberately not read here. The source applies it around the whole 1364 # check operation (`ConcurrentDeclarativeSource._config_overridden_for_check`), which is what makes 1365 # it work for every checker type rather than only this one. Do not wire it in a second time. 1366 return CheckStream( 1367 stream_names=model.stream_names or [], 1368 dynamic_streams_check_configs=dynamic_streams_check_configs, 1369 parameters={}, 1370 ) 1371 1372 @staticmethod 1373 def create_check_dynamic_stream( 1374 model: CheckDynamicStreamModel, config: Config, **kwargs: Any 1375 ) -> CheckDynamicStream: 1376 assert model.use_check_availability is not None # for mypy 1377 1378 use_check_availability = model.use_check_availability 1379 1380 # See `create_check_stream`: `model.config_overrides` is applied by the source, not here. 1381 return CheckDynamicStream( 1382 stream_count=model.stream_count, 1383 use_check_availability=use_check_availability, 1384 parameters={}, 1385 ) 1386 1387 def create_composite_error_handler( 1388 self, model: CompositeErrorHandlerModel, config: Config, **kwargs: Any 1389 ) -> CompositeErrorHandler: 1390 error_handlers = [ 1391 self._create_component_from_model(model=error_handler_model, config=config) 1392 for error_handler_model in model.error_handlers 1393 ] 1394 return CompositeErrorHandler( 1395 error_handlers=error_handlers, parameters=model.parameters or {} 1396 ) 1397 1398 @staticmethod 1399 def create_concurrency_level( 1400 model: ConcurrencyLevelModel, config: Config, **kwargs: Any 1401 ) -> ConcurrencyLevel: 1402 return ConcurrencyLevel( 1403 default_concurrency=model.default_concurrency, 1404 max_concurrency=model.max_concurrency, 1405 config=config, 1406 parameters={}, 1407 ) 1408 1409 @staticmethod 1410 def apply_stream_state_migrations( 1411 stream_state_migrations: List[Any] | None, stream_state: MutableMapping[str, Any] 1412 ) -> MutableMapping[str, Any]: 1413 if stream_state_migrations: 1414 for state_migration in stream_state_migrations: 1415 if state_migration.should_migrate(stream_state): 1416 # The state variable is expected to be mutable but the migrate method returns an immutable mapping. 1417 stream_state = dict(state_migration.migrate(stream_state)) 1418 return stream_state 1419 1420 def create_concurrent_cursor_from_datetime_based_cursor( 1421 self, 1422 model_type: Type[BaseModel], 1423 component_definition: ComponentDefinition, 1424 stream_name: str, 1425 stream_namespace: Optional[str], 1426 stream_state: MutableMapping[str, Any], 1427 config: Config, 1428 message_repository: Optional[MessageRepository] = None, 1429 runtime_lookback_window: Optional[datetime.timedelta] = None, 1430 **kwargs: Any, 1431 ) -> ConcurrentCursor: 1432 component_type = component_definition.get("type") 1433 if component_definition.get("type") != model_type.__name__: 1434 raise ValueError( 1435 f"Expected manifest component of type {model_type.__name__}, but received {component_type} instead" 1436 ) 1437 1438 # FIXME the interfaces of the concurrent cursor are kind of annoying as they take a `ComponentDefinition` instead of the actual model. This was done because the ConcurrentDeclarativeSource didn't have access to the models [here for example](https://github.com/airbytehq/airbyte-python-cdk/blob/f525803b3fec9329e4cc8478996a92bf884bfde9/airbyte_cdk/sources/declarative/concurrent_declarative_source.py#L354C54-L354C91). So now we have two cases: 1439 # * The ComponentDefinition comes from model.__dict__ in which case we have `parameters` 1440 # * The ComponentDefinition comes from the manifest as a dict in which case we have `$parameters` 1441 # We should change those interfaces to use the model once we clean up the code in CDS at which point the parameter propagation should happen as part of the ModelToComponentFactory. 1442 if "$parameters" not in component_definition and "parameters" in component_definition: 1443 component_definition["$parameters"] = component_definition.get("parameters") # type: ignore # This is a dict 1444 datetime_based_cursor_model = model_type.parse_obj(component_definition) 1445 1446 if not isinstance(datetime_based_cursor_model, DatetimeBasedCursorModel): 1447 raise ValueError( 1448 f"Expected {model_type.__name__} component, but received {datetime_based_cursor_model.__class__.__name__}" 1449 ) 1450 1451 model_parameters = datetime_based_cursor_model.parameters or {} 1452 1453 cursor_field = self._get_catalog_defined_cursor_field( 1454 stream_name=stream_name, 1455 allow_catalog_defined_cursor_field=datetime_based_cursor_model.allow_catalog_defined_cursor_field 1456 or False, 1457 ) 1458 1459 if not cursor_field: 1460 interpolated_cursor_field = InterpolatedString.create( 1461 datetime_based_cursor_model.cursor_field, 1462 parameters=model_parameters, 1463 ) 1464 cursor_field = CursorField( 1465 cursor_field_key=interpolated_cursor_field.eval(config=config), 1466 supports_catalog_defined_cursor_field=datetime_based_cursor_model.allow_catalog_defined_cursor_field 1467 or False, 1468 ) 1469 1470 interpolated_partition_field_start = InterpolatedString.create( 1471 datetime_based_cursor_model.partition_field_start or "start_time", 1472 parameters=model_parameters, 1473 ) 1474 interpolated_partition_field_end = InterpolatedString.create( 1475 datetime_based_cursor_model.partition_field_end or "end_time", 1476 parameters=model_parameters, 1477 ) 1478 1479 slice_boundary_fields = ( 1480 interpolated_partition_field_start.eval(config=config), 1481 interpolated_partition_field_end.eval(config=config), 1482 ) 1483 1484 datetime_format = datetime_based_cursor_model.datetime_format 1485 1486 cursor_granularity = ( 1487 parse_duration(datetime_based_cursor_model.cursor_granularity) 1488 if datetime_based_cursor_model.cursor_granularity 1489 else None 1490 ) 1491 1492 lookback_window = None 1493 interpolated_lookback_window = ( 1494 InterpolatedString.create( 1495 datetime_based_cursor_model.lookback_window, 1496 parameters=model_parameters, 1497 ) 1498 if datetime_based_cursor_model.lookback_window 1499 else None 1500 ) 1501 if interpolated_lookback_window: 1502 evaluated_lookback_window = interpolated_lookback_window.eval(config=config) 1503 if evaluated_lookback_window: 1504 lookback_window = parse_duration(evaluated_lookback_window) 1505 1506 connector_state_converter: DateTimeStreamStateConverter 1507 connector_state_converter = CustomFormatConcurrentStreamStateConverter( 1508 datetime_format=datetime_format, 1509 input_datetime_formats=datetime_based_cursor_model.cursor_datetime_formats, 1510 is_sequential_state=True, # ConcurrentPerPartitionCursor only works with sequential state 1511 cursor_granularity=cursor_granularity, 1512 ) 1513 1514 # Adjusts the stream state by applying the runtime lookback window. 1515 # This is used to ensure correct state handling in case of failed partitions. 1516 stream_state_value = stream_state.get(cursor_field.cursor_field_key) 1517 if runtime_lookback_window and stream_state_value: 1518 new_stream_state = ( 1519 connector_state_converter.parse_timestamp(stream_state_value) 1520 - runtime_lookback_window 1521 ) 1522 stream_state[cursor_field.cursor_field_key] = connector_state_converter.output_format( 1523 new_stream_state 1524 ) 1525 1526 start_date_runtime_value: Union[InterpolatedString, str, MinMaxDatetime] 1527 if isinstance(datetime_based_cursor_model.start_datetime, MinMaxDatetimeModel): 1528 start_date_runtime_value = self.create_min_max_datetime( 1529 model=datetime_based_cursor_model.start_datetime, config=config 1530 ) 1531 else: 1532 start_date_runtime_value = datetime_based_cursor_model.start_datetime 1533 1534 end_date_runtime_value: Optional[Union[InterpolatedString, str, MinMaxDatetime]] 1535 if isinstance(datetime_based_cursor_model.end_datetime, MinMaxDatetimeModel): 1536 end_date_runtime_value = self.create_min_max_datetime( 1537 model=datetime_based_cursor_model.end_datetime, config=config 1538 ) 1539 else: 1540 end_date_runtime_value = datetime_based_cursor_model.end_datetime 1541 1542 interpolated_start_date = MinMaxDatetime.create( 1543 interpolated_string_or_min_max_datetime=start_date_runtime_value, 1544 parameters=datetime_based_cursor_model.parameters, 1545 ) 1546 interpolated_end_date = ( 1547 None 1548 if not end_date_runtime_value 1549 else MinMaxDatetime.create( 1550 end_date_runtime_value, datetime_based_cursor_model.parameters 1551 ) 1552 ) 1553 1554 # If datetime format is not specified then start/end datetime should inherit it from the stream slicer 1555 if not interpolated_start_date.datetime_format: 1556 interpolated_start_date.datetime_format = datetime_format 1557 if interpolated_end_date and not interpolated_end_date.datetime_format: 1558 interpolated_end_date.datetime_format = datetime_format 1559 1560 start_date = interpolated_start_date.get_datetime(config=config) 1561 end_date_provider = ( 1562 partial(interpolated_end_date.get_datetime, config) 1563 if interpolated_end_date 1564 else connector_state_converter.get_end_provider() 1565 ) 1566 1567 if ( 1568 datetime_based_cursor_model.step and not datetime_based_cursor_model.cursor_granularity 1569 ) or ( 1570 not datetime_based_cursor_model.step and datetime_based_cursor_model.cursor_granularity 1571 ): 1572 raise ValueError( 1573 f"If step is defined, cursor_granularity should be as well and vice-versa. " 1574 f"Right now, step is `{datetime_based_cursor_model.step}` and cursor_granularity is `{datetime_based_cursor_model.cursor_granularity}`" 1575 ) 1576 1577 # When step is not defined, default to a step size from the starting date to the present moment 1578 step_length = datetime.timedelta.max 1579 interpolated_step = ( 1580 InterpolatedString.create( 1581 datetime_based_cursor_model.step, 1582 parameters=model_parameters, 1583 ) 1584 if datetime_based_cursor_model.step 1585 else None 1586 ) 1587 if interpolated_step: 1588 evaluated_step = interpolated_step.eval(config) 1589 if evaluated_step: 1590 step_length = parse_duration(evaluated_step) 1591 1592 clamping_strategy: ClampingStrategy = NoClamping() 1593 if datetime_based_cursor_model.clamping: 1594 # While it is undesirable to interpolate within the model factory (as opposed to at runtime), 1595 # it is still better than shifting interpolation low-code concept into the ConcurrentCursor runtime 1596 # object which we want to keep agnostic of being low-code 1597 target = InterpolatedString( 1598 string=datetime_based_cursor_model.clamping.target, 1599 parameters=model_parameters, 1600 ) 1601 evaluated_target = target.eval(config=config) 1602 match evaluated_target: 1603 case "DAY": 1604 clamping_strategy = DayClampingStrategy() 1605 end_date_provider = ClampingEndProvider( 1606 DayClampingStrategy(is_ceiling=False), 1607 end_date_provider, # type: ignore # Having issues w/ inspection for GapType and CursorValueType as shown in existing tests. Confirmed functionality is working in practice 1608 granularity=cursor_granularity or datetime.timedelta(seconds=1), 1609 ) 1610 case "WEEK": 1611 if ( 1612 not datetime_based_cursor_model.clamping.target_details 1613 or "weekday" not in datetime_based_cursor_model.clamping.target_details 1614 ): 1615 raise ValueError( 1616 "Given WEEK clamping, weekday needs to be provided as target_details" 1617 ) 1618 weekday = self._assemble_weekday( 1619 datetime_based_cursor_model.clamping.target_details["weekday"] 1620 ) 1621 clamping_strategy = WeekClampingStrategy(weekday) 1622 end_date_provider = ClampingEndProvider( 1623 WeekClampingStrategy(weekday, is_ceiling=False), 1624 end_date_provider, # type: ignore # Having issues w/ inspection for GapType and CursorValueType as shown in existing tests. Confirmed functionality is working in practice 1625 granularity=cursor_granularity or datetime.timedelta(days=1), 1626 ) 1627 case "MONTH": 1628 clamping_strategy = MonthClampingStrategy() 1629 end_date_provider = ClampingEndProvider( 1630 MonthClampingStrategy(is_ceiling=False), 1631 end_date_provider, # type: ignore # Having issues w/ inspection for GapType and CursorValueType as shown in existing tests. Confirmed functionality is working in practice 1632 granularity=cursor_granularity or datetime.timedelta(days=1), 1633 ) 1634 case _: 1635 raise ValueError( 1636 f"Invalid clamping target {evaluated_target}, expected DAY, WEEK, MONTH" 1637 ) 1638 1639 return ConcurrentCursor( 1640 stream_name=stream_name, 1641 stream_namespace=stream_namespace, 1642 stream_state=stream_state, 1643 message_repository=message_repository or self._message_repository, 1644 connector_state_manager=self._connector_state_manager, 1645 connector_state_converter=connector_state_converter, 1646 cursor_field=cursor_field, 1647 slice_boundary_fields=slice_boundary_fields, 1648 start=start_date, # type: ignore # Having issues w/ inspection for GapType and CursorValueType as shown in existing tests. Confirmed functionality is working in practice 1649 end_provider=end_date_provider, # type: ignore # Having issues w/ inspection for GapType and CursorValueType as shown in existing tests. Confirmed functionality is working in practice 1650 lookback_window=lookback_window, 1651 slice_range=step_length, 1652 cursor_granularity=cursor_granularity, 1653 clamping_strategy=clamping_strategy, 1654 is_compare_strictly=datetime_based_cursor_model.is_compare_strictly or False, 1655 ) 1656 1657 def create_concurrent_cursor_from_incrementing_count_cursor( 1658 self, 1659 model_type: Type[BaseModel], 1660 component_definition: ComponentDefinition, 1661 stream_name: str, 1662 stream_namespace: Optional[str], 1663 stream_state: MutableMapping[str, Any], 1664 config: Config, 1665 message_repository: Optional[MessageRepository] = None, 1666 **kwargs: Any, 1667 ) -> ConcurrentCursor: 1668 component_type = component_definition.get("type") 1669 if component_definition.get("type") != model_type.__name__: 1670 raise ValueError( 1671 f"Expected manifest component of type {model_type.__name__}, but received {component_type} instead" 1672 ) 1673 1674 incrementing_count_cursor_model = model_type.parse_obj(component_definition) 1675 1676 if not isinstance(incrementing_count_cursor_model, IncrementingCountCursorModel): 1677 raise ValueError( 1678 f"Expected {model_type.__name__} component, but received {incrementing_count_cursor_model.__class__.__name__}" 1679 ) 1680 1681 start_value: Union[int, str, None] = incrementing_count_cursor_model.start_value 1682 # Pydantic Union type coercion can convert int 0 to string '0' depending on Union order. 1683 # We need to handle both int and str representations of numeric values. 1684 # Evaluate the InterpolatedString and convert to int for the ConcurrentCursor. 1685 if start_value is not None: 1686 interpolated_start_value = InterpolatedString.create( 1687 str(start_value), # Ensure we pass a string to InterpolatedString.create 1688 parameters=incrementing_count_cursor_model.parameters or {}, 1689 ) 1690 evaluated_start_value: int = int(interpolated_start_value.eval(config=config)) 1691 else: 1692 evaluated_start_value = 0 1693 1694 cursor_field = self._get_catalog_defined_cursor_field( 1695 stream_name=stream_name, 1696 allow_catalog_defined_cursor_field=incrementing_count_cursor_model.allow_catalog_defined_cursor_field 1697 or False, 1698 ) 1699 1700 if not cursor_field: 1701 interpolated_cursor_field = InterpolatedString.create( 1702 incrementing_count_cursor_model.cursor_field, 1703 parameters=incrementing_count_cursor_model.parameters or {}, 1704 ) 1705 cursor_field = CursorField( 1706 cursor_field_key=interpolated_cursor_field.eval(config=config), 1707 supports_catalog_defined_cursor_field=incrementing_count_cursor_model.allow_catalog_defined_cursor_field 1708 or False, 1709 ) 1710 1711 connector_state_converter = IncrementingCountStreamStateConverter( 1712 is_sequential_state=True, # ConcurrentPerPartitionCursor only works with sequential state 1713 ) 1714 1715 return ConcurrentCursor( 1716 stream_name=stream_name, 1717 stream_namespace=stream_namespace, 1718 stream_state=stream_state, 1719 message_repository=message_repository or self._message_repository, 1720 connector_state_manager=self._connector_state_manager, 1721 connector_state_converter=connector_state_converter, 1722 cursor_field=cursor_field, 1723 slice_boundary_fields=None, 1724 start=evaluated_start_value, # type: ignore # Having issues w/ inspection for GapType and CursorValueType as shown in existing tests. Confirmed functionality is working in practice 1725 end_provider=connector_state_converter.get_end_provider(), # type: ignore # Having issues w/ inspection for GapType and CursorValueType as shown in existing tests. Confirmed functionality is working in practice 1726 ) 1727 1728 def _assemble_weekday(self, weekday: str) -> Weekday: 1729 match weekday: 1730 case "MONDAY": 1731 return Weekday.MONDAY 1732 case "TUESDAY": 1733 return Weekday.TUESDAY 1734 case "WEDNESDAY": 1735 return Weekday.WEDNESDAY 1736 case "THURSDAY": 1737 return Weekday.THURSDAY 1738 case "FRIDAY": 1739 return Weekday.FRIDAY 1740 case "SATURDAY": 1741 return Weekday.SATURDAY 1742 case "SUNDAY": 1743 return Weekday.SUNDAY 1744 case _: 1745 raise ValueError(f"Unknown weekday {weekday}") 1746 1747 def create_concurrent_cursor_from_perpartition_cursor( 1748 self, 1749 state_manager: ConnectorStateManager, 1750 model_type: Type[BaseModel], 1751 component_definition: ComponentDefinition, 1752 stream_name: str, 1753 stream_namespace: Optional[str], 1754 config: Config, 1755 stream_state: MutableMapping[str, Any], 1756 partition_router: PartitionRouter, 1757 attempt_to_create_cursor_if_not_provided: bool = False, 1758 **kwargs: Any, 1759 ) -> ConcurrentPerPartitionCursor: 1760 component_type = component_definition.get("type") 1761 if component_definition.get("type") != model_type.__name__: 1762 raise ValueError( 1763 f"Expected manifest component of type {model_type.__name__}, but received {component_type} instead" 1764 ) 1765 1766 # FIXME the interfaces of the concurrent cursor are kind of annoying as they take a `ComponentDefinition` instead of the actual model. This was done because the ConcurrentDeclarativeSource didn't have access to the models [here for example](https://github.com/airbytehq/airbyte-python-cdk/blob/f525803b3fec9329e4cc8478996a92bf884bfde9/airbyte_cdk/sources/declarative/concurrent_declarative_source.py#L354C54-L354C91). So now we have two cases: 1767 # * The ComponentDefinition comes from model.__dict__ in which case we have `parameters` 1768 # * The ComponentDefinition comes from the manifest as a dict in which case we have `$parameters` 1769 # We should change those interfaces to use the model once we clean up the code in CDS at which point the parameter propagation should happen as part of the ModelToComponentFactory. 1770 if "$parameters" not in component_definition and "parameters" in component_definition: 1771 component_definition["$parameters"] = component_definition.get("parameters") # type: ignore # This is a dict 1772 datetime_based_cursor_model = model_type.parse_obj(component_definition) 1773 1774 if not isinstance(datetime_based_cursor_model, DatetimeBasedCursorModel): 1775 raise ValueError( 1776 f"Expected {model_type.__name__} component, but received {datetime_based_cursor_model.__class__.__name__}" 1777 ) 1778 1779 cursor_field = self._get_catalog_defined_cursor_field( 1780 stream_name=stream_name, 1781 allow_catalog_defined_cursor_field=datetime_based_cursor_model.allow_catalog_defined_cursor_field 1782 or False, 1783 ) 1784 1785 if not cursor_field: 1786 interpolated_cursor_field = InterpolatedString.create( 1787 datetime_based_cursor_model.cursor_field, 1788 # FIXME the interfaces of the concurrent cursor are kind of annoying as they take a `ComponentDefinition` instead of the actual model. This was done because the ConcurrentDeclarativeSource didn't have access to the models [here for example](https://github.com/airbytehq/airbyte-python-cdk/blob/f525803b3fec9329e4cc8478996a92bf884bfde9/airbyte_cdk/sources/declarative/concurrent_declarative_source.py#L354C54-L354C91). So now we have two cases: 1789 # * The ComponentDefinition comes from model.__dict__ in which case we have `parameters` 1790 # * The ComponentDefinition comes from the manifest as a dict in which case we have `$parameters` 1791 # We should change those interfaces to use the model once we clean up the code in CDS at which point the parameter propagation should happen as part of the ModelToComponentFactory. 1792 parameters=datetime_based_cursor_model.parameters or {}, 1793 ) 1794 cursor_field = CursorField( 1795 cursor_field_key=interpolated_cursor_field.eval(config=config), 1796 supports_catalog_defined_cursor_field=datetime_based_cursor_model.allow_catalog_defined_cursor_field 1797 or False, 1798 ) 1799 1800 datetime_format = datetime_based_cursor_model.datetime_format 1801 1802 cursor_granularity = ( 1803 parse_duration(datetime_based_cursor_model.cursor_granularity) 1804 if datetime_based_cursor_model.cursor_granularity 1805 else None 1806 ) 1807 1808 connector_state_converter: DateTimeStreamStateConverter 1809 connector_state_converter = CustomFormatConcurrentStreamStateConverter( 1810 datetime_format=datetime_format, 1811 input_datetime_formats=datetime_based_cursor_model.cursor_datetime_formats, 1812 is_sequential_state=True, # ConcurrentPerPartitionCursor only works with sequential state 1813 cursor_granularity=cursor_granularity, 1814 ) 1815 1816 # Create the cursor factory 1817 cursor_factory = ConcurrentCursorFactory( 1818 partial( 1819 self.create_concurrent_cursor_from_datetime_based_cursor, 1820 state_manager=state_manager, 1821 model_type=model_type, 1822 component_definition=component_definition, 1823 stream_name=stream_name, 1824 stream_namespace=stream_namespace, 1825 config=config, 1826 message_repository=NoopMessageRepository(), 1827 ) 1828 ) 1829 1830 # Per-partition state doesn't make sense for GroupingPartitionRouter, so force the global state 1831 use_global_cursor = isinstance( 1832 partition_router, GroupingPartitionRouter 1833 ) or component_definition.get("global_substream_cursor", False) 1834 1835 # Return the concurrent cursor and state converter 1836 return ConcurrentPerPartitionCursor( 1837 cursor_factory=cursor_factory, 1838 partition_router=partition_router, 1839 stream_name=stream_name, 1840 stream_namespace=stream_namespace, 1841 stream_state=stream_state, 1842 message_repository=self._message_repository, # type: ignore 1843 connector_state_manager=state_manager, 1844 connector_state_converter=connector_state_converter, 1845 cursor_field=cursor_field, 1846 use_global_cursor=use_global_cursor, 1847 attempt_to_create_cursor_if_not_provided=attempt_to_create_cursor_if_not_provided, 1848 ) 1849 1850 @staticmethod 1851 def create_constant_backoff_strategy( 1852 model: ConstantBackoffStrategyModel, config: Config, **kwargs: Any 1853 ) -> ConstantBackoffStrategy: 1854 ModelToComponentFactory._validate_jitter_range(model.jitter_range_in_seconds) 1855 return ConstantBackoffStrategy( 1856 backoff_time_in_seconds=model.backoff_time_in_seconds, 1857 jitter_range_in_seconds=model.jitter_range_in_seconds, 1858 config=config, 1859 parameters=model.parameters or {}, 1860 ) 1861 1862 @staticmethod 1863 def _validate_jitter_range(jitter_range_in_seconds: Optional[float]) -> None: 1864 if jitter_range_in_seconds is not None and jitter_range_in_seconds < 0: 1865 raise ValueError("jitter_range_in_seconds must be greater than or equal to 0") 1866 1867 def create_cursor_pagination( 1868 self, 1869 model: CursorPaginationModel, 1870 config: Config, 1871 decoder: Optional[Decoder] = None, 1872 **kwargs: Any, 1873 ) -> CursorPaginationStrategy: 1874 # None when nested under a custom component, which cannot pass a decoder 1875 if decoder is None: 1876 decoder = JsonDecoder(parameters={}) 1877 if isinstance(decoder, PaginationDecoderDecorator): 1878 inner_decoder = decoder.decoder 1879 else: 1880 inner_decoder = decoder 1881 decoder = PaginationDecoderDecorator(decoder=decoder) 1882 1883 if self._is_supported_decoder_for_pagination(inner_decoder): 1884 decoder_to_use = decoder 1885 else: 1886 raise ValueError( 1887 self._UNSUPPORTED_DECODER_ERROR.format(decoder_type=type(inner_decoder)) 1888 ) 1889 1890 # Pydantic v1 Union type coercion can convert int to string depending on Union order. 1891 # If page_size is a string that represents an integer (not an interpolation), convert it back. 1892 page_size = model.page_size 1893 if isinstance(page_size, str) and page_size.isdigit(): 1894 page_size = int(page_size) 1895 1896 return CursorPaginationStrategy( 1897 cursor_value=model.cursor_value, 1898 decoder=decoder_to_use, 1899 page_size=page_size, 1900 stop_condition=model.stop_condition, 1901 config=config, 1902 parameters=model.parameters or {}, 1903 ) 1904 1905 def create_custom_component(self, model: Any, config: Config, **kwargs: Any) -> Any: 1906 """ 1907 Generically creates a custom component based on the model type and a class_name reference to the custom Python class being 1908 instantiated. Only the model's additional properties that match the custom class definition are passed to the constructor 1909 :param model: The Pydantic model of the custom component being created 1910 :param config: The custom defined connector config 1911 :return: The declarative component built from the Pydantic model to be used at runtime 1912 """ 1913 # Instantiating a custom component means importing and executing arbitrary code referenced 1914 # by `class_name`. Manifests supplied by a caller, whether through the config or directly to 1915 # the manifest server, are untrusted input and could point `class_name` at any importable 1916 # callable, so they honor the same `AIRBYTE_ENABLE_UNSAFE_CODE` gate as injected 1917 # `components.py` code. Manifests bundled in a published connector image are trusted and may 1918 # always use their bundled custom components. 1919 manifest_is_untrusted = not self._custom_components_trusted or bool( 1920 config.get(INJECTED_MANIFEST) 1921 ) 1922 if manifest_is_untrusted and not custom_code_execution_permitted(): 1923 raise AirbyteCustomCodeNotPermittedError 1924 1925 custom_component_class = self._get_class_from_fully_qualified_class_name(model.class_name) 1926 component_fields = get_type_hints(custom_component_class) 1927 model_args = model.dict() 1928 model_args["config"] = config 1929 1930 # There are cases where a parent component will pass arguments to a child component via kwargs. When there are field collisions 1931 # we defer to these arguments over the component's definition 1932 for key, arg in kwargs.items(): 1933 model_args[key] = arg 1934 1935 # Pydantic is unable to parse a custom component's fields that are subcomponents into models because their fields and types are not 1936 # defined in the schema. The fields and types are defined within the Python class implementation. Pydantic can only parse down to 1937 # the custom component and this code performs a second parse to convert the sub-fields first into models, then declarative components 1938 for model_field, model_value in model_args.items(): 1939 # If a custom component field doesn't have a type set, we try to use the type hints to infer the type 1940 if ( 1941 isinstance(model_value, dict) 1942 and "type" not in model_value 1943 and model_field in component_fields 1944 ): 1945 derived_type = self._derive_component_type_from_type_hints( 1946 component_fields.get(model_field) 1947 ) 1948 if derived_type: 1949 model_value["type"] = derived_type 1950 1951 if self._is_component(model_value): 1952 model_args[model_field] = self._create_nested_component( 1953 model, 1954 model_field, 1955 model_value, 1956 config, 1957 **kwargs, 1958 ) 1959 elif isinstance(model_value, list): 1960 vals = [] 1961 for v in model_value: 1962 if isinstance(v, dict) and "type" not in v and model_field in component_fields: 1963 derived_type = self._derive_component_type_from_type_hints( 1964 component_fields.get(model_field) 1965 ) 1966 if derived_type: 1967 v["type"] = derived_type 1968 if self._is_component(v): 1969 vals.append( 1970 self._create_nested_component( 1971 model, 1972 model_field, 1973 v, 1974 config, 1975 **kwargs, 1976 ) 1977 ) 1978 else: 1979 vals.append(v) 1980 model_args[model_field] = vals 1981 1982 kwargs = { 1983 class_field: model_args[class_field] 1984 for class_field in component_fields.keys() 1985 if class_field in model_args 1986 } 1987 1988 if "api_budget" in component_fields and kwargs.get("api_budget") is None: 1989 kwargs["api_budget"] = self._api_budget 1990 1991 return custom_component_class(**kwargs) 1992 1993 @staticmethod 1994 def _get_class_from_fully_qualified_class_name( 1995 full_qualified_class_name: str, 1996 ) -> Any: 1997 """Get a class from its fully qualified name. 1998 1999 If a custom components module is needed, we assume it is already registered - probably 2000 as `source_declarative_manifest.components` or `components`. 2001 2002 Args: 2003 full_qualified_class_name (str): The fully qualified name of the class (e.g., "module.ClassName"). 2004 2005 Returns: 2006 Any: The class object. 2007 2008 Raises: 2009 ValueError: If the class cannot be loaded. 2010 """ 2011 split = full_qualified_class_name.split(".") 2012 module_name_full = ".".join(split[:-1]) 2013 class_name = split[-1] 2014 2015 try: 2016 module_ref = importlib.import_module(module_name_full) 2017 except ModuleNotFoundError as e: 2018 if split[0] == "source_declarative_manifest": 2019 # During testing, the modules containing the custom components are not moved to source_declarative_manifest. In order to run the test, add the source folder to your PYTHONPATH or add it runtime using sys.path.append 2020 try: 2021 import os 2022 2023 module_name_with_source_declarative_manifest = ".".join(split[1:-1]) 2024 module_ref = importlib.import_module( 2025 module_name_with_source_declarative_manifest 2026 ) 2027 except ModuleNotFoundError: 2028 raise ValueError(f"Could not load module `{module_name_full}`.") from e 2029 else: 2030 raise ValueError(f"Could not load module `{module_name_full}`.") from e 2031 2032 try: 2033 return getattr(module_ref, class_name) 2034 except AttributeError as e: 2035 raise ValueError( 2036 f"Could not load class `{class_name}` from module `{module_name_full}`.", 2037 ) from e 2038 2039 @staticmethod 2040 def _derive_component_type_from_type_hints(field_type: Any) -> Optional[str]: 2041 interface = field_type 2042 while True: 2043 origin = get_origin(interface) 2044 if origin: 2045 # Unnest types until we reach the raw type 2046 # List[T] -> T 2047 # Optional[List[T]] -> T 2048 args = get_args(interface) 2049 interface = args[0] 2050 else: 2051 break 2052 if isinstance(interface, type) and not ModelToComponentFactory.is_builtin_type(interface): 2053 return interface.__name__ 2054 return None 2055 2056 @staticmethod 2057 def is_builtin_type(cls: Optional[Type[Any]]) -> bool: 2058 if not cls: 2059 return False 2060 return cls.__module__ == "builtins" 2061 2062 @staticmethod 2063 def _extract_missing_parameters(error: TypeError) -> List[str]: 2064 parameter_search = re.search(r"keyword-only.*:\s(.*)", str(error)) 2065 if parameter_search: 2066 return re.findall(r"\'(.+?)\'", parameter_search.group(1)) 2067 else: 2068 return [] 2069 2070 def _create_nested_component( 2071 self, model: Any, model_field: str, model_value: Any, config: Config, **kwargs: Any 2072 ) -> Any: 2073 type_name = model_value.get("type", None) 2074 if not type_name: 2075 # If no type is specified, we can assume this is a dictionary object which can be returned instead of a subcomponent 2076 return model_value 2077 2078 model_type = self.TYPE_NAME_TO_MODEL.get(type_name, None) 2079 if model_type: 2080 parsed_model = model_type.parse_obj(model_value) 2081 try: 2082 # To improve usability of the language, certain fields are shared between components. This can come in the form of 2083 # a parent component passing some of its fields to a child component or the parent extracting fields from other child 2084 # components and passing it to others. One example is the DefaultPaginator referencing the HttpRequester url_base 2085 # while constructing a SimpleRetriever. However, custom components don't support this behavior because they are created 2086 # generically in create_custom_component(). This block allows developers to specify extra arguments in $parameters that 2087 # are needed by a component and could not be shared. 2088 model_constructor = self.PYDANTIC_MODEL_TO_CONSTRUCTOR.get(parsed_model.__class__) 2089 constructor_kwargs = inspect.getfullargspec(model_constructor).kwonlyargs 2090 model_parameters = model_value.get("$parameters", {}) 2091 matching_parameters = { 2092 kwarg: model_parameters[kwarg] 2093 for kwarg in constructor_kwargs 2094 if kwarg in model_parameters 2095 } 2096 matching_kwargs = { 2097 kwarg: kwargs[kwarg] for kwarg in constructor_kwargs if kwarg in kwargs 2098 } 2099 return self._create_component_from_model( 2100 model=parsed_model, config=config, **(matching_parameters | matching_kwargs) 2101 ) 2102 except TypeError as error: 2103 missing_parameters = self._extract_missing_parameters(error) 2104 if missing_parameters: 2105 raise ValueError( 2106 f"Error creating component '{type_name}' with parent custom component {model.class_name}: Please provide " 2107 + ", ".join( 2108 ( 2109 f"{type_name}.$parameters.{parameter}" 2110 for parameter in missing_parameters 2111 ) 2112 ) 2113 ) 2114 raise TypeError( 2115 f"Error creating component '{type_name}' with parent custom component {model.class_name}: {error}" 2116 ) 2117 else: 2118 raise ValueError( 2119 f"Error creating custom component {model.class_name}. Subcomponent creation has not been implemented for '{type_name}'" 2120 ) 2121 2122 @staticmethod 2123 def _is_component(model_value: Any) -> bool: 2124 return isinstance(model_value, dict) and model_value.get("type") is not None 2125 2126 def create_default_stream( 2127 self, model: DeclarativeStreamModel, config: Config, is_parent: bool = False, **kwargs: Any 2128 ) -> AbstractStream: 2129 primary_key = model.primary_key.__root__ if model.primary_key else None 2130 self._migrate_state(model, config) 2131 self._warn_on_ineffective_incremental_dependency(model) 2132 2133 partition_router = self._build_stream_slicer_from_partition_router( 2134 model.retriever, 2135 config, 2136 stream_name=model.name, 2137 **kwargs, 2138 ) 2139 concurrent_cursor = self._build_concurrent_cursor(model, partition_router, config) 2140 if model.incremental_sync and isinstance(model.incremental_sync, DatetimeBasedCursorModel): 2141 cursor_model: DatetimeBasedCursorModel = model.incremental_sync 2142 2143 end_time_option = ( 2144 self._create_component_from_model( 2145 cursor_model.end_time_option, config, parameters=cursor_model.parameters or {} 2146 ) 2147 if cursor_model.end_time_option 2148 else None 2149 ) 2150 start_time_option = ( 2151 self._create_component_from_model( 2152 cursor_model.start_time_option, config, parameters=cursor_model.parameters or {} 2153 ) 2154 if cursor_model.start_time_option 2155 else None 2156 ) 2157 2158 datetime_request_options_provider = DatetimeBasedRequestOptionsProvider( 2159 start_time_option=start_time_option, 2160 end_time_option=end_time_option, 2161 partition_field_start=cursor_model.partition_field_start, 2162 partition_field_end=cursor_model.partition_field_end, 2163 config=config, 2164 parameters=model.parameters or {}, 2165 ) 2166 request_options_provider = ( 2167 datetime_request_options_provider 2168 if not isinstance(concurrent_cursor, ConcurrentPerPartitionCursor) 2169 else PerPartitionRequestOptionsProvider( 2170 partition_router, datetime_request_options_provider 2171 ) 2172 ) 2173 elif model.incremental_sync and isinstance( 2174 model.incremental_sync, IncrementingCountCursorModel 2175 ): 2176 if isinstance(concurrent_cursor, ConcurrentPerPartitionCursor): 2177 raise ValueError( 2178 "PerPartition does not support per partition states because switching to global state is time based" 2179 ) 2180 2181 cursor_model: IncrementingCountCursorModel = model.incremental_sync # type: ignore 2182 2183 start_time_option = ( 2184 self._create_component_from_model( 2185 cursor_model.start_value_option, # type: ignore # mypy still thinks cursor_model of type DatetimeBasedCursor 2186 config, 2187 parameters=cursor_model.parameters or {}, 2188 ) 2189 if cursor_model.start_value_option # type: ignore # mypy still thinks cursor_model of type DatetimeBasedCursor 2190 else None 2191 ) 2192 2193 # The concurrent engine defaults the start/end fields on the slice to "start" and "end", but 2194 # the default DatetimeBasedRequestOptionsProvider() sets them to start_time/end_time 2195 partition_field_start = "start" 2196 2197 request_options_provider = DatetimeBasedRequestOptionsProvider( 2198 start_time_option=start_time_option, 2199 partition_field_start=partition_field_start, 2200 config=config, 2201 parameters=model.parameters or {}, 2202 ) 2203 else: 2204 request_options_provider = None 2205 2206 transformations = [] 2207 if model.transformations: 2208 for transformation_model in model.transformations: 2209 transformations.append( 2210 self._create_component_from_model(model=transformation_model, config=config) 2211 ) 2212 file_uploader = None 2213 if model.file_uploader: 2214 file_uploader = self._create_component_from_model( 2215 model=model.file_uploader, config=config 2216 ) 2217 2218 stream_slicer: ConcurrentStreamSlicer = ( 2219 partition_router 2220 if isinstance(concurrent_cursor, FinalStateCursor) 2221 else concurrent_cursor 2222 ) 2223 2224 retriever = self._create_component_from_model( 2225 model=model.retriever, 2226 config=config, 2227 name=model.name, 2228 primary_key=primary_key, 2229 request_options_provider=request_options_provider, 2230 stream_slicer=stream_slicer, 2231 partition_router=partition_router, 2232 has_stop_condition_cursor=self._is_stop_condition_on_cursor(model), 2233 is_client_side_incremental_sync=self._is_client_side_filtering_enabled(model), 2234 cursor=concurrent_cursor, 2235 transformations=transformations, 2236 file_uploader=file_uploader, 2237 incremental_sync=model.incremental_sync, 2238 ) 2239 if isinstance(retriever, AsyncRetriever): 2240 stream_slicer = retriever.stream_slicer 2241 2242 schema_loader: SchemaLoader 2243 if model.schema_loader and isinstance(model.schema_loader, list): 2244 nested_schema_loaders = [ 2245 self._create_component_from_model(model=nested_schema_loader, config=config) 2246 for nested_schema_loader in model.schema_loader 2247 ] 2248 schema_loader = CompositeSchemaLoader( 2249 schema_loaders=nested_schema_loaders, parameters={} 2250 ) 2251 elif model.schema_loader: 2252 schema_loader = self._create_component_from_model( 2253 model=model.schema_loader, # type: ignore # If defined, schema_loader is guaranteed not to be a list and will be one of the existing base models 2254 config=config, 2255 ) 2256 else: 2257 options = model.parameters or {} 2258 if "name" not in options: 2259 options["name"] = model.name 2260 schema_loader = DefaultSchemaLoader(config=config, parameters=options) 2261 schema_loader = CachingSchemaLoaderDecorator(schema_loader) 2262 2263 stream_name = model.name or "" 2264 return DefaultStream( 2265 partition_generator=StreamSlicerPartitionGenerator( 2266 DeclarativePartitionFactory( 2267 stream_name, 2268 schema_loader, 2269 retriever, 2270 self._message_repository, 2271 ), 2272 stream_slicer, 2273 slice_limit=self._limit_slices_fetched, 2274 ), 2275 name=stream_name, 2276 json_schema=schema_loader.get_json_schema, 2277 primary_key=get_primary_key_from_stream(primary_key), 2278 cursor_field=( 2279 concurrent_cursor.cursor_field 2280 if hasattr(concurrent_cursor, "cursor_field") 2281 else None 2282 ), 2283 logger=logging.getLogger(f"airbyte.{stream_name}"), 2284 cursor=concurrent_cursor, 2285 supports_file_transfer=hasattr(model, "file_uploader") and bool(model.file_uploader), 2286 ) 2287 2288 def _warn_on_ineffective_incremental_dependency(self, model: DeclarativeStreamModel) -> None: 2289 """ 2290 `incremental_dependency: true` only takes effect when the substream defines its own 2291 `incremental_sync`: the parent cursor is persisted under the `parent_state` key of the 2292 substream's state, which is only emitted by incremental substreams. On a stream without 2293 `incremental_sync`, the setting is silently ignored and all parent records are re-read on 2294 every sync, so we warn about the misconfiguration instead. 2295 """ 2296 if model.incremental_sync: 2297 return 2298 2299 partition_router = getattr(model.retriever, "partition_router", None) 2300 if not partition_router: 2301 return 2302 2303 routers = partition_router if isinstance(partition_router, list) else [partition_router] 2304 for router in routers: 2305 if isinstance(router, GroupingPartitionRouterModel): 2306 router = router.underlying_partition_router 2307 if isinstance(router, SubstreamPartitionRouterModel) and any( 2308 parent_stream_config.incremental_dependency 2309 for parent_stream_config in router.parent_stream_configs 2310 ): 2311 LOGGER.warning( 2312 f"Stream `{model.name}` has `incremental_dependency: true` in its parent stream configuration but does not define `incremental_sync`. " 2313 "The parent stream's cursor is only persisted in the state of an incremental substream, so this setting has no effect and all parent records will be re-read on every sync. " 2314 "Define `incremental_sync` on this stream or remove `incremental_dependency`." 2315 ) 2316 return 2317 2318 def _migrate_state(self, model: DeclarativeStreamModel, config: Config) -> None: 2319 stream_name = model.name or "" 2320 stream_state = self._connector_state_manager.get_stream_state( 2321 stream_name=stream_name, namespace=None 2322 ) 2323 if model.state_migrations: 2324 state_transformations = [ 2325 self._create_component_from_model(state_migration, config, declarative_stream=model) 2326 for state_migration in model.state_migrations 2327 ] 2328 else: 2329 state_transformations = [] 2330 stream_state = self.apply_stream_state_migrations(state_transformations, stream_state) 2331 self._connector_state_manager.update_state_for_stream( 2332 stream_name=stream_name, namespace=None, value=stream_state 2333 ) 2334 2335 def _is_stop_condition_on_cursor(self, model: DeclarativeStreamModel) -> bool: 2336 return bool( 2337 model.incremental_sync 2338 and hasattr(model.incremental_sync, "is_data_feed") 2339 and model.incremental_sync.is_data_feed 2340 ) 2341 2342 def _is_client_side_filtering_enabled(self, model: DeclarativeStreamModel) -> bool: 2343 return bool( 2344 model.incremental_sync 2345 and hasattr(model.incremental_sync, "is_client_side_incremental") 2346 and model.incremental_sync.is_client_side_incremental 2347 ) 2348 2349 def _build_stream_slicer_from_partition_router( 2350 self, 2351 model: Union[ 2352 AsyncRetrieverModel, 2353 CustomRetrieverModel, 2354 SimpleRetrieverModel, 2355 ], 2356 config: Config, 2357 stream_name: Optional[str] = None, 2358 **kwargs: Any, 2359 ) -> PartitionRouter: 2360 if ( 2361 hasattr(model, "partition_router") 2362 and isinstance(model, (SimpleRetrieverModel, AsyncRetrieverModel, CustomRetrieverModel)) 2363 and model.partition_router 2364 ): 2365 stream_slicer_model = model.partition_router 2366 if isinstance(stream_slicer_model, list): 2367 return CartesianProductStreamSlicer( 2368 [ 2369 self._create_component_from_model( 2370 model=slicer, config=config, stream_name=stream_name or "" 2371 ) 2372 for slicer in stream_slicer_model 2373 ], 2374 parameters={}, 2375 ) 2376 elif isinstance(stream_slicer_model, dict): 2377 # partition router comes from CustomRetrieverModel therefore has not been parsed as a model 2378 params = stream_slicer_model.get("$parameters") 2379 if not isinstance(params, dict): 2380 params = {} 2381 stream_slicer_model["$parameters"] = params 2382 2383 if stream_name is not None: 2384 params["stream_name"] = stream_name 2385 2386 return self._create_nested_component( # type: ignore[no-any-return] # There is no guarantee that this will return a stream slicer. If not, we expect an AttributeError during the call to `stream_slices` 2387 model, 2388 "partition_router", 2389 stream_slicer_model, 2390 config, 2391 **kwargs, 2392 ) 2393 else: 2394 return self._create_component_from_model( # type: ignore[no-any-return] # Will be created PartitionRouter as stream_slicer_model is model.partition_router 2395 model=stream_slicer_model, config=config, stream_name=stream_name or "" 2396 ) 2397 return SinglePartitionRouter(parameters={}) 2398 2399 def _build_concurrent_cursor( 2400 self, 2401 model: DeclarativeStreamModel, 2402 stream_slicer: Optional[PartitionRouter], 2403 config: Config, 2404 ) -> Cursor: 2405 stream_name = model.name or "" 2406 stream_state = self._connector_state_manager.get_stream_state(stream_name, None) 2407 2408 if ( 2409 model.incremental_sync 2410 and stream_slicer 2411 and not isinstance(stream_slicer, SinglePartitionRouter) 2412 ): 2413 if isinstance(model.incremental_sync, IncrementingCountCursorModel): 2414 # We don't currently support usage of partition routing and IncrementingCountCursor at the 2415 # same time because we didn't solve for design questions like what the lookback window would 2416 # be as well as global cursor fall backs. We have not seen customers that have needed both 2417 # at the same time yet and are currently punting on this until we need to solve it. 2418 raise ValueError( 2419 f"The low-code framework does not currently support usage of a PartitionRouter and an IncrementingCountCursor at the same time. Please specify only one of these options for stream {stream_name}." 2420 ) 2421 return self.create_concurrent_cursor_from_perpartition_cursor( # type: ignore # This is a known issue that we are creating and returning a ConcurrentCursor which does not technically implement the (low-code) StreamSlicer. However, (low-code) StreamSlicer and ConcurrentCursor both implement StreamSlicer.stream_slices() which is the primary method needed for checkpointing 2422 state_manager=self._connector_state_manager, 2423 model_type=DatetimeBasedCursorModel, 2424 component_definition=model.incremental_sync.__dict__, 2425 stream_name=stream_name, 2426 stream_state=stream_state, 2427 stream_namespace=None, 2428 config=config or {}, 2429 partition_router=stream_slicer, 2430 attempt_to_create_cursor_if_not_provided=True, # FIXME can we remove that now? 2431 ) 2432 elif model.incremental_sync: 2433 if type(model.incremental_sync) == IncrementingCountCursorModel: 2434 return self.create_concurrent_cursor_from_incrementing_count_cursor( # type: ignore # This is a known issue that we are creating and returning a ConcurrentCursor which does not technically implement the (low-code) StreamSlicer. However, (low-code) StreamSlicer and ConcurrentCursor both implement StreamSlicer.stream_slices() which is the primary method needed for checkpointing 2435 model_type=IncrementingCountCursorModel, 2436 component_definition=model.incremental_sync.__dict__, 2437 stream_name=stream_name, 2438 stream_namespace=None, 2439 stream_state=stream_state, 2440 config=config or {}, 2441 ) 2442 elif type(model.incremental_sync) == DatetimeBasedCursorModel: 2443 return self.create_concurrent_cursor_from_datetime_based_cursor( # type: ignore # This is a known issue that we are creating and returning a ConcurrentCursor which does not technically implement the (low-code) StreamSlicer. However, (low-code) StreamSlicer and ConcurrentCursor both implement StreamSlicer.stream_slices() which is the primary method needed for checkpointing 2444 model_type=type(model.incremental_sync), 2445 component_definition=model.incremental_sync.__dict__, 2446 stream_name=stream_name, 2447 stream_namespace=None, 2448 stream_state=stream_state, 2449 config=config or {}, 2450 attempt_to_create_cursor_if_not_provided=True, 2451 ) 2452 else: 2453 raise ValueError( 2454 f"Incremental sync of type {type(model.incremental_sync)} is not supported" 2455 ) 2456 return FinalStateCursor(stream_name, None, self._message_repository) 2457 2458 def create_default_error_handler( 2459 self, model: DefaultErrorHandlerModel, config: Config, **kwargs: Any 2460 ) -> DefaultErrorHandler: 2461 backoff_strategies = [] 2462 if model.backoff_strategies: 2463 for backoff_strategy_model in model.backoff_strategies: 2464 backoff_strategies.append( 2465 self._create_component_from_model(model=backoff_strategy_model, config=config) 2466 ) 2467 2468 response_filters = [] 2469 if model.response_filters: 2470 for response_filter_model in model.response_filters: 2471 response_filters.append( 2472 self._create_component_from_model(model=response_filter_model, config=config) 2473 ) 2474 response_filters.append( 2475 HttpResponseFilter(config=config, parameters=model.parameters or {}) 2476 ) 2477 2478 return DefaultErrorHandler( 2479 backoff_strategies=backoff_strategies, 2480 max_retries=model.max_retries, 2481 response_filters=response_filters, 2482 config=config, 2483 parameters=model.parameters or {}, 2484 ) 2485 2486 def create_default_paginator( 2487 self, 2488 model: DefaultPaginatorModel, 2489 config: Config, 2490 *, 2491 url_base: str, 2492 extractor_model: Optional[ 2493 Union[CustomRecordExtractorModel, DpathExtractorModel, CombinedExtractorModel] 2494 ] = None, 2495 decoder: Optional[Decoder] = None, 2496 cursor_used_for_stop_condition: Optional[Cursor] = None, 2497 ) -> Union[DefaultPaginator, PaginatorTestReadDecorator]: 2498 if decoder: 2499 if self._is_supported_decoder_for_pagination(decoder): 2500 decoder_to_use = PaginationDecoderDecorator(decoder=decoder) 2501 else: 2502 raise ValueError(self._UNSUPPORTED_DECODER_ERROR.format(decoder_type=type(decoder))) 2503 else: 2504 decoder_to_use = PaginationDecoderDecorator(decoder=JsonDecoder(parameters={})) 2505 page_size_option = ( 2506 self._create_component_from_model(model=model.page_size_option, config=config) 2507 if model.page_size_option 2508 else None 2509 ) 2510 page_token_option = ( 2511 self._create_component_from_model(model=model.page_token_option, config=config) 2512 if model.page_token_option 2513 else None 2514 ) 2515 pagination_strategy = self._create_component_from_model( 2516 model=model.pagination_strategy, 2517 config=config, 2518 decoder=decoder_to_use, 2519 extractor_model=extractor_model, 2520 ) 2521 if cursor_used_for_stop_condition: 2522 pagination_strategy = StopConditionPaginationStrategyDecorator( 2523 pagination_strategy, CursorStopCondition(cursor_used_for_stop_condition) 2524 ) 2525 paginator = DefaultPaginator( 2526 decoder=decoder_to_use, 2527 page_size_option=page_size_option, 2528 page_token_option=page_token_option, 2529 pagination_strategy=pagination_strategy, 2530 url_base=url_base, 2531 config=config, 2532 parameters=model.parameters or {}, 2533 ) 2534 if self._limit_pages_fetched_per_slice: 2535 return PaginatorTestReadDecorator(paginator, self._limit_pages_fetched_per_slice) 2536 return paginator 2537 2538 @staticmethod 2539 def _is_decoder_downgraded_by_connector_builder(decoder: Decoder) -> bool: 2540 """Is this the buffered stand-in the Connector Builder builds for a streaming decoder? 2541 2542 `create_csv_decoder`, `create_jsonl_decoder`, `create_json_items_decoder` and 2543 `create_gzip_decoder` build a `CompositeRawDecoder` with `stream_response=False` when 2544 `_emit_connector_builder_messages` is set, so a Builder test read can be replayed. Those 2545 four instances are the only ones that stream in production while reporting 2546 `is_stream_response() == False` in the Builder. 2547 2548 The match is on the exact class and on the parser, not on `isinstance`: a `CustomDecoder` 2549 subclassing `CompositeRawDecoder` with `stream_response=False`, or a buffered 2550 `CompositeRawDecoder` built around a `JsonParser`, reads the body from `response.content` 2551 in production too and must not be rejected. 2552 """ 2553 return type(decoder) is CompositeRawDecoder and isinstance( 2554 decoder.parser, (CsvParser, JsonLineParser, JsonItemsParser, GzipParser) 2555 ) 2556 2557 def _reject_combined_extractor_over_streaming_decoder(self, decoder: Optional[Decoder]) -> None: 2558 """Refuse to build a `CombinedExtractor` whose response body can only be read once. 2559 2560 Every sub-extractor is handed the same `requests.Response`. A streaming decoder consumes 2561 and closes `response.raw`, so every sub-extractor after the first reads a closed 2562 `urllib3.HTTPResponse`, which returns an empty body instead of raising: `union` emits only 2563 the first sub-extractor's records and `first_match` emits nothing when the first path 2564 misses. Rejecting the manifest is the only way to make that loud. 2565 2566 The Connector Builder downgrades those same decoders to `stream_response=False` so a test 2567 read can be replayed, which would let a manifest look correct in the Builder and lose 2568 records once published. The downgraded decoders are therefore rejected in the Builder too. 2569 """ 2570 if decoder is None: 2571 return 2572 inner_decoder = ( 2573 decoder.decoder if isinstance(decoder, PaginationDecoderDecorator) else decoder 2574 ) 2575 streams_in_production = inner_decoder.is_stream_response() or ( 2576 self._emit_connector_builder_messages 2577 and self._is_decoder_downgraded_by_connector_builder(inner_decoder) 2578 ) 2579 if not streams_in_production: 2580 return 2581 raise AirbyteTracedException( 2582 message="CombinedExtractor is not supported with a streaming decoder.", 2583 internal_message=( 2584 f"CombinedExtractor was configured with {type(inner_decoder).__name__}, which " 2585 f"streams the response and can only be read once. Sub-extractors after the first " 2586 f"would read a closed response and silently return no records. The streaming " 2587 f"decoders are CsvDecoder, JsonlDecoder, JsonItemsDecoder, GzipDecoder and " 2588 f"IterableDecoder; use JsonDecoder, XmlDecoder or ZipfileDecoder, or declare a " 2589 f"single extractor." 2590 ), 2591 failure_type=FailureType.config_error, 2592 ) 2593 2594 def create_combined_extractor( 2595 self, 2596 model: CombinedExtractorModel, 2597 config: Config, 2598 decoder: Optional[Decoder] = None, 2599 **kwargs: Any, 2600 ) -> CombinedExtractor: 2601 self._reject_combined_extractor_over_streaming_decoder(decoder) 2602 extractors = [ 2603 self._create_component_from_model(model=sub_extractor, config=config, decoder=decoder) 2604 for sub_extractor in model.extractors 2605 ] 2606 return CombinedExtractor( 2607 extractors=extractors, 2608 mode=CombineMode(model.mode.value) if model.mode else CombineMode.union, 2609 skip_empty_records=bool(model.skip_empty_records), 2610 parameters=model.parameters or {}, 2611 ) 2612 2613 def create_dpath_extractor( 2614 self, 2615 model: DpathExtractorModel, 2616 config: Config, 2617 decoder: Optional[Decoder] = None, 2618 **kwargs: Any, 2619 ) -> DpathExtractor: 2620 if decoder: 2621 decoder_to_use = decoder 2622 else: 2623 decoder_to_use = JsonDecoder(parameters={}) 2624 model_field_path: List[Union[InterpolatedString, str]] = [x for x in model.field_path] 2625 2626 record_expander = None 2627 if model.record_expander: 2628 record_expander = self._create_component_from_model( 2629 model=model.record_expander, 2630 config=config, 2631 ) 2632 2633 return DpathExtractor( 2634 decoder=decoder_to_use, 2635 field_path=model_field_path, 2636 config=config, 2637 parameters=model.parameters or {}, 2638 record_expander=record_expander, 2639 ) 2640 2641 def create_record_expander( 2642 self, 2643 model: RecordExpanderModel, 2644 config: Config, 2645 **kwargs: Any, 2646 ) -> RecordExpander: 2647 truncated_list_retriever = None 2648 suppress_incomplete_fetch_warning = False 2649 if model.truncated_list_retriever: 2650 retriever_model = model.truncated_list_retriever 2651 name = "record_expander_truncated_list" 2652 # `CustomRetriever` allows extra fields, so read from the dumped model to cover both types. 2653 retriever_fields = retriever_model.dict() 2654 for unsupported_option in ("partition_router", "pagination_reset"): 2655 if retriever_fields.get(unsupported_option): 2656 raise ValueError( 2657 f"`{unsupported_option}` is not supported on `truncated_list_retriever`." 2658 ) 2659 log_formatter = lambda response: format_http_message( 2660 response, 2661 f"Record expander '{name}' request", 2662 "Request performed in order to fetch the complete nested list of a truncated record.", 2663 name, 2664 is_auxiliary=True, 2665 ) 2666 # `name`/`primary_key`/`transformations` are also what a `CustomRetriever` forwards to its 2667 # nested `requester`/`record_selector`, so they are passed for both retriever types. 2668 truncated_list_retriever = self._create_component_from_model( 2669 model=retriever_model, 2670 config=config, 2671 name=name, 2672 primary_key=None, 2673 transformations=[], 2674 log_formatter=log_formatter, 2675 ) 2676 # Only a capped paginator makes a shortfall expected; `NoPagination` and retrievers 2677 # without a paginator are never capped. 2678 suppress_incomplete_fetch_warning = isinstance( 2679 truncated_list_retriever, SimpleRetriever 2680 ) and isinstance(truncated_list_retriever.paginator, PaginatorTestReadDecorator) 2681 return RecordExpander( 2682 expand_records_from_field=model.expand_records_from_field, 2683 config=config, 2684 parameters=model.parameters or {}, 2685 remain_original_record=model.remain_original_record or False, 2686 parent_fields=[ 2687 self._create_component_from_model(model=parent_field, config=config) 2688 for parent_field in model.parent_fields 2689 ] 2690 if model.parent_fields 2691 else None, 2692 merge_parent=model.merge_parent or False, 2693 on_no_records=OnNoRecords(model.on_no_records.value) 2694 if model.on_no_records 2695 else OnNoRecords.skip, 2696 truncation_indicator_path=model.truncation_indicator_path, 2697 truncated_list_retriever=truncated_list_retriever, 2698 message_repository=self._message_repository, 2699 suppress_incomplete_fetch_warning=suppress_incomplete_fetch_warning, 2700 ) 2701 2702 @staticmethod 2703 def create_parent_field_path( 2704 model: ParentFieldPathModel, 2705 config: Config, 2706 **kwargs: Any, 2707 ) -> ParentFieldPath: 2708 return ParentFieldPath( 2709 parent_path=model.parent_path, 2710 record_path=model.record_path, 2711 config=config, 2712 parameters=model.parameters or {}, 2713 ) 2714 2715 @staticmethod 2716 def create_response_to_file_extractor( 2717 model: ResponseToFileExtractorModel, 2718 **kwargs: Any, 2719 ) -> ResponseToFileExtractor: 2720 return ResponseToFileExtractor( 2721 parameters=model.parameters or {}, 2722 preserve_na_values=model.preserve_na_values or False, 2723 ) 2724 2725 @staticmethod 2726 def create_exponential_backoff_strategy( 2727 model: ExponentialBackoffStrategyModel, config: Config 2728 ) -> ExponentialBackoffStrategy: 2729 ModelToComponentFactory._validate_jitter_range(model.jitter_range_in_seconds) 2730 return ExponentialBackoffStrategy( 2731 factor=model.factor or 5, 2732 jitter_range_in_seconds=model.jitter_range_in_seconds, 2733 parameters=model.parameters or {}, 2734 config=config, 2735 ) 2736 2737 @staticmethod 2738 def create_group_by_key(model: GroupByKeyMergeStrategyModel, config: Config) -> GroupByKey: 2739 return GroupByKey(model.key, config=config, parameters=model.parameters or {}) 2740 2741 def create_http_requester( 2742 self, 2743 model: HttpRequesterModel, 2744 config: Config, 2745 decoder: Decoder = JsonDecoder(parameters={}), 2746 query_properties_key: Optional[str] = None, 2747 use_cache: Optional[bool] = None, 2748 *, 2749 name: str, 2750 ) -> HttpRequester: 2751 authenticator = ( 2752 self._create_component_from_model( 2753 model=model.authenticator, 2754 config=config, 2755 url_base=model.url or model.url_base, 2756 name=name, 2757 decoder=decoder, 2758 ) 2759 if model.authenticator 2760 else None 2761 ) 2762 error_handler = ( 2763 self._create_component_from_model(model=model.error_handler, config=config) 2764 if model.error_handler 2765 else DefaultErrorHandler( 2766 backoff_strategies=[], 2767 response_filters=[], 2768 config=config, 2769 parameters=model.parameters or {}, 2770 ) 2771 ) 2772 2773 api_budget = self._api_budget 2774 2775 request_options_provider = InterpolatedRequestOptionsProvider( 2776 request_body=model.request_body, 2777 request_body_data=model.request_body_data, 2778 request_body_json=model.request_body_json, 2779 request_headers=model.request_headers, 2780 request_parameters=model.request_parameters, # type: ignore # QueryProperties have been removed in `create_simple_retriever` 2781 query_properties_key=query_properties_key, 2782 config=config, 2783 parameters=model.parameters or {}, 2784 ) 2785 2786 assert model.use_cache is not None # for mypy 2787 assert model.http_method is not None # for mypy 2788 2789 should_use_cache = (model.use_cache or bool(use_cache)) and not self._disable_cache 2790 2791 return HttpRequester( 2792 name=name, 2793 url=model.url, 2794 url_base=model.url_base, 2795 path=model.path, 2796 authenticator=authenticator, 2797 error_handler=error_handler, 2798 api_budget=api_budget, 2799 http_method=HttpMethod[model.http_method.value], 2800 request_options_provider=request_options_provider, 2801 config=config, 2802 disable_retries=self._disable_retries, 2803 parameters=model.parameters or {}, 2804 message_repository=self._message_repository, 2805 use_cache=should_use_cache, 2806 connect_timeout_in_seconds=model.connect_timeout_in_seconds, 2807 read_timeout_in_seconds=model.read_timeout_in_seconds, 2808 use_tcp_keepalive=model.use_tcp_keepalive or False, 2809 decoder=decoder, 2810 stream_response=decoder.is_stream_response() if decoder else False, 2811 ) 2812 2813 @staticmethod 2814 def create_http_response_filter( 2815 model: HttpResponseFilterModel, config: Config, **kwargs: Any 2816 ) -> HttpResponseFilter: 2817 if model.action: 2818 action = ResponseAction(model.action.value) 2819 else: 2820 action = None 2821 2822 failure_type = FailureType(model.failure_type.value) if model.failure_type else None 2823 2824 http_codes = ( 2825 set(model.http_codes) if model.http_codes else set() 2826 ) # JSON schema notation has no set data type. The schema enforces an array of unique elements 2827 2828 return HttpResponseFilter( 2829 action=action, 2830 failure_type=failure_type, 2831 error_message=model.error_message or "", 2832 error_message_contains=model.error_message_contains or "", 2833 http_codes=http_codes, 2834 predicate=model.predicate or "", 2835 config=config, 2836 parameters=model.parameters or {}, 2837 ) 2838 2839 @staticmethod 2840 def create_inline_schema_loader( 2841 model: InlineSchemaLoaderModel, config: Config, **kwargs: Any 2842 ) -> InlineSchemaLoader: 2843 return InlineSchemaLoader(schema=model.schema_ or {}, parameters={}) 2844 2845 def create_complex_field_type( 2846 self, model: ComplexFieldTypeModel, config: Config, **kwargs: Any 2847 ) -> ComplexFieldType: 2848 items = ( 2849 self._create_component_from_model(model=model.items, config=config) 2850 if isinstance(model.items, ComplexFieldTypeModel) 2851 else model.items 2852 ) 2853 2854 return ComplexFieldType(field_type=model.field_type, items=items) 2855 2856 def create_types_map(self, model: TypesMapModel, config: Config, **kwargs: Any) -> TypesMap: 2857 target_type = ( 2858 self._create_component_from_model(model=model.target_type, config=config) 2859 if isinstance(model.target_type, ComplexFieldTypeModel) 2860 else model.target_type 2861 ) 2862 2863 return TypesMap( 2864 target_type=target_type, 2865 current_type=model.current_type, 2866 condition=model.condition if model.condition is not None else "True", 2867 ) 2868 2869 def create_schema_type_identifier( 2870 self, model: SchemaTypeIdentifierModel, config: Config, **kwargs: Any 2871 ) -> SchemaTypeIdentifier: 2872 types_mapping = [] 2873 if model.types_mapping: 2874 types_mapping.extend( 2875 [ 2876 self._create_component_from_model(types_map, config=config) 2877 for types_map in model.types_mapping 2878 ] 2879 ) 2880 model_schema_pointer: List[Union[InterpolatedString, str]] = ( 2881 [x for x in model.schema_pointer] if model.schema_pointer else [] 2882 ) 2883 model_key_pointer: List[Union[InterpolatedString, str]] = [x for x in model.key_pointer] 2884 model_type_pointer: Optional[List[Union[InterpolatedString, str]]] = ( 2885 [x for x in model.type_pointer] if model.type_pointer else None 2886 ) 2887 2888 return SchemaTypeIdentifier( 2889 schema_pointer=model_schema_pointer, 2890 key_pointer=model_key_pointer, 2891 type_pointer=model_type_pointer, 2892 types_mapping=types_mapping, 2893 parameters=model.parameters or {}, 2894 ) 2895 2896 def create_dynamic_schema_loader( 2897 self, model: DynamicSchemaLoaderModel, config: Config, **kwargs: Any 2898 ) -> DynamicSchemaLoader: 2899 schema_transformations = [] 2900 if model.schema_transformations: 2901 for transformation_model in model.schema_transformations: 2902 schema_transformations.append( 2903 self._create_component_from_model(model=transformation_model, config=config) 2904 ) 2905 name = "dynamic_properties" 2906 partition_router = self._build_stream_slicer_from_partition_router(model.retriever, config) 2907 retriever_kwargs: Dict[str, Any] = {} 2908 if isinstance(model.retriever, AsyncRetrieverModel): 2909 if model.retriever.partition_router: 2910 raise ValueError( 2911 "DynamicSchemaLoader does not support an AsyncRetriever with a partition_router: only the first record is read, so jobs for other partitions would be orphaned. Remove the partition_router from the schema loader's AsyncRetriever." 2912 ) 2913 # create_async_retriever requires stream_slicer; passing it to a CustomRetriever would override its own 2914 retriever_kwargs["stream_slicer"] = partition_router 2915 retriever = self._create_component_from_model( 2916 model=model.retriever, 2917 config=config, 2918 name=name, 2919 primary_key=None, 2920 partition_router=partition_router, 2921 **retriever_kwargs, 2922 transformations=[], 2923 use_cache=True, 2924 log_formatter=( 2925 lambda response: format_http_message( 2926 response, 2927 f"Schema loader '{name}' request", 2928 f"Request performed in order to extract schema.", 2929 name, 2930 is_auxiliary=True, 2931 ) 2932 ), 2933 ) 2934 schema_type_identifier = self._create_component_from_model( 2935 model.schema_type_identifier, config=config, parameters=model.parameters or {} 2936 ) 2937 schema_filter = ( 2938 self._create_component_from_model( 2939 model.schema_filter, config=config, parameters=model.parameters or {} 2940 ) 2941 if model.schema_filter is not None 2942 else None 2943 ) 2944 2945 return DynamicSchemaLoader( 2946 retriever=retriever, 2947 config=config, 2948 schema_transformations=schema_transformations, 2949 schema_filter=schema_filter, 2950 schema_type_identifier=schema_type_identifier, 2951 parameters=model.parameters or {}, 2952 ) 2953 2954 @staticmethod 2955 def create_json_decoder(model: JsonDecoderModel, config: Config, **kwargs: Any) -> Decoder: 2956 return JsonDecoder(parameters={}) 2957 2958 def create_csv_decoder(self, model: CsvDecoderModel, config: Config, **kwargs: Any) -> Decoder: 2959 return CompositeRawDecoder( 2960 parser=ModelToComponentFactory._get_parser(model, config), 2961 stream_response=False if self._emit_connector_builder_messages else True, 2962 ) 2963 2964 def create_jsonl_decoder( 2965 self, model: JsonlDecoderModel, config: Config, **kwargs: Any 2966 ) -> Decoder: 2967 return CompositeRawDecoder( 2968 parser=ModelToComponentFactory._get_parser(model, config), 2969 stream_response=False if self._emit_connector_builder_messages else True, 2970 ) 2971 2972 def create_json_items_decoder( 2973 self, model: JsonItemsDecoderModel, config: Config, **kwargs: Any 2974 ) -> Decoder: 2975 return CompositeRawDecoder( 2976 parser=ModelToComponentFactory._get_parser(model, config), 2977 stream_response=False if self._emit_connector_builder_messages else True, 2978 ) 2979 2980 def create_gzip_decoder( 2981 self, model: GzipDecoderModel, config: Config, **kwargs: Any 2982 ) -> Decoder: 2983 _compressed_response_types = { 2984 "gzip", 2985 "x-gzip", 2986 "gzip, deflate", 2987 "x-gzip, deflate", 2988 "application/zip", 2989 "application/gzip", 2990 "application/x-gzip", 2991 "application/x-zip-compressed", 2992 } 2993 2994 gzip_parser: GzipParser = ModelToComponentFactory._get_parser(model, config) # type: ignore # based on the model, we know this will be a GzipParser 2995 2996 if self._emit_connector_builder_messages: 2997 return CompositeRawDecoder(gzip_parser, False) 2998 2999 transport_gzip_parser = GzipParser(inner_parser=gzip_parser) 3000 return CompositeRawDecoder.by_headers( 3001 [ 3002 ({"Content-Encoding"}, {"gzip"}, transport_gzip_parser), 3003 ({"Content-Type"}, _compressed_response_types, gzip_parser), 3004 ], 3005 stream_response=True, 3006 fallback_parser=gzip_parser, 3007 ) 3008 3009 @staticmethod 3010 def create_iterable_decoder( 3011 model: IterableDecoderModel, config: Config, **kwargs: Any 3012 ) -> IterableDecoder: 3013 return IterableDecoder(parameters={}) 3014 3015 @staticmethod 3016 def create_xml_decoder(model: XmlDecoderModel, config: Config, **kwargs: Any) -> XmlDecoder: 3017 return XmlDecoder(parameters={}) 3018 3019 def create_zipfile_decoder( 3020 self, model: ZipfileDecoderModel, config: Config, **kwargs: Any 3021 ) -> ZipfileDecoder: 3022 return ZipfileDecoder(parser=ModelToComponentFactory._get_parser(model.decoder, config)) 3023 3024 @staticmethod 3025 def _get_parser(model: BaseModel, config: Config) -> Parser: 3026 if isinstance(model, JsonDecoderModel): 3027 # Note that the logic is a bit different from the JsonDecoder as there is some legacy that is maintained to return {} on error cases 3028 return JsonParser() 3029 elif isinstance(model, JsonItemsDecoderModel): 3030 return JsonItemsParser( 3031 items_path=model.items_path, 3032 encoding=model.encoding, 3033 ) 3034 elif isinstance(model, JsonlDecoderModel): 3035 return JsonLineParser() 3036 elif isinstance(model, CsvDecoderModel): 3037 return CsvParser( 3038 encoding=model.encoding, 3039 delimiter=model.delimiter, 3040 set_values_to_none=model.set_values_to_none, 3041 ) 3042 elif isinstance(model, GzipDecoderModel): 3043 return GzipParser( 3044 inner_parser=ModelToComponentFactory._get_parser(model.decoder, config) 3045 ) 3046 elif isinstance( 3047 model, (CustomDecoderModel, IterableDecoderModel, XmlDecoderModel, ZipfileDecoderModel) 3048 ): 3049 raise ValueError(f"Decoder type {model} does not have parser associated to it") 3050 3051 raise ValueError(f"Unknown decoder type {model}") 3052 3053 @staticmethod 3054 def create_json_file_schema_loader( 3055 model: JsonFileSchemaLoaderModel, config: Config, **kwargs: Any 3056 ) -> JsonFileSchemaLoader: 3057 return JsonFileSchemaLoader( 3058 file_path=model.file_path or "", config=config, parameters=model.parameters or {} 3059 ) 3060 3061 def create_jwt_authenticator( 3062 self, model: JwtAuthenticatorModel, config: Config, **kwargs: Any 3063 ) -> JwtAuthenticator: 3064 jwt_headers = model.jwt_headers or JwtHeadersModel(kid=None, typ="JWT", cty=None) 3065 jwt_payload = model.jwt_payload or JwtPayloadModel(iss=None, sub=None, aud=None) 3066 request_option = ( 3067 self._create_component_from_model(model.request_option, config) 3068 if model.request_option 3069 else None 3070 ) 3071 return JwtAuthenticator( 3072 config=config, 3073 parameters=model.parameters or {}, 3074 algorithm=JwtAlgorithm(model.algorithm.value), 3075 secret_key=model.secret_key, 3076 base64_encode_secret_key=model.base64_encode_secret_key, 3077 token_duration=model.token_duration, 3078 header_prefix=model.header_prefix, 3079 kid=jwt_headers.kid, 3080 typ=jwt_headers.typ, 3081 cty=jwt_headers.cty, 3082 iss=jwt_payload.iss, 3083 sub=jwt_payload.sub, 3084 aud=jwt_payload.aud, 3085 additional_jwt_headers=model.additional_jwt_headers, 3086 additional_jwt_payload=model.additional_jwt_payload, 3087 passphrase=model.passphrase, 3088 request_option=request_option, 3089 ) 3090 3091 def create_list_partition_router( 3092 self, model: ListPartitionRouterModel, config: Config, **kwargs: Any 3093 ) -> ListPartitionRouter: 3094 request_option = ( 3095 self._create_component_from_model(model.request_option, config) 3096 if model.request_option 3097 else None 3098 ) 3099 return ListPartitionRouter( 3100 cursor_field=model.cursor_field, 3101 request_option=request_option, 3102 values=model.values, 3103 config=config, 3104 parameters=model.parameters or {}, 3105 ) 3106 3107 @staticmethod 3108 def create_min_max_datetime( 3109 model: MinMaxDatetimeModel, config: Config, **kwargs: Any 3110 ) -> MinMaxDatetime: 3111 return MinMaxDatetime( 3112 datetime=model.datetime, 3113 datetime_format=model.datetime_format or "", 3114 max_datetime=model.max_datetime or "", 3115 min_datetime=model.min_datetime or "", 3116 parameters=model.parameters or {}, 3117 ) 3118 3119 @staticmethod 3120 def create_no_auth(model: NoAuthModel, config: Config, **kwargs: Any) -> NoAuth: 3121 return NoAuth(parameters=model.parameters or {}) 3122 3123 @staticmethod 3124 def create_no_pagination( 3125 model: NoPaginationModel, config: Config, **kwargs: Any 3126 ) -> NoPagination: 3127 return NoPagination(parameters={}) 3128 3129 def create_oauth_authenticator( 3130 self, model: OAuthAuthenticatorModel, config: Config, **kwargs: Any 3131 ) -> DeclarativeOauth2Authenticator: 3132 profile_assertion = ( 3133 self._create_component_from_model(model.profile_assertion, config=config) 3134 if model.profile_assertion 3135 else None 3136 ) 3137 3138 refresh_token_error_status_codes, refresh_token_error_key, refresh_token_error_values = ( 3139 self._get_refresh_token_error_information(model) 3140 ) 3141 if model.refresh_token_updater: 3142 # ignore type error because fixing it would have a lot of dependencies, revisit later 3143 return DeclarativeSingleUseRefreshTokenOauth2Authenticator( # type: ignore 3144 config, 3145 InterpolatedString.create( 3146 model.token_refresh_endpoint, # type: ignore 3147 parameters=model.parameters or {}, 3148 ).eval(config), 3149 access_token_name=InterpolatedString.create( 3150 model.access_token_name or "access_token", parameters=model.parameters or {} 3151 ).eval(config), 3152 refresh_token_name=model.refresh_token_updater.refresh_token_name, 3153 expires_in_name=InterpolatedString.create( 3154 model.expires_in_name or "expires_in", parameters=model.parameters or {} 3155 ).eval(config), 3156 client_id_name=InterpolatedString.create( 3157 model.client_id_name or "client_id", parameters=model.parameters or {} 3158 ).eval(config), 3159 client_id=InterpolatedString.create( 3160 model.client_id, parameters=model.parameters or {} 3161 ).eval(config) 3162 if model.client_id 3163 else model.client_id, 3164 client_secret_name=InterpolatedString.create( 3165 model.client_secret_name or "client_secret", parameters=model.parameters or {} 3166 ).eval(config), 3167 client_secret=InterpolatedString.create( 3168 model.client_secret, parameters=model.parameters or {} 3169 ).eval(config) 3170 if model.client_secret 3171 else model.client_secret, 3172 access_token_config_path=model.refresh_token_updater.access_token_config_path, 3173 refresh_token_config_path=model.refresh_token_updater.refresh_token_config_path, 3174 token_expiry_date_config_path=model.refresh_token_updater.token_expiry_date_config_path, 3175 grant_type_name=InterpolatedString.create( 3176 model.grant_type_name or "grant_type", parameters=model.parameters or {} 3177 ).eval(config), 3178 grant_type=InterpolatedString.create( 3179 model.grant_type or "refresh_token", parameters=model.parameters or {} 3180 ).eval(config), 3181 refresh_request_body=InterpolatedMapping( 3182 model.refresh_request_body or {}, parameters=model.parameters or {} 3183 ).eval(config), 3184 refresh_request_headers=InterpolatedMapping( 3185 model.refresh_request_headers or {}, parameters=model.parameters or {} 3186 ).eval(config), 3187 send_refresh_request_as_query_params=bool( 3188 model.send_refresh_request_as_query_params 3189 ), 3190 scopes=model.scopes, 3191 token_expiry_date_format=model.token_expiry_date_format, 3192 token_expiry_is_time_of_expiration=bool(model.token_expiry_date_format), 3193 message_repository=self._message_repository, 3194 refresh_token_error_status_codes=refresh_token_error_status_codes, 3195 refresh_token_error_key=refresh_token_error_key, 3196 refresh_token_error_values=refresh_token_error_values, 3197 ) 3198 # ignore type error because fixing it would have a lot of dependencies, revisit later 3199 return DeclarativeOauth2Authenticator( # type: ignore 3200 access_token_name=model.access_token_name or "access_token", 3201 access_token_value=model.access_token_value, 3202 client_id_name=model.client_id_name or "client_id", 3203 client_id=model.client_id, 3204 client_secret_name=model.client_secret_name or "client_secret", 3205 client_secret=model.client_secret, 3206 expires_in_name=model.expires_in_name or "expires_in", 3207 grant_type_name=model.grant_type_name or "grant_type", 3208 grant_type=model.grant_type or "refresh_token", 3209 refresh_request_body=model.refresh_request_body, 3210 refresh_request_headers=model.refresh_request_headers, 3211 send_refresh_request_as_query_params=bool(model.send_refresh_request_as_query_params), 3212 refresh_token_name=model.refresh_token_name or "refresh_token", 3213 refresh_token=model.refresh_token, 3214 scopes=model.scopes, 3215 token_expiry_date=model.token_expiry_date, 3216 token_expiry_date_format=model.token_expiry_date_format, 3217 token_expiry_is_time_of_expiration=bool(model.token_expiry_date_format), 3218 token_refresh_endpoint=model.token_refresh_endpoint, 3219 config=config, 3220 parameters=model.parameters or {}, 3221 message_repository=self._message_repository, 3222 profile_assertion=profile_assertion, 3223 use_profile_assertion=model.use_profile_assertion, 3224 refresh_token_error_status_codes=refresh_token_error_status_codes, 3225 refresh_token_error_key=refresh_token_error_key, 3226 refresh_token_error_values=refresh_token_error_values, 3227 ) 3228 3229 @staticmethod 3230 def _get_refresh_token_error_information( 3231 model: OAuthAuthenticatorModel, 3232 ) -> Tuple[Tuple[int, ...], str, Tuple[str, ...]]: 3233 """ 3234 In a previous version of the CDK, the auth error as config_error was only done if a refresh token updater was 3235 defined. As a transition, we added those fields on the OAuthAuthenticatorModel. This method ensures that the 3236 information is defined only once and return the right fields. 3237 """ 3238 refresh_token_updater = model.refresh_token_updater 3239 is_defined_on_refresh_token_updated = refresh_token_updater and ( 3240 refresh_token_updater.refresh_token_error_status_codes 3241 or refresh_token_updater.refresh_token_error_key 3242 or refresh_token_updater.refresh_token_error_values 3243 ) 3244 is_defined_on_oauth_authenticator = ( 3245 model.refresh_token_error_status_codes 3246 or model.refresh_token_error_key 3247 or model.refresh_token_error_values 3248 ) 3249 if is_defined_on_refresh_token_updated and is_defined_on_oauth_authenticator: 3250 raise ValueError( 3251 "refresh_token_error should either be defined on the OAuthAuthenticatorModel or the RefreshTokenUpdaterModel, not both" 3252 ) 3253 3254 if is_defined_on_refresh_token_updated: 3255 not_optional_refresh_token_updater: RefreshTokenUpdaterModel = refresh_token_updater # type: ignore # we know from the condition that this is not None 3256 return ( 3257 tuple(not_optional_refresh_token_updater.refresh_token_error_status_codes) 3258 if not_optional_refresh_token_updater.refresh_token_error_status_codes 3259 else (), 3260 not_optional_refresh_token_updater.refresh_token_error_key or "", 3261 tuple(not_optional_refresh_token_updater.refresh_token_error_values) 3262 if not_optional_refresh_token_updater.refresh_token_error_values 3263 else (), 3264 ) 3265 elif is_defined_on_oauth_authenticator: 3266 return ( 3267 tuple(model.refresh_token_error_status_codes) 3268 if model.refresh_token_error_status_codes 3269 else (), 3270 model.refresh_token_error_key or "", 3271 tuple(model.refresh_token_error_values) if model.refresh_token_error_values else (), 3272 ) 3273 3274 # returning default values we think cover most cases 3275 return (400,), "error", ("invalid_grant", "invalid_permissions") 3276 3277 @staticmethod 3278 def _reject_union_combined_extractor_for_offset_increment( 3279 extractor_model: Optional[BaseModel], 3280 ) -> None: 3281 """Refuse an `OffsetIncrement` paginator driven by a `union` `CombinedExtractor`. 3282 3283 `OffsetIncrement.next_page_token` advances the offset by the number of records its 3284 extractor returns for the page. Under `union` that number is the sum over all 3285 sub-extractors, so the offset overshoots the API page size and every page after the first 3286 starts past the records that were never read: with two sub-extractors returning two records 3287 each and `page_size: 2`, the requested offsets are 0, 4, 8 instead of 0, 2, 4 and two 3288 thirds of the records are silently dropped. 3289 3290 `first_match` returns the winning sub-extractor's count, which does not inflate the count, 3291 and is left alone. A `union` nested anywhere in the tree inflates the count of the node 3292 above it, so the whole tree is walked. 3293 """ 3294 if not isinstance(extractor_model, CombinedExtractorModel): 3295 return 3296 if not ModelToComponentFactory._combined_extractor_tree_contains_union(extractor_model): 3297 return 3298 raise AirbyteTracedException( 3299 message=( 3300 'CombinedExtractor mode "union" is not supported with an OffsetIncrement paginator.' 3301 ), 3302 internal_message=( 3303 "OffsetIncrement counts the records of its extractor to advance the offset. A " 3304 "`union` CombinedExtractor returns the sum of its sub-extractors' records, which " 3305 "overshoots the page the API returned, so records would be skipped. Use the " 3306 "`first_match` mode, a CursorPagination or PageIncrement paginator, or a single " 3307 "extractor." 3308 ), 3309 failure_type=FailureType.config_error, 3310 ) 3311 3312 @staticmethod 3313 def _combined_extractor_tree_contains_union(model: CombinedExtractorModel) -> bool: 3314 mode = CombineMode(model.mode.value) if model.mode else CombineMode.union 3315 if mode == CombineMode.union: 3316 return True 3317 return any( 3318 isinstance(sub_extractor, CombinedExtractorModel) 3319 and ModelToComponentFactory._combined_extractor_tree_contains_union(sub_extractor) 3320 for sub_extractor in model.extractors 3321 ) 3322 3323 @staticmethod 3324 def _count_dropped_empty_records(extractor: RecordExtractor) -> None: 3325 """Make the record selector count the empty records a `CombinedExtractor` drops. 3326 3327 `OffsetIncrement` and `PageIncrement` stop when that count falls below the page size, so a 3328 full page holding a null would otherwise look short and silently end pagination. 3329 """ 3330 if not isinstance(extractor, CombinedExtractor): 3331 return 3332 extractor.count_dropped_empty_records = True 3333 for sub_extractor in extractor.extractors: 3334 ModelToComponentFactory._count_dropped_empty_records(sub_extractor) 3335 3336 def create_offset_increment( 3337 self, 3338 model: OffsetIncrementModel, 3339 config: Config, 3340 decoder: Optional[Decoder] = None, 3341 extractor_model: Optional[ 3342 Union[CustomRecordExtractorModel, DpathExtractorModel, CombinedExtractorModel] 3343 ] = None, 3344 **kwargs: Any, 3345 ) -> OffsetIncrement: 3346 # None when nested under a custom component, which cannot pass a decoder 3347 if decoder is None: 3348 decoder = JsonDecoder(parameters={}) 3349 if isinstance(decoder, PaginationDecoderDecorator): 3350 inner_decoder = decoder.decoder 3351 else: 3352 inner_decoder = decoder 3353 decoder = PaginationDecoderDecorator(decoder=decoder) 3354 3355 if self._is_supported_decoder_for_pagination(inner_decoder): 3356 decoder_to_use = decoder 3357 else: 3358 raise ValueError( 3359 self._UNSUPPORTED_DECODER_ERROR.format(decoder_type=type(inner_decoder)) 3360 ) 3361 3362 self._reject_union_combined_extractor_for_offset_increment(extractor_model) 3363 3364 # Pydantic v1 Union type coercion can convert int to string depending on Union order. 3365 # If page_size is a string that represents an integer (not an interpolation), convert it back. 3366 page_size = model.page_size 3367 if isinstance(page_size, str) and page_size.isdigit(): 3368 page_size = int(page_size) 3369 3370 return OffsetIncrement( 3371 page_size=page_size, 3372 config=config, 3373 decoder=decoder_to_use, 3374 inject_on_first_request=model.inject_on_first_request or False, 3375 parameters=model.parameters or {}, 3376 ) 3377 3378 @staticmethod 3379 def create_page_increment( 3380 model: PageIncrementModel, config: Config, **kwargs: Any 3381 ) -> PageIncrement: 3382 # Pydantic v1 Union type coercion can convert int to string depending on Union order. 3383 # If page_size is a string that represents an integer (not an interpolation), convert it back. 3384 page_size = model.page_size 3385 if isinstance(page_size, str) and page_size.isdigit(): 3386 page_size = int(page_size) 3387 3388 return PageIncrement( 3389 page_size=page_size, 3390 config=config, 3391 start_from_page=model.start_from_page or 0, 3392 inject_on_first_request=model.inject_on_first_request or False, 3393 parameters=model.parameters or {}, 3394 ) 3395 3396 def create_parent_stream_config( 3397 self, model: ParentStreamConfigModel, config: Config, *, stream_name: str, **kwargs: Any 3398 ) -> ParentStreamConfig: 3399 declarative_stream = self._create_component_from_model( 3400 model.stream, 3401 config=config, 3402 is_parent=True, 3403 **kwargs, 3404 ) 3405 request_option = ( 3406 self._create_component_from_model(model.request_option, config=config) 3407 if model.request_option 3408 else None 3409 ) 3410 3411 if model.lazy_read_pointer and any("*" in pointer for pointer in model.lazy_read_pointer): 3412 raise ValueError( 3413 "The '*' wildcard in 'lazy_read_pointer' is not supported — only direct paths are allowed." 3414 ) 3415 3416 model_lazy_read_pointer: List[Union[InterpolatedString, str]] = ( 3417 [x for x in model.lazy_read_pointer] if model.lazy_read_pointer else [] 3418 ) 3419 3420 return ParentStreamConfig( 3421 parent_key=model.parent_key, 3422 request_option=request_option, 3423 stream=declarative_stream, 3424 partition_field=model.partition_field, 3425 config=config, 3426 incremental_dependency=model.incremental_dependency or False, 3427 parameters=model.parameters or {}, 3428 extra_fields=model.extra_fields, 3429 lazy_read_pointer=model_lazy_read_pointer, 3430 ) 3431 3432 def create_properties_from_endpoint( 3433 self, model: PropertiesFromEndpointModel, config: Config, **kwargs: Any 3434 ) -> PropertiesFromEndpoint: 3435 retriever = self._create_component_from_model( 3436 model=model.retriever, 3437 config=config, 3438 name="dynamic_properties", 3439 primary_key=None, 3440 stream_slicer=None, 3441 transformations=[], 3442 use_cache=True, # Enable caching on the HttpRequester/HttpClient because the properties endpoint will be called for every slice being processed, and it is highly unlikely for the response to different 3443 ) 3444 return PropertiesFromEndpoint( 3445 property_field_path=model.property_field_path, 3446 retriever=retriever, 3447 config=config, 3448 parameters=model.parameters or {}, 3449 ) 3450 3451 def create_property_chunking( 3452 self, model: PropertyChunkingModel, config: Config, **kwargs: Any 3453 ) -> PropertyChunking: 3454 record_merge_strategy = ( 3455 self._create_component_from_model( 3456 model=model.record_merge_strategy, config=config, **kwargs 3457 ) 3458 if model.record_merge_strategy 3459 else None 3460 ) 3461 3462 property_limit_type: PropertyLimitType 3463 match model.property_limit_type: 3464 case PropertyLimitTypeModel.property_count: 3465 property_limit_type = PropertyLimitType.property_count 3466 case PropertyLimitTypeModel.characters: 3467 property_limit_type = PropertyLimitType.characters 3468 case _: 3469 raise ValueError(f"Invalid PropertyLimitType {property_limit_type}") 3470 3471 return PropertyChunking( 3472 property_limit_type=property_limit_type, 3473 property_limit=model.property_limit, 3474 record_merge_strategy=record_merge_strategy, 3475 config=config, 3476 parameters=model.parameters or {}, 3477 ) 3478 3479 def create_query_properties( 3480 self, model: QueryPropertiesModel, config: Config, *, stream_name: str, **kwargs: Any 3481 ) -> QueryProperties: 3482 if isinstance(model.property_list, list): 3483 property_list = model.property_list 3484 else: 3485 property_list = self._create_component_from_model( 3486 model=model.property_list, config=config, **kwargs 3487 ) 3488 3489 property_chunking = ( 3490 self._create_component_from_model( 3491 model=model.property_chunking, config=config, **kwargs 3492 ) 3493 if model.property_chunking 3494 else None 3495 ) 3496 3497 property_selector = ( 3498 self._create_component_from_model( 3499 model=model.property_selector, config=config, stream_name=stream_name, **kwargs 3500 ) 3501 if model.property_selector 3502 else None 3503 ) 3504 3505 return QueryProperties( 3506 property_list=property_list, 3507 always_include_properties=model.always_include_properties, 3508 property_chunking=property_chunking, 3509 property_selector=property_selector, 3510 config=config, 3511 parameters=model.parameters or {}, 3512 ) 3513 3514 def create_json_schema_property_selector( 3515 self, 3516 model: JsonSchemaPropertySelectorModel, 3517 config: Config, 3518 *, 3519 stream_name: str, 3520 **kwargs: Any, 3521 ) -> JsonSchemaPropertySelector: 3522 configured_stream = self._stream_name_to_configured_stream.get(stream_name) 3523 3524 transformations = [] 3525 if model.transformations: 3526 for transformation_model in model.transformations: 3527 transformations.append( 3528 self._create_component_from_model(model=transformation_model, config=config) 3529 ) 3530 3531 return JsonSchemaPropertySelector( 3532 configured_stream=configured_stream, 3533 properties_transformations=transformations, 3534 config=config, 3535 parameters=model.parameters or {}, 3536 ) 3537 3538 @staticmethod 3539 def create_record_filter( 3540 model: RecordFilterModel, config: Config, **kwargs: Any 3541 ) -> RecordFilter: 3542 return RecordFilter( 3543 condition=model.condition or "", config=config, parameters=model.parameters or {} 3544 ) 3545 3546 @staticmethod 3547 def create_request_path(model: RequestPathModel, config: Config, **kwargs: Any) -> RequestPath: 3548 return RequestPath(parameters={}) 3549 3550 @staticmethod 3551 def create_request_option( 3552 model: RequestOptionModel, config: Config, **kwargs: Any 3553 ) -> RequestOption: 3554 inject_into = RequestOptionType(model.inject_into.value) 3555 field_path: Optional[List[Union[InterpolatedString, str]]] = ( 3556 [ 3557 InterpolatedString.create(segment, parameters=kwargs.get("parameters", {})) 3558 for segment in model.field_path 3559 ] 3560 if model.field_path 3561 else None 3562 ) 3563 field_name = ( 3564 InterpolatedString.create(model.field_name, parameters=kwargs.get("parameters", {})) 3565 if model.field_name 3566 else None 3567 ) 3568 return RequestOption( 3569 field_name=field_name, 3570 field_path=field_path, 3571 inject_into=inject_into, 3572 parameters=kwargs.get("parameters", {}), 3573 ) 3574 3575 def create_record_selector( 3576 self, 3577 model: RecordSelectorModel, 3578 config: Config, 3579 *, 3580 name: str, 3581 transformations: List[RecordTransformation] | None = None, 3582 decoder: Decoder | None = None, 3583 client_side_incremental_sync_cursor: Optional[Cursor] = None, 3584 is_client_side_incremental_sync: bool = False, 3585 file_uploader: Optional[DefaultFileUploader] = None, 3586 **kwargs: Any, 3587 ) -> RecordSelector: 3588 extractor = self._create_component_from_model( 3589 model=model.extractor, decoder=decoder, config=config 3590 ) 3591 self._count_dropped_empty_records(extractor) 3592 record_filter = ( 3593 self._create_component_from_model(model.record_filter, config=config) 3594 if model.record_filter 3595 else None 3596 ) 3597 3598 # A client-side incremental stream transforms before filtering by default. That default belongs to the flag, 3599 # not to the component that ends up doing the cursor comparison: a data feed does it in the retriever and 3600 # receives no cursor here, but its `record_filter` condition must keep running after the transformations. 3601 default_transform_before_filtering = bool( 3602 client_side_incremental_sync_cursor or is_client_side_incremental_sync 3603 ) 3604 transform_before_filtering = ( 3605 default_transform_before_filtering 3606 if model.transform_before_filtering is None 3607 else model.transform_before_filtering 3608 ) 3609 if client_side_incremental_sync_cursor: 3610 record_filter = ClientSideIncrementalRecordFilterDecorator( 3611 config=config, 3612 parameters=model.parameters, 3613 condition=model.record_filter.condition 3614 if (model.record_filter and hasattr(model.record_filter, "condition")) 3615 else None, 3616 cursor=client_side_incremental_sync_cursor, 3617 ) 3618 3619 if model.schema_normalization is None: 3620 # default to no schema normalization if not set 3621 model.schema_normalization = SchemaNormalizationModel.None_ 3622 3623 schema_normalization = ( 3624 TypeTransformer(SCHEMA_TRANSFORMER_TYPE_MAPPING[model.schema_normalization]) 3625 if isinstance(model.schema_normalization, SchemaNormalizationModel) 3626 else self._create_component_from_model(model.schema_normalization, config=config) # type: ignore[arg-type] # custom normalization model expected here 3627 ) 3628 3629 return RecordSelector( 3630 extractor=extractor, 3631 name=name, 3632 config=config, 3633 record_filter=record_filter, 3634 transformations=transformations or [], 3635 file_uploader=file_uploader, 3636 schema_normalization=schema_normalization, 3637 parameters=model.parameters or {}, 3638 transform_before_filtering=transform_before_filtering, 3639 ) 3640 3641 @staticmethod 3642 def create_remove_fields( 3643 model: RemoveFieldsModel, config: Config, **kwargs: Any 3644 ) -> RemoveFields: 3645 return RemoveFields( 3646 field_pointers=model.field_pointers, condition=model.condition or "", parameters={} 3647 ) 3648 3649 def create_selective_authenticator( 3650 self, 3651 model: SelectiveAuthenticatorModel, 3652 config: Config, 3653 *, 3654 name: Optional[str] = None, 3655 url_base: Optional[str] = None, 3656 **kwargs: Any, 3657 ) -> DeclarativeAuthenticator: 3658 # Keyword-only so that _create_nested_component also fills them under a custom component, 3659 # from the parent's kwargs or $parameters. None is not forwarded, so a missing required one 3660 # keeps its hint. 3661 nested_kwargs = { 3662 key: value 3663 for key, value in {"name": name, "url_base": url_base}.items() 3664 if value is not None 3665 } 3666 authenticators = { 3667 key: self._create_component_from_model( 3668 model=auth, config=config, **nested_kwargs, **kwargs 3669 ) 3670 for key, auth in model.authenticators.items() 3671 } 3672 # SelectiveAuthenticator will return instance of DeclarativeAuthenticator or raise ValueError error 3673 return SelectiveAuthenticator( # type: ignore[abstract] 3674 config=config, 3675 authenticators=authenticators, 3676 authenticator_selection_path=model.authenticator_selection_path, 3677 **kwargs, 3678 ) 3679 3680 @staticmethod 3681 def create_legacy_session_token_authenticator( 3682 model: LegacySessionTokenAuthenticatorModel, config: Config, *, url_base: str, **kwargs: Any 3683 ) -> LegacySessionTokenAuthenticator: 3684 return LegacySessionTokenAuthenticator( 3685 api_url=url_base, 3686 header=model.header, 3687 login_url=model.login_url, 3688 password=model.password or "", 3689 session_token=model.session_token or "", 3690 session_token_response_key=model.session_token_response_key or "", 3691 username=model.username or "", 3692 validate_session_url=model.validate_session_url, 3693 config=config, 3694 parameters=model.parameters or {}, 3695 ) 3696 3697 def create_simple_retriever( 3698 self, 3699 model: SimpleRetrieverModel, 3700 config: Config, 3701 *, 3702 name: str, 3703 primary_key: Optional[Union[str, List[str], List[List[str]]]], 3704 request_options_provider: Optional[RequestOptionsProvider] = None, 3705 cursor: Optional[Cursor] = None, 3706 has_stop_condition_cursor: bool = False, 3707 is_client_side_incremental_sync: bool = False, 3708 transformations: List[RecordTransformation], 3709 file_uploader: Optional[DefaultFileUploader] = None, 3710 incremental_sync: Optional[ 3711 Union[IncrementingCountCursorModel, DatetimeBasedCursorModel] 3712 ] = None, 3713 use_cache: Optional[bool] = None, 3714 log_formatter: Optional[Callable[[Response], Any]] = None, 3715 partition_router: Optional[PartitionRouter] = None, 3716 **kwargs: Any, 3717 ) -> SimpleRetriever: 3718 def _get_url(req: Requester) -> str: 3719 """ 3720 Closure to get the URL from the requester. This is used to get the URL in the case of a lazy retriever. 3721 This is needed because the URL is not set until the requester is created. 3722 """ 3723 3724 _url: str = ( 3725 model.requester.url 3726 if hasattr(model.requester, "url") and model.requester.url is not None 3727 else req.get_url(stream_state=None, stream_slice=None, next_page_token=None) 3728 ) 3729 _url_base: str = ( 3730 model.requester.url_base 3731 if hasattr(model.requester, "url_base") and model.requester.url_base is not None 3732 else req.get_url_base(stream_state=None, stream_slice=None, next_page_token=None) 3733 ) 3734 3735 return _url or _url_base 3736 3737 if cursor is None: 3738 cursor = FinalStateCursor(name, None, self._message_repository) 3739 3740 # A data feed drops the records the cursor considers already synced in the retriever, which sits downstream of 3741 # the paginator. Letting the record selector drop them as well would be redundant and would hide them from the 3742 # pagination stop condition, so a data feed never delegates that filtering to the record selector, whether 3743 # `is_client_side_incremental` is set or not. The `condition` from `record_filter` is intentionally left out of 3744 # the post-pagination filter and stays in the record selector, which preserves the existing behaviour: the 3745 # selector runs inside the page loop, so the records the condition rejects never reach the paginator's 3746 # accounting. Moving it downstream would start counting them. 3747 post_pagination_filter = ( 3748 ClientSideIncrementalRecordFilterDecorator( 3749 config=config, 3750 parameters=model.parameters or {}, 3751 condition=None, 3752 cursor=cursor, 3753 ) 3754 if has_stop_condition_cursor 3755 else None 3756 ) 3757 client_side_incremental_cursor = ( 3758 cursor if is_client_side_incremental_sync and not post_pagination_filter else None 3759 ) 3760 if post_pagination_filter and is_client_side_incremental_sync: 3761 LOGGER.warning( 3762 f"Stream {name}: `is_client_side_incremental` adds no record filtering when `is_data_feed` is set, " 3763 "as a data feed already filters on the cursor value. It still makes the record selector apply the " 3764 "transformations before the `record_filter` condition." 3765 ) 3766 3767 decoder = ( 3768 self._create_component_from_model(model=model.decoder, config=config) 3769 if model.decoder 3770 else JsonDecoder(parameters={}) 3771 ) 3772 record_selector = self._create_component_from_model( 3773 model=model.record_selector, 3774 name=name, 3775 config=config, 3776 decoder=decoder, 3777 transformations=transformations, 3778 client_side_incremental_sync_cursor=client_side_incremental_cursor, 3779 is_client_side_incremental_sync=is_client_side_incremental_sync, 3780 file_uploader=file_uploader, 3781 ) 3782 3783 query_properties: Optional[QueryProperties] = None 3784 query_properties_key: Optional[str] = None 3785 self._ensure_query_properties_to_model(model.requester) 3786 if self._has_query_properties_in_request_parameters(model.requester): 3787 # It is better to be explicit about an error if PropertiesFromEndpoint is defined in multiple 3788 # places instead of default to request_parameters which isn't clearly documented 3789 if ( 3790 hasattr(model.requester, "fetch_properties_from_endpoint") 3791 and model.requester.fetch_properties_from_endpoint 3792 ): 3793 raise ValueError( 3794 f"PropertiesFromEndpoint should only be specified once per stream, but found in {model.requester.type}.fetch_properties_from_endpoint and {model.requester.type}.request_parameters" 3795 ) 3796 3797 query_properties_definitions = [] 3798 for key, request_parameter in model.requester.request_parameters.items(): # type: ignore # request_parameters is already validated to be a Mapping using _has_query_properties_in_request_parameters() 3799 if isinstance(request_parameter, QueryPropertiesModel): 3800 query_properties_key = key 3801 query_properties_definitions.append(request_parameter) 3802 3803 if len(query_properties_definitions) > 1: 3804 raise ValueError( 3805 f"request_parameters only supports defining one QueryProperties field, but found {len(query_properties_definitions)} usages" 3806 ) 3807 3808 if len(query_properties_definitions) == 1: 3809 query_properties = self._create_component_from_model( 3810 model=query_properties_definitions[0], stream_name=name, config=config 3811 ) 3812 3813 # Removes QueryProperties components from the interpolated mappings because it has been designed 3814 # to be used by the SimpleRetriever and will be resolved from the provider from the slice directly 3815 # instead of through jinja interpolation 3816 if hasattr(model.requester, "request_parameters") and isinstance( 3817 model.requester.request_parameters, Mapping 3818 ): 3819 model.requester.request_parameters = self._remove_query_properties( 3820 model.requester.request_parameters 3821 ) 3822 elif ( 3823 hasattr(model.requester, "fetch_properties_from_endpoint") 3824 and model.requester.fetch_properties_from_endpoint 3825 ): 3826 # todo: Deprecate this condition once dependent connectors migrate to query_properties 3827 query_properties_definition = QueryPropertiesModel( 3828 type="QueryProperties", 3829 property_list=model.requester.fetch_properties_from_endpoint, 3830 always_include_properties=None, 3831 property_chunking=None, 3832 ) # type: ignore # $parameters has a default value 3833 3834 query_properties = self.create_query_properties( 3835 model=query_properties_definition, 3836 stream_name=name, 3837 config=config, 3838 ) 3839 elif hasattr(model.requester, "query_properties") and model.requester.query_properties: 3840 query_properties = self.create_query_properties( 3841 model=model.requester.query_properties, 3842 stream_name=name, 3843 config=config, 3844 ) 3845 3846 requester = self._create_component_from_model( 3847 model=model.requester, 3848 decoder=decoder, 3849 name=name, 3850 query_properties_key=query_properties_key, 3851 use_cache=use_cache, 3852 config=config, 3853 ) 3854 3855 if not request_options_provider: 3856 request_options_provider = DefaultRequestOptionsProvider(parameters={}) 3857 if isinstance(request_options_provider, DefaultRequestOptionsProvider) and isinstance( 3858 partition_router, PartitionRouter 3859 ): 3860 request_options_provider = partition_router 3861 3862 paginator = ( 3863 self._create_component_from_model( 3864 model=model.paginator, 3865 config=config, 3866 url_base=_get_url(requester), 3867 extractor_model=model.record_selector.extractor, 3868 decoder=decoder, 3869 cursor_used_for_stop_condition=cursor if has_stop_condition_cursor else None, 3870 ) 3871 if model.paginator 3872 else NoPagination(parameters={}) 3873 ) 3874 3875 ignore_stream_slicer_parameters_on_paginated_requests = ( 3876 model.ignore_stream_slicer_parameters_on_paginated_requests or False 3877 ) 3878 3879 reads_parent_stream_lazily = bool( 3880 model.partition_router 3881 and isinstance(model.partition_router, SubstreamPartitionRouterModel) 3882 and any( 3883 parent_stream_config.lazy_read_pointer 3884 for parent_stream_config in model.partition_router.parent_stream_configs 3885 ) 3886 ) 3887 if reads_parent_stream_lazily and ( 3888 model.page_size_reduction 3889 or self._uses_reduce_page_size_action(getattr(model.requester, "error_handler", None)) 3890 ): 3891 # Checked outside of the LazySimpleRetriever branch below, which only applies on the first 3892 # sync of a stream: gating it on the absence of state would accept the same manifest from the 3893 # second sync onwards. LazySimpleRetriever paginates the parent's embedded pages, so there is 3894 # no page of its own to re-issue with a smaller page size. 3895 raise ValueError( 3896 f"`page_size_reduction` and the REDUCE_PAGE_SIZE response action are not supported when " 3897 f"reading a parent stream lazily. Remove either the page size reduction or the parent " 3898 f"stream's `lazy_read_pointer` for stream {name}." 3899 ) 3900 3901 if reads_parent_stream_lazily and ( 3902 model.request_window_splitting 3903 or self._uses_split_request_window_action( 3904 getattr(model.requester, "error_handler", None) 3905 ) 3906 ): 3907 # Same reasoning as the page_size_reduction check above: LazySimpleRetriever reads records 3908 # embedded in the parent's own pages rather than requesting a window of its own, so there is 3909 # nothing here for a split child window to be read from. 3910 raise ValueError( 3911 f"`request_window_splitting` and the SPLIT_REQUEST_WINDOW response action are not " 3912 f"supported when reading a parent stream lazily. Remove either the request window " 3913 f"splitting or the parent stream's `lazy_read_pointer` for stream {name}." 3914 ) 3915 3916 if reads_parent_stream_lazily and not bool( 3917 self._connector_state_manager.get_stream_state(name, None) 3918 ): 3919 if incremental_sync: 3920 if incremental_sync.type != "DatetimeBasedCursor": 3921 raise ValueError( 3922 f"LazySimpleRetriever only supports DatetimeBasedCursor. Found: {incremental_sync.type}." 3923 ) 3924 3925 elif incremental_sync.step or incremental_sync.cursor_granularity: 3926 raise ValueError( 3927 f"Found more that one slice per parent. LazySimpleRetriever only supports single slice read for stream - {name}." 3928 ) 3929 3930 if model.decoder and model.decoder.type != "JsonDecoder": 3931 raise ValueError( 3932 f"LazySimpleRetriever only supports JsonDecoder. Found: {model.decoder.type}." 3933 ) 3934 3935 return LazySimpleRetriever( 3936 name=name, 3937 paginator=paginator, 3938 primary_key=primary_key, 3939 requester=requester, 3940 record_selector=record_selector, 3941 stream_slicer=_NO_STREAM_SLICING, 3942 request_option_provider=request_options_provider, 3943 config=config, 3944 ignore_stream_slicer_parameters_on_paginated_requests=ignore_stream_slicer_parameters_on_paginated_requests, 3945 post_pagination_filter=post_pagination_filter, 3946 parameters=model.parameters or {}, 3947 ) 3948 3949 if ( 3950 model.record_selector.record_filter 3951 and model.pagination_reset 3952 and model.pagination_reset.limits 3953 ): 3954 raise ValueError("PaginationResetLimits are not supported while having record filter.") 3955 3956 request_window_splitting = self._create_request_window_splitting( 3957 model, name, cursor, incremental_sync, query_properties, file_uploader 3958 ) 3959 return SimpleRetriever( 3960 name=name, 3961 paginator=paginator, 3962 primary_key=primary_key, 3963 requester=requester, 3964 record_selector=record_selector, 3965 stream_slicer=_NO_STREAM_SLICING, 3966 request_option_provider=request_options_provider, 3967 config=config, 3968 ignore_stream_slicer_parameters_on_paginated_requests=ignore_stream_slicer_parameters_on_paginated_requests, 3969 additional_query_properties=query_properties, 3970 log_formatter=self._get_log_formatter(log_formatter, name), 3971 pagination_tracker_factory=self._create_pagination_tracker_factory( 3972 model.pagination_reset, cursor 3973 ), 3974 page_size_reduction=self._create_page_size_reduction( 3975 model, name, query_properties, file_uploader 3976 ), 3977 request_window_splitting=request_window_splitting, 3978 request_window_splitter=( 3979 cursor.split_request_window 3980 if request_window_splitting and hasattr(cursor, "split_request_window") 3981 else None 3982 ), 3983 post_pagination_filter=post_pagination_filter, 3984 parameters=model.parameters or {}, 3985 ) 3986 3987 def _create_page_size_reduction( 3988 self, 3989 model: SimpleRetrieverModel, 3990 name: str, 3991 query_properties: Optional[QueryProperties], 3992 file_uploader: Optional[DefaultFileUploader] = None, 3993 ) -> Optional[PageSizeReduction]: 3994 # A CustomRequester does not necessarily define an error handler. A CustomRequester that does define 3995 # one keeps it as a raw dict rather than a typed model, so this returns False for it as well. 3996 error_handler = getattr(model.requester, "error_handler", None) 3997 uses_action = self._uses_reduce_page_size_action(error_handler) 3998 if uses_action and not model.page_size_reduction: 3999 raise ValueError( 4000 f"Stream {name} has a response filter with the REDUCE_PAGE_SIZE action but the retriever does not " 4001 f"define `page_size_reduction`. Add a `page_size_reduction` block to the retriever." 4002 ) 4003 4004 if not model.page_size_reduction: 4005 return None 4006 4007 if not uses_action: 4008 # Not raised: a CustomErrorHandler can resolve to REDUCE_PAGE_SIZE without us being able to see 4009 # it, so the only safe reaction to a block we cannot tie to an action is a warning. Without it a 4010 # misspelled action leaves the feature silently dead on a stream that only exists because it 4011 # would otherwise fail. 4012 LOGGER.warning( 4013 f"Stream {name} defines `page_size_reduction` but no response filter with the " 4014 f"REDUCE_PAGE_SIZE action was found on its requester. The page size will never be reduced " 4015 f"unless a custom error handler resolves to that action." 4016 ) 4017 4018 self._validate_page_size_reduction_is_supported( 4019 model, name, query_properties, file_uploader 4020 ) 4021 4022 reset_policy = model.page_size_reduction.reset_policy 4023 return PageSizeReduction( 4024 reduction_factor=model.page_size_reduction.reduction_factor, # type: ignore[arg-type] # the schema defines a default 4025 minimum_page_size=model.page_size_reduction.minimum_page_size, # type: ignore[arg-type] # the schema defines a default 4026 max_attempts=model.page_size_reduction.max_attempts, # type: ignore[arg-type] # the schema defines a default 4027 backoff_seconds=model.page_size_reduction.backoff_seconds, # type: ignore[arg-type] # the schema defines a default 4028 retries_at_minimum_page_size=model.page_size_reduction.retries_at_minimum_page_size, # type: ignore[arg-type] # the schema defines a default 4029 failure_message=model.page_size_reduction.failure_message, 4030 reset_policy=PageSizeResetPolicy(reset_policy.value) 4031 if reset_policy is not None 4032 else PageSizeResetPolicy.NEVER, 4033 ) 4034 4035 def _validate_page_size_reduction_is_supported( 4036 self, 4037 model: SimpleRetrieverModel, 4038 name: str, 4039 query_properties: Optional[QueryProperties], 4040 file_uploader: Optional[DefaultFileUploader] = None, 4041 ) -> None: 4042 """ 4043 Page size reduction re-issues the same page with a smaller page size. That is only correct when the next 4044 page does not depend on the page size, and it only has an effect when the paginator injects the page size 4045 in the request. A custom pagination strategy is accepted when it can receive the reduced page size, which 4046 is checked by inspecting its signature rather than by recognizing its type. 4047 """ 4048 if query_properties: 4049 raise ValueError( 4050 f"`page_size_reduction` cannot be used together with query properties on stream {name}. Records " 4051 f"from the earlier property chunks have already been emitted when a chunk asks for a smaller page, " 4052 f"so retrying the page would emit them twice." 4053 ) 4054 4055 if file_uploader: 4056 raise ValueError( 4057 f"`page_size_reduction` cannot be used together with a `file_uploader` on stream {name}. The " 4058 f"file uploader sends one request per record from inside the page's record generator, so a " 4059 f"reduction asked for halfway through a page would re-emit the records already yielded by it." 4060 ) 4061 4062 if not isinstance(model.paginator, DefaultPaginatorModel): 4063 raise ValueError( 4064 f"`page_size_reduction` requires a DefaultPaginator on stream {name} so that the connector can " 4065 f"send a smaller page size." 4066 ) 4067 4068 if not model.paginator.page_size_option: 4069 raise ValueError( 4070 f"`page_size_reduction` requires `page_size_option` on the paginator of stream {name}: without it " 4071 f"the connector cannot tell the API to send a smaller page." 4072 ) 4073 4074 if isinstance(model.paginator.page_token_option, RequestPathModel): 4075 # A RequestPath page token is a full URL built by the API, and it already carries the page size the 4076 # API echoed back. The reduced page size is injected as a request option on top of that URL, so the 4077 # request goes out with the page size twice - the original one from the URL and the reduced one - 4078 # and which of the two the API honors is up to the API. Every page after the first would then keep 4079 # asking for the page size that just failed. 4080 raise ValueError( 4081 f"`page_size_reduction` does not support a `page_token_option` of type RequestPath on stream " 4082 f"{name}. The next page is then requested through a URL returned by the API, which already " 4083 f"carries the page size, so the reduced page size would be sent alongside the original one. Use " 4084 f"a CursorPagination strategy with a `page_token_option` of type RequestOption instead." 4085 ) 4086 4087 strategy = model.paginator.pagination_strategy 4088 if isinstance(strategy, PageIncrementModel): 4089 raise ValueError( 4090 f"`page_size_reduction` does not support the PageIncrement pagination strategy used by stream " 4091 f"{name}. Pages are addressed as page number * page size, so a smaller page size shifts every " 4092 f"following page boundary and would skip records. Use OffsetIncrement or CursorPagination." 4093 ) 4094 if isinstance(strategy, CustomPaginationStrategyModel): 4095 # A custom strategy is written by the same person enabling the reduction, so the 4096 # question is not whether we recognize it but whether it can be told the reduced 4097 # page size. Checking the signature keeps a strategy that would raise TypeError 4098 # mid-sync from being accepted at config time. 4099 custom_class = self._get_class_from_fully_qualified_class_name(strategy.class_name) 4100 if isinstance(custom_class, type) and issubclass(custom_class, PageIncrement): 4101 # `PageIncrement.next_page_token` declares `page_size_override` only to reject it, so a 4102 # subclass that does not override the method would pass the signature check below and only 4103 # fail once the first reduction is requested, mid-sync. 4104 raise ValueError( 4105 f"`page_size_reduction` does not support the PageIncrement pagination strategy that the " 4106 f"custom pagination strategy {strategy.class_name} used by stream {name} inherits from. " 4107 f"Pages are addressed as page number * page size, so a smaller page size shifts every " 4108 f"following page boundary and would skip records. Use OffsetIncrement or CursorPagination." 4109 ) 4110 try: 4111 parameters = inspect.signature(custom_class.next_page_token).parameters 4112 except (AttributeError, TypeError, ValueError) as exception: 4113 raise ValueError( 4114 f"`page_size_reduction` could not check the signature of `next_page_token` on the custom " 4115 f"pagination strategy {strategy.class_name} used by stream {name}: {exception}. Make sure " 4116 f"`class_name` points at a PaginationStrategy subclass whose `next_page_token` accepts a " 4117 f"`page_size_override` keyword argument." 4118 ) 4119 accepts_override = "page_size_override" in parameters or any( 4120 parameter.kind == inspect.Parameter.VAR_KEYWORD for parameter in parameters.values() 4121 ) 4122 if not accepts_override: 4123 raise ValueError( 4124 f"`page_size_reduction` requires the custom pagination strategy " 4125 f"{strategy.class_name} used by stream {name} to accept a `page_size_override` keyword " 4126 f"argument in `next_page_token`, so that it can honor the reduced page size. Add " 4127 f"`page_size_override: Optional[int] = None` to its signature; a strategy that does not " 4128 f"use its page size as a stop condition can ignore the value." 4129 ) 4130 elif not isinstance(strategy, (CursorPaginationModel, OffsetIncrementModel)): 4131 raise ValueError( 4132 f"`page_size_reduction` only supports the CursorPagination, OffsetIncrement and " 4133 f"CustomPaginationStrategy pagination strategies. Stream {name} uses " 4134 f"{type(strategy).__name__}." 4135 ) 4136 else: 4137 # A CustomPaginationStrategy carries its page size in its own code, so this is only checkable 4138 # for the strategies the CDK defines. `page_size` is optional on both of them, and without it 4139 # `get_page_size` returns None, the paginator injects nothing, and the reduction is a dead end 4140 # that only surfaces on the first failing response - after records have been emitted. 4141 self._validate_page_size_is_reducible(strategy, model.page_size_reduction, name) 4142 if isinstance(strategy, CursorPaginationModel): 4143 self._validate_stop_condition_is_reduction_aware( 4144 strategy, model.page_size_reduction, name 4145 ) 4146 4147 @staticmethod 4148 def _validate_stop_condition_is_reduction_aware( 4149 strategy: CursorPaginationModel, 4150 page_size_reduction: Optional[PageSizeReductionModel], 4151 name: str, 4152 ) -> None: 4153 """ 4154 A `stop_condition` comparing `last_page_size` to a hardcoded page size reads a full reduced page as a 4155 short page and ends the pagination early, dropping the rest of the partition without failing. The 4156 strategy exposes the page size that was actually requested as `page_size`, so the condition can be 4157 written correctly - but only if it is, which is what this checks. 4158 4159 The condition is parsed as a Jinja expression rather than matched as a string: only the AST tells 4160 `page_size`, which follows the reduction, apart from `config['page_size']`, which does not, and only the 4161 AST tells an inequality, which a reduction can invalidate, apart from `last_page_size == 0`, which it 4162 cannot. A condition that never names `last_page_size` is not waved through: a count read from the 4163 response body - `{{ response['data'] | length < 100 }}` - truncates in exactly the same way, and which 4164 response field counts the records of a page is not knowable here. A shape the analysis does not 4165 understand is warned about rather than rejected - this runs at stream construction, so a false 4166 rejection takes `check`, `discover` and `read` down with it. 4167 """ 4168 stop_condition = strategy.stop_condition 4169 if not stop_condition: 4170 return 4171 4172 minimum_page_size = ( 4173 page_size_reduction.minimum_page_size if page_size_reduction else None 4174 ) or 1 4175 verdict, reason = classify_stop_condition(stop_condition, minimum_page_size) 4176 if verdict is StopConditionSafety.TRUNCATES: 4177 raise ValueError( 4178 f"`page_size_reduction` on stream {name} cannot be used with the `stop_condition` " 4179 f"{stop_condition!r}: {reason}. The pagination would then end early, silently dropping the " 4180 f"rest of the partition. Compare against the `page_size` interpolation variable instead, " 4181 f"which holds the page size that was actually requested (for example " 4182 f"`{{{{ last_page_size < page_size }}}}`), or test the page for emptiness with " 4183 f"`{{{{ last_page_size == 0 }}}}`." 4184 ) 4185 if verdict is StopConditionSafety.UNKNOWN: 4186 LOGGER.warning( 4187 f"Stream {name} uses `page_size_reduction` with the `stop_condition` {stop_condition!r}, " 4188 f"which could not be checked against the reduction because {reason}. Make sure a page that is " 4189 f"full at a reduced page size does not satisfy it, otherwise the pagination ends early and the " 4190 f"rest of the partition is silently dropped. Comparing against the `page_size` interpolation " 4191 f"variable, which holds the page size that was actually requested, is always safe." 4192 ) 4193 4194 @staticmethod 4195 def _validate_page_size_is_reducible( 4196 strategy: Union[CursorPaginationModel, OffsetIncrementModel], 4197 page_size_reduction: Optional[PageSizeReductionModel], 4198 name: str, 4199 ) -> None: 4200 page_size = strategy.page_size 4201 if page_size is None: 4202 raise ValueError( 4203 f"`page_size_reduction` requires `page_size` on the pagination strategy of stream {name}: " 4204 f"without it the paginator does not send a page size, so there is nothing to reduce." 4205 ) 4206 4207 # The schema allows a string so that the page size can be interpolated. `page_size` is 4208 # `Optional[Union[int, str]]` and pydantic v1 tries `int` first, so both `100` and `"100"` arrive as 4209 # an int and only a genuinely non-numeric template stays a str. Such a template is only known once 4210 # the config is available, so it is left to the runtime check in `PageSizeReducer.reduce`. 4211 try: 4212 configured_page_size: Optional[int] = int(page_size) 4213 except ValueError: 4214 configured_page_size = None 4215 4216 minimum_page_size = ( 4217 page_size_reduction.minimum_page_size if page_size_reduction else None 4218 ) or 1 4219 if configured_page_size is not None and configured_page_size <= minimum_page_size: 4220 raise ValueError( 4221 f"`page_size_reduction` on stream {name} can never reduce its page size: the pagination " 4222 f"strategy's `page_size` is {configured_page_size} and `minimum_page_size` is " 4223 f"{minimum_page_size}. Lower `minimum_page_size` or raise `page_size`." 4224 ) 4225 4226 # Each reduction divides the page size by `reduction_factor` and spends one attempt, so an unbroken 4227 # run of failures bottoms out at `page_size / reduction_factor ** max_attempts` whatever 4228 # `minimum_page_size` says. Only warned about, and not raised: pages that succeed in between restart 4229 # the budget while `NEVER` keeps the page size, so the floor is reachable over a partition even when 4230 # it is out of reach of a single run - and a manifest that deliberately gives up earlier than its 4231 # floor is not wrong, only worth pointing out. 4232 reduction_factor = ( 4233 page_size_reduction.reduction_factor if page_size_reduction else None 4234 ) or 2.0 4235 max_attempts = (page_size_reduction.max_attempts if page_size_reduction else None) or 5 4236 # Only when there is a floor worth reaching. The default of 1 is out of reach of the default budget on 4237 # any page size above 32, so this would otherwise fire on nearly every stream that opts in - and 4238 # `__fields_set__` would not help, since it records that a value was supplied and not that it differs 4239 # from the default, so spelling `minimum_page_size: 1` out longhand would earn the warning. 4240 if ( 4241 configured_page_size is not None 4242 and minimum_page_size > 1 4243 and reduction_factor**max_attempts < configured_page_size / minimum_page_size 4244 ): 4245 reachable_page_size = max( 4246 minimum_page_size, int(configured_page_size // reduction_factor**max_attempts) 4247 ) 4248 LOGGER.warning( 4249 f"Stream {name} sets `minimum_page_size` to {minimum_page_size}, which `page_size_reduction` " 4250 f"cannot reach in one run of failing pages: `max_attempts` is {max_attempts} and " 4251 f"`reduction_factor` is {reduction_factor}, so {max_attempts} reductions of a page size of " 4252 f"{configured_page_size} stop at {reachable_page_size} records per page and the sync then " 4253 f"fails with a transient error. Whichever of the two bounds is tighter wins; reaching " 4254 f"{minimum_page_size} in one run needs `max_attempts` of at least " 4255 f"{math.ceil(math.log(configured_page_size / minimum_page_size, reduction_factor))}." 4256 ) 4257 4258 def _reject_reduce_page_size_action(self, requester: Any, description: str) -> None: 4259 """ 4260 `error_handler` is defined on `HttpRequester`, which is referenced by requesters that have no page of 4261 their own, so REDUCE_PAGE_SIZE is schema-legal in places where nothing can honor it. Only 4262 `SimpleRetriever._read_pages` re-issues a page, so every other requester is rejected here rather than 4263 surfacing the exception mid-sync as a generic failure. 4264 """ 4265 if requester is None: 4266 return 4267 if self._uses_reduce_page_size_action(getattr(requester, "error_handler", None)): 4268 raise ValueError( 4269 f"The REDUCE_PAGE_SIZE response action is not supported on the {description}: only the main " 4270 f"requester of a SimpleRetriever can re-issue its page with a smaller page size. Use a " 4271 f"different action on that error handler." 4272 ) 4273 4274 def _uses_reduce_page_size_action(self, error_handler: Any) -> bool: 4275 if isinstance(error_handler, CompositeErrorHandlerModel): 4276 return any( 4277 self._uses_reduce_page_size_action(nested) 4278 for nested in error_handler.error_handlers 4279 ) 4280 if isinstance(error_handler, DefaultErrorHandlerModel): 4281 return any( 4282 response_filter.action == HttpResponseFilterActionModel.REDUCE_PAGE_SIZE 4283 for response_filter in error_handler.response_filters or [] 4284 ) 4285 # A CustomErrorHandler can return any action and we cannot inspect it, so we do not validate it. 4286 return False 4287 4288 def _create_request_window_splitting( 4289 self, 4290 model: SimpleRetrieverModel, 4291 name: str, 4292 cursor: Optional[Cursor], 4293 incremental_sync: Optional[ 4294 Union[IncrementingCountCursorModel, DatetimeBasedCursorModel] 4295 ] = None, 4296 query_properties: Optional[QueryProperties] = None, 4297 file_uploader: Optional[DefaultFileUploader] = None, 4298 ) -> Optional[RequestWindowSplitting]: 4299 # A CustomRequester does not necessarily define an error handler. A CustomRequester that does define 4300 # one keeps it as a raw dict rather than a typed model, so this returns False for it as well. 4301 error_handler = getattr(model.requester, "error_handler", None) 4302 uses_action = self._uses_split_request_window_action(error_handler) 4303 if uses_action and not model.request_window_splitting: 4304 raise ValueError( 4305 f"Stream {name} has a response filter with the SPLIT_REQUEST_WINDOW action but the " 4306 f"retriever does not define `request_window_splitting`. Add a `request_window_splitting` " 4307 f"block to the retriever." 4308 ) 4309 4310 if not model.request_window_splitting: 4311 return None 4312 4313 if not uses_action: 4314 # Not raised: a CustomErrorHandler can resolve to SPLIT_REQUEST_WINDOW without us being able to 4315 # see it, and custom code can raise RequestWindowSplitRequiredException directly without 4316 # going through any error handler at all - so the only safe reaction to a block we cannot tie to 4317 # a known trigger is a warning. 4318 LOGGER.warning( 4319 f"Stream {name} defines `request_window_splitting` but no response filter with the " 4320 f"SPLIT_REQUEST_WINDOW action was found on its requester. The request window will never " 4321 f"be split unless a custom error handler or custom code resolves to that action." 4322 ) 4323 4324 self._validate_request_window_splitting_is_supported( 4325 name, 4326 cursor, 4327 incremental_sync, 4328 model.request_window_splitting.min_split_window, 4329 query_properties, 4330 file_uploader, 4331 ) 4332 4333 return RequestWindowSplitting( 4334 failure_message=model.request_window_splitting.failure_message, 4335 min_split_window=parse_duration(model.request_window_splitting.min_split_window) 4336 if model.request_window_splitting.min_split_window 4337 else None, 4338 ) 4339 4340 def _validate_request_window_splitting_is_supported( 4341 self, 4342 name: str, 4343 cursor: Optional[Cursor], 4344 incremental_sync: Optional[ 4345 Union[IncrementingCountCursorModel, DatetimeBasedCursorModel] 4346 ] = None, 4347 min_split_window: Optional[str] = None, 4348 query_properties: Optional[QueryProperties] = None, 4349 file_uploader: Optional[DefaultFileUploader] = None, 4350 ) -> None: 4351 """ 4352 Request window splitting replaces a failing slice with smaller children derived from the stream's own 4353 cursor, so it only makes sense on a stream whose cursor exposes a `split_request_window` method and it 4354 is rejected for the same structural reasons `page_size_reduction` is: `additional_query_properties` and 4355 a `file_uploader` both mean records of the failing window may already have been emitted by the time the 4356 split is requested, from inside the record generator rather than from a page boundary this retriever 4357 controls. 4358 """ 4359 if not hasattr(cursor, "split_request_window"): 4360 raise ValueError( 4361 f"`request_window_splitting` requires an incremental cursor that supports window splitting on " 4362 f"stream {name}. Found {type(cursor).__name__ if cursor is not None else 'no cursor'}: this is " 4363 f"only supported today for a `DatetimeBasedCursor` with `cursor_granularity` set." 4364 ) 4365 4366 if not ( 4367 isinstance(incremental_sync, DatetimeBasedCursorModel) 4368 and incremental_sync.cursor_granularity 4369 ): 4370 # `cursor_granularity` is both the smallest window the connector will ever request and what keeps 4371 # two child windows from overlapping at their shared edge, so `split_request_window` cannot split 4372 # at all without it - checked here, on the manifest model, rather than by reaching into the 4373 # cursor's private state from outside the class it belongs to. 4374 raise ValueError( 4375 f"`request_window_splitting` requires `cursor_granularity` on the `DatetimeBasedCursor` of " 4376 f"stream {name}: without it there is no smallest window to stop splitting at, and no way to " 4377 f"keep two child windows from overlapping at their shared edge." 4378 ) 4379 4380 if incremental_sync.is_client_side_incremental: 4381 # A client-side-incremental cursor filters records after they are read rather than sending the 4382 # window as request parameters, so every child window sends the exact same request as its parent: 4383 # splitting changes nothing, and the sync would keep splitting all the way to `min_split_window` 4384 # or the safety-net depth before failing. 4385 raise ValueError( 4386 f"`request_window_splitting` cannot be used together with `is_client_side_incremental` on " 4387 f"stream {name}: the window is filtered client-side rather than sent to the API, so splitting " 4388 f"it would not change the request and could not resolve a SPLIT_REQUEST_WINDOW response." 4389 ) 4390 4391 parsed_cursor_granularity = parse_duration(incremental_sync.cursor_granularity) 4392 if isinstance( 4393 parsed_cursor_granularity, datetime.timedelta 4394 ) and parsed_cursor_granularity <= datetime.timedelta(0): 4395 # Passes the truthy check above (the string is non-empty), but a zero-or-negative granularity can 4396 # never produce a child strictly smaller than the parent. 4397 raise ValueError( 4398 f"`request_window_splitting` requires a `cursor_granularity` greater than zero on stream " 4399 f"{name}: `{incremental_sync.cursor_granularity}` parses to a zero-length duration, which can " 4400 f"never produce a smaller child window." 4401 ) 4402 4403 smallest_format_unit = self._smallest_datetime_format_unit(incremental_sync.datetime_format) 4404 if ( 4405 smallest_format_unit is not None 4406 and isinstance(parsed_cursor_granularity, datetime.timedelta) 4407 and parsed_cursor_granularity < smallest_format_unit 4408 ): 4409 # Two children split closer together than `datetime_format` can render would format to the same 4410 # boundary value instead of splitting. 4411 raise ValueError( 4412 f"`request_window_splitting` requires `cursor_granularity` to be no finer than what " 4413 f"`datetime_format` can represent on stream {name}: `datetime_format` " 4414 f"{incremental_sync.datetime_format!r} cannot represent a value finer than " 4415 f"{smallest_format_unit}, but `cursor_granularity` {incremental_sync.cursor_granularity!r} is " 4416 f"finer than that. Use a `datetime_format` precise enough to represent `cursor_granularity`, " 4417 f"or a coarser `cursor_granularity`." 4418 ) 4419 4420 if min_split_window: 4421 parsed_min_split_window = parse_duration(min_split_window) 4422 if not isinstance( 4423 parsed_min_split_window, datetime.timedelta 4424 ) or parsed_min_split_window <= datetime.timedelta(0): 4425 # A duration with years or months (e.g. `P1M`) parses to an `isodate.Duration`, which has no 4426 # fixed length to compare a window's span against - `split_request_window` would raise 4427 # mid-sync trying to. A zero-or-negative duration would never stop splitting. 4428 raise ValueError( 4429 f"`min_split_window` must be a positive duration expressible as a fixed number of " 4430 f"days/hours/minutes/seconds on stream {name}: `{min_split_window}` is not." 4431 ) 4432 4433 if query_properties: 4434 raise ValueError( 4435 f"`request_window_splitting` cannot be used together with query properties on stream {name}. " 4436 f"Records from the earlier property chunks may have already been emitted when a chunk asks to " 4437 f"split the window, so splitting and re-reading it could duplicate them." 4438 ) 4439 4440 if file_uploader: 4441 raise ValueError( 4442 f"`request_window_splitting` cannot be used together with a `file_uploader` on stream {name}. " 4443 f"The file uploader sends one request per record from inside the window's record generator, so " 4444 f"a split requested partway through could re-emit records already yielded by it." 4445 ) 4446 4447 @staticmethod 4448 def _smallest_datetime_format_unit(datetime_format: str) -> Optional[datetime.timedelta]: 4449 """ 4450 :return: the finest duration `datetime_format` can distinguish, per 4451 `_DATETIME_FORMAT_DIRECTIVE_GRANULARITY`, or `None` if it contains none of those directives. 4452 """ 4453 matches = [ 4454 granularity 4455 for directive, granularity in _DATETIME_FORMAT_DIRECTIVE_GRANULARITY 4456 if directive in datetime_format 4457 ] 4458 return min(matches) if matches else None 4459 4460 def _reject_split_request_window_action(self, requester: Any, description: str) -> None: 4461 """ 4462 `error_handler` is defined on `HttpRequester`, which is referenced by requesters that have no window of 4463 their own to split, so SPLIT_REQUEST_WINDOW is schema-legal in places where nothing can honor it. Only 4464 the main requester of a `SimpleRetriever` reads a cursor-sliced window, so every other requester is 4465 rejected here rather than surfacing the exception mid-sync as a generic failure. 4466 """ 4467 if requester is None: 4468 return 4469 if self._uses_split_request_window_action(getattr(requester, "error_handler", None)): 4470 raise ValueError( 4471 f"The SPLIT_REQUEST_WINDOW response action is not supported on the {description}: only the " 4472 f"main requester of a SimpleRetriever reads a window that can be split. Use a different " 4473 f"action on that error handler." 4474 ) 4475 4476 def _uses_split_request_window_action(self, error_handler: Any) -> bool: 4477 if isinstance(error_handler, CompositeErrorHandlerModel): 4478 return any( 4479 self._uses_split_request_window_action(nested) 4480 for nested in error_handler.error_handlers 4481 ) 4482 if isinstance(error_handler, DefaultErrorHandlerModel): 4483 return any( 4484 response_filter.action == HttpResponseFilterActionModel.SPLIT_REQUEST_WINDOW 4485 for response_filter in error_handler.response_filters or [] 4486 ) 4487 # A CustomErrorHandler can return any action and we cannot inspect it, so we do not validate it. 4488 return False 4489 4490 def _create_pagination_tracker_factory( 4491 self, model: Optional[PaginationResetModel], cursor: Cursor 4492 ) -> Callable[[], PaginationTracker]: 4493 if model is None: 4494 return lambda: PaginationTracker() 4495 4496 # Until we figure out a way to use any cursor for PaginationTracker, we will have to have this cursor selector logic 4497 cursor_factory: Callable[[], Optional[ConcurrentCursor]] = lambda: None 4498 if model.action == PaginationResetActionModel.RESET: 4499 # in that case, we will let cursor_factory to return None even if the stream has a cursor 4500 pass 4501 elif model.action == PaginationResetActionModel.SPLIT_USING_CURSOR: 4502 if isinstance(cursor, ConcurrentCursor): 4503 cursor_factory = lambda: cursor.copy_without_state() # type: ignore # the if condition validates that it is a ConcurrentCursor 4504 elif isinstance(cursor, ConcurrentPerPartitionCursor): 4505 cursor_factory = lambda: cursor._cursor_factory.create( # type: ignore # if this becomes a problem, we would need to extract the cursor_factory instantiation logic and make it accessible here 4506 {}, datetime.timedelta(0) 4507 ) 4508 elif not isinstance(cursor, FinalStateCursor): 4509 LOGGER.warning( 4510 "Unknown cursor for PaginationTracker. Pagination resets might not work properly" 4511 ) 4512 else: 4513 raise ValueError(f"Unknown PaginationReset action: {model.action}") 4514 4515 limit = model.limits.number_of_records if model and model.limits else None 4516 return lambda: PaginationTracker(cursor_factory(), limit) 4517 4518 def _get_log_formatter( 4519 self, log_formatter: Callable[[Response], Any] | None, name: str 4520 ) -> Callable[[Response], Any] | None: 4521 if self._should_limit_slices_fetched(): 4522 return ( 4523 ( 4524 lambda response: format_http_message( 4525 response, 4526 f"Stream '{name}' request", 4527 f"Request performed in order to extract records for stream '{name}'", 4528 name, 4529 ) 4530 ) 4531 if not log_formatter 4532 else log_formatter 4533 ) 4534 return None 4535 4536 def _should_limit_slices_fetched(self) -> bool: 4537 """ 4538 Returns True if the number of slices fetched should be limited, False otherwise. 4539 This is used to limit the number of slices fetched during tests. 4540 """ 4541 return bool(self._limit_slices_fetched or self._emit_connector_builder_messages) 4542 4543 @staticmethod 4544 def _has_query_properties_in_request_parameters( 4545 requester: Union[HttpRequesterModel, CustomRequesterModel], 4546 ) -> bool: 4547 if not hasattr(requester, "request_parameters"): 4548 return False 4549 request_parameters = requester.request_parameters 4550 if request_parameters and isinstance(request_parameters, Mapping): 4551 for request_parameter in request_parameters.values(): 4552 if isinstance(request_parameter, QueryPropertiesModel): 4553 return True 4554 return False 4555 4556 @staticmethod 4557 def _remove_query_properties( 4558 request_parameters: Mapping[str, Union[str, QueryPropertiesModel]], 4559 ) -> Mapping[str, str]: 4560 return { 4561 parameter_field: request_parameter 4562 for parameter_field, request_parameter in request_parameters.items() 4563 if not isinstance(request_parameter, QueryPropertiesModel) 4564 } 4565 4566 def create_state_delegating_stream( 4567 self, 4568 model: StateDelegatingStreamModel, 4569 config: Config, 4570 **kwargs: Any, 4571 ) -> DefaultStream: 4572 if ( 4573 model.full_refresh_stream.name != model.name 4574 or model.name != model.incremental_stream.name 4575 ): 4576 raise ValueError( 4577 f"state_delegating_stream, full_refresh_stream name and incremental_stream must have equal names. Instead has {model.name}, {model.full_refresh_stream.name} and {model.incremental_stream.name}." 4578 ) 4579 4580 # Resolve api_retention_period with config context (supports Jinja2 interpolation) 4581 resolved_retention_period: Optional[str] = None 4582 if model.api_retention_period: 4583 interpolated_retention = InterpolatedString.create( 4584 model.api_retention_period, parameters=model.parameters or {} 4585 ) 4586 resolved_value = interpolated_retention.eval(config=config) 4587 if resolved_value: 4588 resolved_retention_period = str(resolved_value) 4589 4590 if resolved_retention_period: 4591 for stream_model in (model.full_refresh_stream, model.incremental_stream): 4592 if isinstance(stream_model.incremental_sync, IncrementingCountCursorModel): 4593 raise ValueError( 4594 f"Stream '{model.name}' uses IncrementingCountCursor which is not supported " 4595 f"with api_retention_period. IncrementingCountCursor does not use datetime-based " 4596 f"cursors, so cursor age validation cannot be performed." 4597 ) 4598 4599 stream_state = self._connector_state_manager.get_stream_state(model.name, None) 4600 4601 if not stream_state: 4602 return self._create_component_from_model( # type: ignore[no-any-return] 4603 model.full_refresh_stream, config=config, **kwargs 4604 ) 4605 4606 incremental_stream: DefaultStream = self._create_component_from_model( 4607 model.incremental_stream, config=config, **kwargs 4608 ) # type: ignore[assignment] 4609 4610 # Only run cursor age validation for streams that are in the configured 4611 # catalog (or when no catalog was provided, e.g. during discover / connector 4612 # builder). Streams not selected by the user but instantiated as parent-stream 4613 # dependencies must not go through this path because it emits state messages 4614 # that the destination does not know about, causing "Stream not found" crashes. 4615 stream_is_in_catalog = ( 4616 not self._stream_name_to_configured_stream # no catalog → validate by default 4617 or model.name in self._stream_name_to_configured_stream 4618 ) 4619 if resolved_retention_period and stream_is_in_catalog: 4620 full_refresh_stream: DefaultStream = self._create_component_from_model( 4621 model.full_refresh_stream, config=config, **kwargs 4622 ) # type: ignore[assignment] 4623 if self._is_cursor_older_than_retention_period( 4624 stream_state, 4625 full_refresh_stream.cursor, 4626 incremental_stream.cursor, 4627 resolved_retention_period, 4628 model.name, 4629 ): 4630 # Clear state BEFORE constructing the full_refresh_stream so that 4631 # its cursor starts from start_date instead of the stale cursor. 4632 self._connector_state_manager.update_state_for_stream(model.name, None, {}) 4633 state_message = self._connector_state_manager.create_state_message(model.name, None) 4634 self._message_repository.emit_message(state_message) 4635 return self._create_component_from_model( # type: ignore[no-any-return] 4636 model.full_refresh_stream, config=config, **kwargs 4637 ) 4638 4639 return incremental_stream 4640 4641 @staticmethod 4642 def _is_cursor_older_than_retention_period( 4643 stream_state: Mapping[str, Any], 4644 full_refresh_cursor: Cursor, 4645 incremental_cursor: Cursor, 4646 api_retention_period: str, 4647 stream_name: str, 4648 ) -> bool: 4649 """Check if the cursor value in the state is older than the API's retention period. 4650 4651 Checks cursors in sequence: full refresh cursor first, then incremental cursor. 4652 FinalStateCursor returns now() for completed full refresh state (NO_CURSOR_STATE_KEY), 4653 which is always within retention, so we use incremental. For other states, it returns 4654 None and we fall back to checking the incremental cursor. 4655 4656 Returns True if the cursor is older than the retention period (should use full refresh). 4657 Returns False if the cursor is within the retention period (safe to use incremental). 4658 """ 4659 retention_duration = parse_duration(api_retention_period) 4660 retention_cutoff = datetime.datetime.now(datetime.timezone.utc) - retention_duration 4661 4662 # Check full refresh cursor first 4663 cursor_datetime = full_refresh_cursor.get_cursor_datetime_from_state(stream_state) 4664 4665 # If full refresh cursor returns None, check incremental cursor 4666 if cursor_datetime is None: 4667 cursor_datetime = incremental_cursor.get_cursor_datetime_from_state(stream_state) 4668 4669 if cursor_datetime is None: 4670 # Neither cursor could parse the state - fall back to full refresh to be safe 4671 return True 4672 4673 if cursor_datetime < retention_cutoff: 4674 logging.warning( 4675 f"Stream '{stream_name}' has a cursor value older than " 4676 f"the API's retention period of {api_retention_period} " 4677 f"(cutoff: {retention_cutoff.isoformat()}). " 4678 f"Falling back to full refresh to avoid data loss." 4679 ) 4680 return True 4681 4682 return False 4683 4684 def _get_state_delegating_stream_model( 4685 self, 4686 model: StateDelegatingStreamModel, 4687 parent_state: Optional[Mapping[str, Any]] = None, 4688 ) -> DeclarativeStreamModel: 4689 """Return the appropriate underlying stream model based on state.""" 4690 return ( 4691 model.incremental_stream 4692 if self._connector_state_manager.get_stream_state(model.name, None) or parent_state 4693 else model.full_refresh_stream 4694 ) 4695 4696 _OPTIONAL_ASYNC_STATUS_FIELDS = {"skipped"} 4697 4698 def _create_async_job_status_mapping( 4699 self, model: AsyncJobStatusMapModel, config: Config, **kwargs: Any 4700 ) -> Mapping[str, AsyncJobStatus]: 4701 api_status_to_cdk_status = {} 4702 for cdk_status, api_statuses in model.dict().items(): 4703 if cdk_status == "type": 4704 # This is an element of the dict because of the typing of the CDK but it is not a CDK status 4705 continue 4706 4707 if api_statuses is None: 4708 if cdk_status in self._OPTIONAL_ASYNC_STATUS_FIELDS: 4709 continue 4710 raise ValueError( 4711 f"Required CDK status '{cdk_status}' has no API statuses mapped. " 4712 f"Please provide at least an empty list for required status fields." 4713 ) 4714 4715 for status in api_statuses: 4716 if status in api_status_to_cdk_status: 4717 raise ValueError( 4718 f"API status {status} is already set for CDK status {cdk_status}. Please ensure API statuses are only provided once" 4719 ) 4720 api_status_to_cdk_status[status] = self._get_async_job_status(cdk_status) 4721 return api_status_to_cdk_status 4722 4723 def _get_async_job_status(self, status: str) -> AsyncJobStatus: 4724 match status: 4725 case "running": 4726 return AsyncJobStatus.RUNNING 4727 case "completed": 4728 return AsyncJobStatus.COMPLETED 4729 case "failed": 4730 return AsyncJobStatus.FAILED 4731 case "timeout": 4732 return AsyncJobStatus.TIMED_OUT 4733 case "skipped": 4734 return AsyncJobStatus.SKIPPED 4735 case _: 4736 raise ValueError(f"Unsupported CDK status {status}") 4737 4738 def create_async_retriever( 4739 self, 4740 model: AsyncRetrieverModel, 4741 config: Config, 4742 *, 4743 name: str, 4744 primary_key: Optional[ 4745 Union[str, List[str], List[List[str]]] 4746 ], # this seems to be needed to match create_simple_retriever 4747 stream_slicer: Optional[StreamSlicer], 4748 client_side_incremental_sync: Optional[Dict[str, Any]] = None, 4749 transformations: List[RecordTransformation], 4750 **kwargs: Any, 4751 ) -> AsyncRetriever: 4752 if model.download_target_requester and not model.download_target_extractor: 4753 raise ValueError( 4754 f"`download_target_extractor` required if using a `download_target_requester`" 4755 ) 4756 4757 for requester_field in ( 4758 "creation_requester", 4759 "polling_requester", 4760 "download_requester", 4761 "download_target_requester", 4762 "abort_requester", 4763 "delete_requester", 4764 ): 4765 self._reject_reduce_page_size_action( 4766 getattr(model, requester_field, None), 4767 f"`{requester_field}` of the AsyncRetriever of stream {name}", 4768 ) 4769 self._reject_split_request_window_action( 4770 getattr(model, requester_field, None), 4771 f"`{requester_field}` of the AsyncRetriever of stream {name}", 4772 ) 4773 4774 def _get_download_retriever( 4775 requester: Requester, extractor: RecordExtractor, _decoder: Decoder 4776 ) -> SimpleRetriever: 4777 self._count_dropped_empty_records(extractor) 4778 # We create a record selector for the download retriever 4779 # with no schema normalization and no transformations, neither record filter 4780 # as all this occurs in the record_selector of the AsyncRetriever 4781 record_selector = RecordSelector( 4782 extractor=extractor, 4783 name=name, 4784 record_filter=None, 4785 transformations=[], 4786 schema_normalization=TypeTransformer(TransformConfig.NoTransform), 4787 config=config, 4788 parameters={}, 4789 ) 4790 paginator = ( 4791 self._create_component_from_model( 4792 model=model.download_paginator, 4793 decoder=_decoder, 4794 config=config, 4795 url_base="", 4796 # Only read by the `union` rejection of `OffsetIncrement`. 4797 extractor_model=( 4798 model.download_extractor 4799 if isinstance(model.download_extractor, CombinedExtractorModel) 4800 else None 4801 ), 4802 ) 4803 if model.download_paginator 4804 else NoPagination(parameters={}) 4805 ) 4806 4807 return SimpleRetriever( 4808 requester=requester, 4809 record_selector=record_selector, 4810 primary_key=None, 4811 name=name, 4812 paginator=paginator, 4813 config=config, 4814 parameters={}, 4815 log_formatter=self._get_log_formatter(None, name), 4816 ) 4817 4818 def _get_job_timeout() -> datetime.timedelta: 4819 user_defined_timeout: Optional[int] = ( 4820 int( 4821 InterpolatedString.create( 4822 str(model.polling_job_timeout), 4823 parameters={}, 4824 ).eval(config) 4825 ) 4826 if model.polling_job_timeout 4827 else None 4828 ) 4829 4830 # check for user defined timeout during the test read or 15 minutes 4831 test_read_timeout = datetime.timedelta(minutes=user_defined_timeout or 15) 4832 # default value for non-connector builder is 60 minutes. 4833 default_sync_timeout = datetime.timedelta(minutes=user_defined_timeout or 60) 4834 4835 return ( 4836 test_read_timeout if self._emit_connector_builder_messages else default_sync_timeout 4837 ) 4838 4839 decoder = ( 4840 self._create_component_from_model(model=model.decoder, config=config) 4841 if model.decoder 4842 else JsonDecoder(parameters={}) 4843 ) 4844 record_selector = self._create_component_from_model( 4845 model=model.record_selector, 4846 config=config, 4847 decoder=decoder, 4848 name=name, 4849 transformations=transformations, 4850 client_side_incremental_sync=client_side_incremental_sync, 4851 ) 4852 4853 stream_slicer = stream_slicer or SinglePartitionRouter(parameters={}) 4854 if self._should_limit_slices_fetched(): 4855 stream_slicer = cast( 4856 StreamSlicer, 4857 StreamSlicerTestReadDecorator( 4858 wrapped_slicer=stream_slicer, 4859 maximum_number_of_slices=self._limit_slices_fetched or 5, 4860 ), 4861 ) 4862 4863 creation_requester = self._create_component_from_model( 4864 model=model.creation_requester, 4865 decoder=decoder, 4866 config=config, 4867 name=f"job creation - {name}", 4868 ) 4869 polling_requester = self._create_component_from_model( 4870 model=model.polling_requester, 4871 decoder=decoder, 4872 config=config, 4873 name=f"job polling - {name}", 4874 ) 4875 job_download_components_name = f"job download - {name}" 4876 download_decoder = ( 4877 self._create_component_from_model(model=model.download_decoder, config=config) 4878 if model.download_decoder 4879 else JsonDecoder(parameters={}) 4880 ) 4881 download_extractor = ( 4882 self._create_component_from_model( 4883 model=model.download_extractor, 4884 config=config, 4885 decoder=download_decoder, 4886 parameters=model.parameters, 4887 ) 4888 if model.download_extractor 4889 else DpathExtractor( 4890 [], 4891 config=config, 4892 decoder=download_decoder, 4893 parameters=model.parameters or {}, 4894 ) 4895 ) 4896 download_requester = self._create_component_from_model( 4897 model=model.download_requester, 4898 decoder=download_decoder, 4899 config=config, 4900 name=job_download_components_name, 4901 ) 4902 download_retriever = _get_download_retriever( 4903 download_requester, download_extractor, download_decoder 4904 ) 4905 abort_requester = ( 4906 self._create_component_from_model( 4907 model=model.abort_requester, 4908 decoder=decoder, 4909 config=config, 4910 name=f"job abort - {name}", 4911 ) 4912 if model.abort_requester 4913 else None 4914 ) 4915 delete_requester = ( 4916 self._create_component_from_model( 4917 model=model.delete_requester, 4918 decoder=decoder, 4919 config=config, 4920 name=f"job delete - {name}", 4921 ) 4922 if model.delete_requester 4923 else None 4924 ) 4925 download_target_requester = ( 4926 self._create_component_from_model( 4927 model=model.download_target_requester, 4928 decoder=decoder, 4929 config=config, 4930 name=f"job extract_url - {name}", 4931 ) 4932 if model.download_target_requester 4933 else None 4934 ) 4935 status_extractor = self._create_component_from_model( 4936 model=model.status_extractor, decoder=decoder, config=config, name=name 4937 ) 4938 download_target_extractor = ( 4939 self._create_component_from_model( 4940 model=model.download_target_extractor, 4941 decoder=decoder, 4942 config=config, 4943 name=name, 4944 ) 4945 if model.download_target_extractor 4946 else None 4947 ) 4948 4949 job_repository: AsyncJobRepository = AsyncHttpJobRepository( 4950 creation_requester=creation_requester, 4951 polling_requester=polling_requester, 4952 download_retriever=download_retriever, 4953 download_target_requester=download_target_requester, 4954 abort_requester=abort_requester, 4955 delete_requester=delete_requester, 4956 status_extractor=status_extractor, 4957 status_mapping=self._create_async_job_status_mapping(model.status_mapping, config), 4958 download_target_extractor=download_target_extractor, 4959 job_timeout=_get_job_timeout(), 4960 ) 4961 4962 failed_retry_wait_time_in_seconds: Optional[int] = ( 4963 int( 4964 InterpolatedString.create( 4965 str(model.failed_retry_wait_time_in_seconds), 4966 parameters={}, 4967 ).eval(config) 4968 ) 4969 if model.failed_retry_wait_time_in_seconds 4970 else None 4971 ) 4972 4973 async_job_partition_router = AsyncJobPartitionRouter( 4974 job_orchestrator_factory=lambda stream_slices: AsyncJobOrchestrator( 4975 job_repository, 4976 stream_slices, 4977 self._job_tracker, 4978 self._message_repository, 4979 # FIXME work would need to be done here in order to detect if a stream as a parent stream that is bulk 4980 has_bulk_parent=False, 4981 # set the `job_max_retry` to 1 for the `Connector Builder`` use-case. 4982 # `None` == default retry is set to 3 attempts, under the hood. 4983 job_max_retry=1 if self._emit_connector_builder_messages else None, 4984 failed_retry_wait_time_in_seconds=failed_retry_wait_time_in_seconds, 4985 ), 4986 stream_slicer=stream_slicer, 4987 config=config, 4988 parameters=model.parameters or {}, 4989 ) 4990 4991 return AsyncRetriever( 4992 record_selector=record_selector, 4993 stream_slicer=async_job_partition_router, 4994 config=config, 4995 parameters=model.parameters or {}, 4996 ) 4997 4998 def create_spec(self, model: SpecModel, config: Config, **kwargs: Any) -> Spec: 4999 config_migrations = [ 5000 self._create_component_from_model(migration, config) 5001 for migration in ( 5002 model.config_normalization_rules.config_migrations 5003 if ( 5004 model.config_normalization_rules 5005 and model.config_normalization_rules.config_migrations 5006 ) 5007 else [] 5008 ) 5009 ] 5010 config_transformations = [ 5011 self._create_component_from_model(transformation, config) 5012 for transformation in ( 5013 model.config_normalization_rules.transformations 5014 if ( 5015 model.config_normalization_rules 5016 and model.config_normalization_rules.transformations 5017 ) 5018 else [] 5019 ) 5020 ] 5021 config_validations = [ 5022 self._create_component_from_model(validation, config) 5023 for validation in ( 5024 model.config_normalization_rules.validations 5025 if ( 5026 model.config_normalization_rules 5027 and model.config_normalization_rules.validations 5028 ) 5029 else [] 5030 ) 5031 ] 5032 5033 return Spec( 5034 connection_specification=model.connection_specification, 5035 documentation_url=model.documentation_url, 5036 advanced_auth=model.advanced_auth, 5037 parameters={}, 5038 config_migrations=config_migrations, 5039 config_transformations=config_transformations, 5040 config_validations=config_validations, 5041 ) 5042 5043 def create_substream_partition_router( 5044 self, 5045 model: SubstreamPartitionRouterModel, 5046 config: Config, 5047 *, 5048 stream_name: str, 5049 **kwargs: Any, 5050 ) -> SubstreamPartitionRouter: 5051 parent_stream_configs = [] 5052 if model.parent_stream_configs: 5053 parent_stream_configs.extend( 5054 [ 5055 self.create_parent_stream_config_with_substream_wrapper( 5056 model=parent_stream_config, config=config, stream_name=stream_name, **kwargs 5057 ) 5058 for parent_stream_config in model.parent_stream_configs 5059 ] 5060 ) 5061 5062 return SubstreamPartitionRouter( 5063 parent_stream_configs=parent_stream_configs, 5064 parameters=model.parameters or {}, 5065 config=config, 5066 ) 5067 5068 def create_parent_stream_config_with_substream_wrapper( 5069 self, model: ParentStreamConfigModel, config: Config, *, stream_name: str, **kwargs: Any 5070 ) -> Any: 5071 child_state = self._connector_state_manager.get_stream_state(stream_name, None) 5072 if NO_CURSOR_STATE_KEY in child_state: 5073 # Full refresh streams checkpoint a `{NO_CURSOR_STATE_KEY: true}` sentinel. When such a 5074 # stream is later converted to incremental with an incremental_dependency parent, 5075 # `_instantiate_parent_stream_state_manager` would treat the sentinel's boolean as a legacy 5076 # cursor value and re-key it under the parent's cursor field, crashing cursor initialization. 5077 child_state = { 5078 key: value for key, value in child_state.items() if key != NO_CURSOR_STATE_KEY 5079 } 5080 5081 parent_state: Optional[Mapping[str, Any]] = ( 5082 child_state if model.incremental_dependency and child_state else None 5083 ) 5084 connector_state_manager = self._instantiate_parent_stream_state_manager( 5085 child_state, config, model, parent_state 5086 ) 5087 5088 substream_factory = ModelToComponentFactory( 5089 custom_components_trusted=self._custom_components_trusted, 5090 connector_state_manager=connector_state_manager, 5091 limit_pages_fetched_per_slice=self._limit_pages_fetched_per_slice, 5092 limit_slices_fetched=self._limit_slices_fetched, 5093 emit_connector_builder_messages=self._emit_connector_builder_messages, 5094 disable_retries=self._disable_retries, 5095 disable_cache=self._disable_cache, 5096 message_repository=StateFilteringMessageRepository( 5097 LogAppenderMessageRepositoryDecorator( 5098 { 5099 "airbyte_cdk": {"stream": {"is_substream": True}}, 5100 "http": {"is_auxiliary": True}, 5101 }, 5102 self._message_repository, 5103 self._evaluate_log_level(self._emit_connector_builder_messages), 5104 ), 5105 ), 5106 api_budget=self._api_budget, 5107 # Share the authenticator registry so parent and child streams draw from the 5108 # same token quota counters 5109 rate_limited_authenticators=self._rate_limited_authenticators, 5110 ) 5111 5112 return substream_factory.create_parent_stream_config( 5113 model=model, config=config, stream_name=stream_name, **kwargs 5114 ) 5115 5116 def _instantiate_parent_stream_state_manager( 5117 self, 5118 child_state: MutableMapping[str, Any], 5119 config: Config, 5120 model: ParentStreamConfigModel, 5121 parent_state: Optional[Mapping[str, Any]] = None, 5122 ) -> ConnectorStateManager: 5123 """ 5124 With DefaultStream, the state needs to be provided during __init__ of the cursor as opposed to the 5125 `set_initial_state` flow that existed for the declarative cursors. This state is taken from 5126 self._connector_state_manager.get_stream_state (`self` being a newly created ModelToComponentFactory to account 5127 for the MessageRepository being different). So we need to pass a ConnectorStateManager to the 5128 ModelToComponentFactory that has the parent states. This method populates this if there is a child state and if 5129 incremental_dependency is set. 5130 """ 5131 if model.incremental_dependency and child_state: 5132 parent_stream_name = model.stream.name or "" 5133 extracted_parent_state = ConcurrentPerPartitionCursor.get_parent_state( 5134 child_state, parent_stream_name 5135 ) 5136 5137 if not extracted_parent_state: 5138 extracted_parent_state = ConcurrentPerPartitionCursor.get_global_state( 5139 child_state, parent_stream_name 5140 ) 5141 5142 if not extracted_parent_state and not isinstance(extracted_parent_state, dict): 5143 cursor_values = list(child_state.values()) 5144 # Only a scalar legacy cursor value can seed the parent. Sentinels such as 5145 # `{"__ab_full_refresh_sync_complete": true}` or legacy per-partition `states` 5146 # would crash the parent cursor initialization. 5147 if ( 5148 len(cursor_values) == 1 5149 and isinstance(cursor_values[0], (str, int, float)) 5150 and not isinstance(cursor_values[0], bool) 5151 ): 5152 incremental_sync_model: Union[ 5153 DatetimeBasedCursorModel, 5154 IncrementingCountCursorModel, 5155 ] = ( 5156 model.stream.incremental_sync # type: ignore # if we are there, it is because there is incremental_dependency and therefore there is an incremental_sync on the parent stream 5157 if isinstance(model.stream, DeclarativeStreamModel) 5158 else self._get_state_delegating_stream_model( 5159 model.stream, parent_state=parent_state 5160 ).incremental_sync 5161 ) 5162 cursor_field = InterpolatedString.create( 5163 incremental_sync_model.cursor_field, 5164 parameters=incremental_sync_model.parameters or {}, 5165 ).eval(config) 5166 extracted_parent_state = AirbyteStateMessage( 5167 type=AirbyteStateType.STREAM, 5168 stream=AirbyteStreamState( 5169 stream_descriptor=StreamDescriptor( 5170 name=parent_stream_name, namespace=None 5171 ), 5172 stream_state=AirbyteStateBlob({cursor_field: cursor_values[0]}), 5173 ), 5174 ) 5175 return ConnectorStateManager([extracted_parent_state] if extracted_parent_state else []) 5176 5177 return ConnectorStateManager([]) 5178 5179 @staticmethod 5180 def create_wait_time_from_header( 5181 model: WaitTimeFromHeaderModel, config: Config, **kwargs: Any 5182 ) -> WaitTimeFromHeaderBackoffStrategy: 5183 return WaitTimeFromHeaderBackoffStrategy( 5184 header=model.header, 5185 parameters=model.parameters or {}, 5186 config=config, 5187 regex=model.regex, 5188 max_waiting_time_in_seconds=model.max_waiting_time_in_seconds, 5189 ) 5190 5191 @staticmethod 5192 def create_wait_until_time_from_header( 5193 model: WaitUntilTimeFromHeaderModel, config: Config, **kwargs: Any 5194 ) -> WaitUntilTimeFromHeaderBackoffStrategy: 5195 return WaitUntilTimeFromHeaderBackoffStrategy( 5196 header=model.header, 5197 parameters=model.parameters or {}, 5198 config=config, 5199 min_wait=model.min_wait, 5200 regex=model.regex, 5201 max_waiting_time_in_seconds=model.max_waiting_time_in_seconds, 5202 ) 5203 5204 def get_message_repository(self) -> MessageRepository: 5205 return self._message_repository 5206 5207 def _evaluate_log_level(self, emit_connector_builder_messages: bool) -> Level: 5208 return Level.DEBUG if emit_connector_builder_messages else Level.INFO 5209 5210 @staticmethod 5211 def create_components_mapping_definition( 5212 model: ComponentMappingDefinitionModel, config: Config, **kwargs: Any 5213 ) -> ComponentMappingDefinition: 5214 interpolated_value = InterpolatedString.create( 5215 model.value, parameters=model.parameters or {} 5216 ) 5217 field_path = [ 5218 InterpolatedString.create(path, parameters=model.parameters or {}) 5219 for path in model.field_path 5220 ] 5221 return ComponentMappingDefinition( 5222 field_path=field_path, # type: ignore[arg-type] # field_path can be str and InterpolatedString 5223 value=interpolated_value, 5224 value_type=ModelToComponentFactory._json_schema_type_name_to_type(model.value_type), 5225 create_or_update=model.create_or_update, 5226 condition=model.condition, 5227 parameters=model.parameters or {}, 5228 ) 5229 5230 def create_http_components_resolver( 5231 self, model: HttpComponentsResolverModel, config: Config, stream_name: Optional[str] = None 5232 ) -> Any: 5233 partition_router = self._build_stream_slicer_from_partition_router(model.retriever, config) 5234 retriever = self._create_component_from_model( 5235 model=model.retriever, 5236 config=config, 5237 name=f"{stream_name if stream_name else '__http_components_resolver'}", 5238 primary_key=None, 5239 stream_slicer=partition_router, 5240 partition_router=partition_router, 5241 transformations=[], 5242 ) 5243 5244 components_mapping = [] 5245 for component_mapping_definition_model in model.components_mapping: 5246 if component_mapping_definition_model.condition: 5247 raise ValueError("`condition` is only supported for `ConfigComponentsResolver`") 5248 components_mapping.append( 5249 self._create_component_from_model( 5250 model=component_mapping_definition_model, 5251 value_type=ModelToComponentFactory._json_schema_type_name_to_type( 5252 component_mapping_definition_model.value_type 5253 ), 5254 config=config, 5255 ) 5256 ) 5257 5258 return HttpComponentsResolver( 5259 retriever=retriever, 5260 # AsyncRetriever reads records only from the job slices its own slicer yields 5261 stream_slicer=retriever.stream_slicer 5262 if isinstance(retriever, AsyncRetriever) 5263 else partition_router, 5264 config=config, 5265 components_mapping=components_mapping, 5266 parameters=model.parameters or {}, 5267 ) 5268 5269 @staticmethod 5270 def create_stream_config( 5271 model: StreamConfigModel, config: Config, **kwargs: Any 5272 ) -> StreamConfig: 5273 model_configs_pointer: List[Union[InterpolatedString, str]] = ( 5274 [x for x in model.configs_pointer] if model.configs_pointer else [] 5275 ) 5276 5277 return StreamConfig( 5278 configs_pointer=model_configs_pointer, 5279 default_values=model.default_values, 5280 parameters=model.parameters or {}, 5281 ) 5282 5283 def create_config_components_resolver( 5284 self, 5285 model: ConfigComponentsResolverModel, 5286 config: Config, 5287 ) -> Any: 5288 model_stream_configs = ( 5289 model.stream_config if isinstance(model.stream_config, list) else [model.stream_config] 5290 ) 5291 5292 stream_configs = [ 5293 self._create_component_from_model( 5294 stream_config, config=config, parameters=model.parameters or {} 5295 ) 5296 for stream_config in model_stream_configs 5297 ] 5298 5299 components_mapping = [ 5300 self._create_component_from_model( 5301 model=components_mapping_definition_model, 5302 value_type=ModelToComponentFactory._json_schema_type_name_to_type( 5303 components_mapping_definition_model.value_type 5304 ), 5305 config=config, 5306 parameters=model.parameters, 5307 ) 5308 for components_mapping_definition_model in model.components_mapping 5309 ] 5310 5311 return ConfigComponentsResolver( 5312 stream_configs=stream_configs, 5313 config=config, 5314 components_mapping=components_mapping, 5315 parameters=model.parameters or {}, 5316 ) 5317 5318 def create_parametrized_components_resolver( 5319 self, 5320 model: ParametrizedComponentsResolverModel, 5321 config: Config, 5322 ) -> ParametrizedComponentsResolver: 5323 stream_parameters = StreamParametersDefinition( 5324 list_of_parameters_for_stream=model.stream_parameters.list_of_parameters_for_stream 5325 ) 5326 5327 components_mapping = [] 5328 for components_mapping_definition_model in model.components_mapping: 5329 if components_mapping_definition_model.condition: 5330 raise ValueError("`condition` is only supported for `ConfigComponentsResolver`") 5331 components_mapping.append( 5332 self._create_component_from_model( 5333 model=components_mapping_definition_model, 5334 value_type=ModelToComponentFactory._json_schema_type_name_to_type( 5335 components_mapping_definition_model.value_type 5336 ), 5337 config=config, 5338 ) 5339 ) 5340 return ParametrizedComponentsResolver( 5341 stream_parameters=stream_parameters, 5342 config=config, 5343 components_mapping=components_mapping, 5344 parameters=model.parameters or {}, 5345 ) 5346 5347 _UNSUPPORTED_DECODER_ERROR = ( 5348 "Specified decoder of {decoder_type} is not supported for pagination." 5349 "Please set as `JsonDecoder`, `XmlDecoder`, or a `CompositeRawDecoder` with an inner_parser of `JsonParser` or `GzipParser` instead." 5350 "If using `GzipParser`, please ensure that the lowest level inner_parser is a `JsonParser`." 5351 ) 5352 5353 def _is_supported_decoder_for_pagination(self, decoder: Decoder) -> bool: 5354 if isinstance(decoder, (JsonDecoder, XmlDecoder)): 5355 return True 5356 elif isinstance(decoder, CompositeRawDecoder): 5357 return self._is_supported_parser_for_pagination(decoder.parser) 5358 else: 5359 return False 5360 5361 def _is_supported_parser_for_pagination(self, parser: Parser) -> bool: 5362 if isinstance(parser, JsonParser): 5363 return True 5364 elif isinstance(parser, GzipParser): 5365 return isinstance(parser.inner_parser, JsonParser) 5366 else: 5367 return False 5368 5369 def create_http_api_budget( 5370 self, model: HTTPAPIBudgetModel, config: Config, **kwargs: Any 5371 ) -> HttpAPIBudget: 5372 policies = [ 5373 self._create_component_from_model(model=policy, config=config) 5374 for policy in model.policies 5375 ] 5376 5377 return HttpAPIBudget( 5378 policies=policies, 5379 ratelimit_reset_header=model.ratelimit_reset_header or "ratelimit-reset", 5380 ratelimit_remaining_header=model.ratelimit_remaining_header or "ratelimit-remaining", 5381 status_codes_for_ratelimit_hit=model.status_codes_for_ratelimit_hit or [429], 5382 ) 5383 5384 def create_fixed_window_call_rate_policy( 5385 self, model: FixedWindowCallRatePolicyModel, config: Config, **kwargs: Any 5386 ) -> FixedWindowCallRatePolicy: 5387 matchers = [ 5388 self._create_component_from_model(model=matcher, config=config) 5389 for matcher in model.matchers 5390 ] 5391 5392 # Set the initial reset timestamp to 10 days from now. 5393 # This value will be updated by the first request. 5394 return FixedWindowCallRatePolicy( 5395 next_reset_ts=datetime.datetime.now() + datetime.timedelta(days=10), 5396 period=parse_duration(model.period), 5397 call_limit=model.call_limit, 5398 matchers=matchers, 5399 ) 5400 5401 def create_file_uploader( 5402 self, model: FileUploaderModel, config: Config, **kwargs: Any 5403 ) -> FileUploader: 5404 self._reject_reduce_page_size_action(model.requester, "requester of a `file_uploader`") 5405 self._reject_split_request_window_action(model.requester, "requester of a `file_uploader`") 5406 name = "File Uploader" 5407 requester = self._create_component_from_model( 5408 model=model.requester, 5409 config=config, 5410 name=name, 5411 **kwargs, 5412 ) 5413 download_target_extractor = self._create_component_from_model( 5414 model=model.download_target_extractor, 5415 config=config, 5416 name=name, 5417 **kwargs, 5418 ) 5419 emit_connector_builder_messages = self._emit_connector_builder_messages 5420 file_uploader = DefaultFileUploader( 5421 requester=requester, 5422 download_target_extractor=download_target_extractor, 5423 config=config, 5424 file_writer=NoopFileWriter() 5425 if emit_connector_builder_messages 5426 else LocalFileSystemFileWriter(), 5427 parameters=model.parameters or {}, 5428 filename_extractor=model.filename_extractor if model.filename_extractor else None, 5429 ) 5430 5431 return ( 5432 ConnectorBuilderFileUploader(file_uploader) 5433 if emit_connector_builder_messages 5434 else file_uploader 5435 ) 5436 5437 def create_moving_window_call_rate_policy( 5438 self, model: MovingWindowCallRatePolicyModel, config: Config, **kwargs: Any 5439 ) -> MovingWindowCallRatePolicy: 5440 rates = [ 5441 self._create_component_from_model(model=rate, config=config) for rate in model.rates 5442 ] 5443 matchers = [ 5444 self._create_component_from_model(model=matcher, config=config) 5445 for matcher in model.matchers 5446 ] 5447 return MovingWindowCallRatePolicy( 5448 rates=rates, 5449 matchers=matchers, 5450 ) 5451 5452 def create_unlimited_call_rate_policy( 5453 self, model: UnlimitedCallRatePolicyModel, config: Config, **kwargs: Any 5454 ) -> UnlimitedCallRatePolicy: 5455 matchers = [ 5456 self._create_component_from_model(model=matcher, config=config) 5457 for matcher in model.matchers 5458 ] 5459 5460 return UnlimitedCallRatePolicy( 5461 matchers=matchers, 5462 ) 5463 5464 def create_rate(self, model: RateModel, config: Config, **kwargs: Any) -> Rate: 5465 interpolated_limit = InterpolatedString.create(str(model.limit), parameters={}) 5466 return Rate( 5467 limit=int(interpolated_limit.eval(config=config)), 5468 interval=parse_duration(model.interval), 5469 ) 5470 5471 def create_http_request_matcher( 5472 self, model: HttpRequestRegexMatcherModel, config: Config, **kwargs: Any 5473 ) -> HttpRequestRegexMatcher: 5474 weight = model.weight 5475 if weight is not None: 5476 if isinstance(weight, str): 5477 weight = int(InterpolatedString.create(weight, parameters={}).eval(config)) 5478 else: 5479 weight = int(weight) 5480 if weight < 1: 5481 raise ValueError(f"weight must be >= 1, got {weight}") 5482 return HttpRequestRegexMatcher( 5483 method=model.method, 5484 url_base=model.url_base, 5485 url_path_pattern=model.url_path_pattern, 5486 params=model.params, 5487 headers=model.headers, 5488 weight=weight, 5489 ) 5490 5491 def create_rate_limited_multiple_token_authenticator( 5492 self, 5493 model: RateLimitedMultipleTokenAuthenticatorModel, 5494 config: Config, 5495 **kwargs: Any, 5496 ) -> RateLimitedMultipleTokenAuthenticator: 5497 if isinstance(model.tokens, str): 5498 tokens_value = InterpolatedString.create(model.tokens, parameters={}).eval(config) 5499 delimiter = model.token_delimiter or "," 5500 tokens = [ 5501 token.strip() for token in str(tokens_value).split(delimiter) if token.strip() 5502 ] 5503 else: 5504 tokens = [ 5505 token_value 5506 for token in model.tokens 5507 if ( 5508 token_value := str( 5509 InterpolatedString.create(token, parameters={}).eval(config) 5510 ).strip() 5511 ) 5512 ] 5513 5514 quota_specs = [ 5515 { 5516 "name": quota_model.name, 5517 "remaining_path": quota_model.remaining_path, 5518 "reset_path": quota_model.reset_path, 5519 "limit_path": quota_model.limit_path, 5520 "remaining_header": quota_model.remaining_header, 5521 "reset_header": quota_model.reset_header, 5522 "limit_header": quota_model.limit_header, 5523 # Normalize the same way as the runtime TokenQuota below, so an omitted field 5524 # and an explicit `[]` key identically and keep sharing one set of counters. 5525 "exhaustion_status_codes": quota_model.exhaustion_status_codes or [], 5526 "matchers": [ 5527 { 5528 "method": matcher_model.method, 5529 "url_base": matcher_model.url_base, 5530 "url_path_pattern": matcher_model.url_path_pattern, 5531 "params": matcher_model.params, 5532 "headers": matcher_model.headers, 5533 "weight": matcher_model.weight, 5534 } 5535 for matcher_model in quota_model.matchers or [] 5536 ], 5537 } 5538 for quota_model in model.quotas 5539 ] 5540 5541 quota_status_url = str( 5542 InterpolatedString.create(model.quota_status_source.url, parameters={}).eval(config) 5543 ) 5544 quota_status_http_method = ( 5545 model.quota_status_source.http_method.value 5546 if model.quota_status_source.http_method 5547 else "GET" 5548 ) 5549 quota_status_headers = { 5550 key: str(InterpolatedString.create(value, parameters={}).eval(config)) 5551 for key, value in (model.quota_status_source.request_headers or {}).items() 5552 } 5553 # Normalize the same way as the quota specs above, so an omitted field and an explicit 5554 # `[]` key identically and keep sharing one set of counters. Deduplicated as well as 5555 # sorted, because the runtime turns this into a set: without it `[404]` and `[404, 404]` 5556 # would key differently and stop sharing counters while behaving identically. 5557 quota_status_unavailable_status_codes = sorted( 5558 set(model.quota_status_source.unavailable_status_codes or []) 5559 ) 5560 auth_method = model.auth_method or "Bearer" 5561 header = model.header or "Authorization" 5562 max_wait_time_str = str( 5563 InterpolatedString.create(model.max_wait_time or "PT2H", parameters={}).eval(config) 5564 ) 5565 max_wait_time = parse_duration(max_wait_time_str) 5566 if not isinstance(max_wait_time, datetime.timedelta): 5567 raise ValueError( 5568 f"max_wait_time must be a fixed-length ISO 8601 duration (e.g. 'PT2H'); " 5569 f"calendar-unit durations like '{max_wait_time_str}' are not supported" 5570 ) 5571 budget_reserve_fraction = ( 5572 model.budget_reserve_fraction if model.budget_reserve_fraction is not None else 0.1 5573 ) 5574 budget_min_reserve = ( 5575 model.budget_min_reserve if model.budget_min_reserve is not None else 50 5576 ) 5577 5578 # Reuse the same instance for identical definitions so that all streams share the 5579 # same token quota counters (similar to how api_budget is shared). The key is built 5580 # from the resolved constructor arguments rather than the raw model so that 5581 # stream-specific `$parameters` propagated onto the model (and its nested components) 5582 # cannot break instance sharing. 5583 cache_key = json.dumps( 5584 { 5585 "tokens": tokens, 5586 "quotas": quota_specs, 5587 "quota_status_url": quota_status_url, 5588 "quota_status_http_method": quota_status_http_method, 5589 "quota_status_headers": quota_status_headers, 5590 "quota_status_unavailable_status_codes": quota_status_unavailable_status_codes, 5591 "auth_method": auth_method, 5592 "header": header, 5593 "max_wait_time": max_wait_time.total_seconds(), 5594 "budget_reserve_fraction": budget_reserve_fraction, 5595 "budget_min_reserve": budget_min_reserve, 5596 }, 5597 sort_keys=True, 5598 ) 5599 if cache_key in self._rate_limited_authenticators: 5600 return self._rate_limited_authenticators[cache_key] 5601 5602 quotas = [ 5603 TokenQuota( 5604 name=quota_model.name, 5605 remaining_path=quota_model.remaining_path, 5606 reset_path=quota_model.reset_path, 5607 limit_path=quota_model.limit_path, 5608 remaining_header=quota_model.remaining_header, 5609 reset_header=quota_model.reset_header, 5610 limit_header=quota_model.limit_header, 5611 exhaustion_status_codes=quota_model.exhaustion_status_codes or [], 5612 matchers=[ 5613 self.create_http_request_matcher(matcher_model, config) 5614 for matcher_model in quota_model.matchers or [] 5615 ], 5616 ) 5617 for quota_model in model.quotas 5618 ] 5619 5620 authenticator = RateLimitedMultipleTokenAuthenticator( 5621 tokens=tokens, 5622 quotas=quotas, 5623 quota_status_url=quota_status_url, 5624 quota_status_http_method=quota_status_http_method, 5625 quota_status_headers=quota_status_headers, 5626 quota_status_unavailable_status_codes=quota_status_unavailable_status_codes, 5627 auth_method=auth_method, 5628 header=header, 5629 max_wait_time=max_wait_time, 5630 budget_reserve_fraction=budget_reserve_fraction, 5631 budget_min_reserve=budget_min_reserve, 5632 ) 5633 self._rate_limited_authenticators[cache_key] = authenticator 5634 return authenticator 5635 5636 def set_api_budget(self, component_definition: ComponentDefinition, config: Config) -> None: 5637 self._api_budget = self.create_component( 5638 model_type=HTTPAPIBudgetModel, component_definition=component_definition, config=config 5639 ) 5640 5641 def create_grouping_partition_router( 5642 self, 5643 model: GroupingPartitionRouterModel, 5644 config: Config, 5645 *, 5646 stream_name: str, 5647 **kwargs: Any, 5648 ) -> GroupingPartitionRouter: 5649 underlying_router = self._create_component_from_model( 5650 model=model.underlying_partition_router, 5651 config=config, 5652 stream_name=stream_name, 5653 **kwargs, 5654 ) 5655 if model.group_size < 1: 5656 raise ValueError(f"Group size must be greater than 0, got {model.group_size}") 5657 5658 # Request options in underlying partition routers are not supported for GroupingPartitionRouter 5659 # because they are specific to individual partitions and cannot be aggregated or handled 5660 # when grouping, potentially leading to incorrect API calls. Any request customization 5661 # should be managed at the stream level through the requester's configuration. 5662 if isinstance(underlying_router, SubstreamPartitionRouter): 5663 if any( 5664 parent_config.request_option 5665 for parent_config in underlying_router.parent_stream_configs 5666 ): 5667 raise ValueError("Request options are not supported for GroupingPartitionRouter.") 5668 5669 if isinstance(underlying_router, ListPartitionRouter): 5670 if underlying_router.request_option: 5671 raise ValueError("Request options are not supported for GroupingPartitionRouter.") 5672 5673 return GroupingPartitionRouter( 5674 group_size=model.group_size, 5675 underlying_partition_router=underlying_router, 5676 deduplicate=model.deduplicate if model.deduplicate is not None else True, 5677 config=config, 5678 ) 5679 5680 def create_union_partition_router( 5681 self, 5682 model: UnionPartitionRouterModel, 5683 config: Config, 5684 *, 5685 stream_name: str, 5686 **kwargs: Any, 5687 ) -> UnionPartitionRouter: 5688 # The schema enforces minItems: 2 for manifests; this guard covers construction paths 5689 # that bypass JSON-schema validation (the generated model carries no min_items constraint). 5690 if len(model.partition_routers) < 2: 5691 raise ValueError( 5692 f"UnionPartitionRouter for stream {stream_name} needs at least 2 child partition routers" 5693 ) 5694 5695 partition_routers = [ 5696 self._create_component_from_model( 5697 model=child, 5698 config=config, 5699 stream_name=stream_name, 5700 **kwargs, 5701 ) 5702 for child in model.partition_routers 5703 ] 5704 5705 # partition_field depends only on config/parameters, so it is evaluated once at build 5706 # time; the runtime component always receives a plain string. 5707 partition_field = InterpolatedString.create( 5708 model.partition_field, parameters=model.parameters or {} 5709 ).eval(config) 5710 5711 # Fail fast at build time when a built-in child router is statically known to emit a 5712 # partition field different from the union's. CustomPartitionRouter children are opaque 5713 # and can only be validated at runtime. 5714 for child_model in model.partition_routers: 5715 child_partition_fields: List[str] = [] 5716 if isinstance(child_model, ListPartitionRouterModel): 5717 child_partition_fields.append( 5718 InterpolatedString.create( 5719 child_model.cursor_field, parameters=child_model.parameters or {} 5720 ).eval(config) 5721 ) 5722 elif isinstance(child_model, SubstreamPartitionRouterModel): 5723 for parent_stream_config in child_model.parent_stream_configs: 5724 child_partition_fields.append( 5725 InterpolatedString.create( 5726 parent_stream_config.partition_field, 5727 parameters=parent_stream_config.parameters 5728 or child_model.parameters 5729 or {}, 5730 ).eval(config) 5731 ) 5732 elif isinstance(child_model, UnionPartitionRouterModel): 5733 child_partition_fields.append( 5734 InterpolatedString.create( 5735 child_model.partition_field, parameters=child_model.parameters or {} 5736 ).eval(config) 5737 ) 5738 for child_partition_field in child_partition_fields: 5739 if child_partition_field != partition_field: 5740 raise ValueError( 5741 f"UnionPartitionRouter expects all child partition routers to emit the " 5742 f"partition field '{partition_field}', but a " 5743 f"{child_model.type} child emits '{child_partition_field}'." 5744 ) 5745 5746 # A union slice comes from exactly one child partition router, so request options 5747 # declared on children cannot be applied consistently to requests built from the 5748 # normalized union slices. Partition values should be consumed via interpolation 5749 # (e.g. stream_partition) instead. Note that this validation only covers built-in 5750 # router types; CustomPartitionRouter children are opaque, so any request options 5751 # they implement internally cannot be detected or rejected here. 5752 for router in partition_routers: 5753 if isinstance(router, SubstreamPartitionRouter): 5754 if any( 5755 parent_config.request_option for parent_config in router.parent_stream_configs 5756 ): 5757 raise ValueError("Request options are not supported for UnionPartitionRouter.") 5758 if isinstance(router, ListPartitionRouter) and router.request_option: 5759 raise ValueError("Request options are not supported for UnionPartitionRouter.") 5760 5761 return UnionPartitionRouter( 5762 partition_routers=partition_routers, 5763 partition_field=partition_field, 5764 parameters=model.parameters or {}, 5765 ) 5766 5767 def _ensure_query_properties_to_model( 5768 self, requester: Union[HttpRequesterModel, CustomRequesterModel] 5769 ) -> None: 5770 """ 5771 For some reason, it seems like CustomRequesterModel request_parameters stays as dictionaries which means that 5772 the other conditions relying on it being QueryPropertiesModel instead of a dict fail. Here, we migrate them to 5773 proper model. 5774 """ 5775 if not hasattr(requester, "request_parameters"): 5776 return 5777 5778 request_parameters = requester.request_parameters 5779 if request_parameters and isinstance(request_parameters, Dict): 5780 for request_parameter_key in request_parameters.keys(): 5781 request_parameter = request_parameters[request_parameter_key] 5782 if ( 5783 isinstance(request_parameter, Dict) 5784 and request_parameter.get("type") == "QueryProperties" 5785 ): 5786 request_parameters[request_parameter_key] = QueryPropertiesModel.parse_obj( 5787 request_parameter 5788 ) 5789 5790 def _get_catalog_defined_cursor_field( 5791 self, stream_name: str, allow_catalog_defined_cursor_field: bool 5792 ) -> Optional[CursorField]: 5793 if not allow_catalog_defined_cursor_field: 5794 return None 5795 5796 configured_stream = self._stream_name_to_configured_stream.get(stream_name) 5797 5798 # Depending on the operation is being performed, there may not be a configured stream yet. In this 5799 # case we return None which will then use the default cursor field defined on the cursor model. 5800 # We also treat cursor_field: [""] (list with empty string) as no cursor field, since this can 5801 # occur when the platform serializes "no cursor configured" streams incorrectly. 5802 if ( 5803 not configured_stream 5804 or not configured_stream.cursor_field 5805 or not configured_stream.cursor_field[0] 5806 ): 5807 return None 5808 elif len(configured_stream.cursor_field) > 1: 5809 raise ValueError( 5810 f"The `{stream_name}` stream does not support nested cursor_field. Please specify only a single cursor_field for the stream in the configured catalog." 5811 ) 5812 else: 5813 return CursorField( 5814 cursor_field_key=configured_stream.cursor_field[0], 5815 supports_catalog_defined_cursor_field=allow_catalog_defined_cursor_field, 5816 )
ComponentDefinition =
typing.Mapping[str, typing.Any]
SCHEMA_TRANSFORMER_TYPE_MAPPING =
{<SchemaNormalization.None_: 'None'>: <TransformConfig.NoTransform: 1>, <SchemaNormalization.Default: 'Default'>: <TransformConfig.DefaultSchemaNormalization: 2>}
MAX_SLICES =
5
LOGGER =
<Logger airbyte.model_to_component_factory (INFO)>
class
ModelToComponentFactory:
752class ModelToComponentFactory: 753 EPOCH_DATETIME_FORMAT = "%s" 754 755 def __init__( 756 self, 757 limit_pages_fetched_per_slice: Optional[int] = None, 758 limit_slices_fetched: Optional[int] = None, 759 emit_connector_builder_messages: bool = False, 760 disable_retries: bool = False, 761 disable_cache: bool = False, 762 message_repository: Optional[MessageRepository] = None, 763 connector_state_manager: Optional[ConnectorStateManager] = None, 764 max_concurrent_async_job_count: Optional[int] = None, 765 configured_catalog: Optional[ConfiguredAirbyteCatalog] = None, 766 api_budget: Optional[APIBudget] = None, 767 rate_limited_authenticators: Optional[ 768 Dict[str, RateLimitedMultipleTokenAuthenticator] 769 ] = None, 770 custom_components_trusted: bool = True, 771 ): 772 self._init_mappings() 773 self._custom_components_trusted = custom_components_trusted 774 self._limit_pages_fetched_per_slice = limit_pages_fetched_per_slice 775 self._limit_slices_fetched = limit_slices_fetched 776 self._emit_connector_builder_messages = emit_connector_builder_messages 777 self._disable_retries = disable_retries 778 self._disable_cache = disable_cache 779 self._message_repository = message_repository or InMemoryMessageRepository( 780 self._evaluate_log_level(emit_connector_builder_messages) 781 ) 782 self._stream_name_to_configured_stream = self._create_stream_name_to_configured_stream( 783 configured_catalog 784 ) 785 self._connector_state_manager = connector_state_manager or ConnectorStateManager() 786 self._api_budget: Optional[Union[APIBudget]] = api_budget 787 # Shared instances so all streams see the same token quota counters (like api_budget) 788 self._rate_limited_authenticators: Dict[str, RateLimitedMultipleTokenAuthenticator] = ( 789 rate_limited_authenticators if rate_limited_authenticators is not None else {} 790 ) 791 self._job_tracker: JobTracker = JobTracker(max_concurrent_async_job_count or 1) 792 # placeholder for deprecation warnings 793 self._collected_deprecation_logs: List[ConnectorBuilderLogMessage] = [] 794 795 def _init_mappings(self) -> None: 796 self.PYDANTIC_MODEL_TO_CONSTRUCTOR: Mapping[Type[BaseModel], Callable[..., Any]] = { 797 AddedFieldDefinitionModel: self.create_added_field_definition, 798 AddFieldsModel: self.create_add_fields, 799 ApiKeyAuthenticatorModel: self.create_api_key_authenticator, 800 BasicHttpAuthenticatorModel: self.create_basic_http_authenticator, 801 BearerAuthenticatorModel: self.create_bearer_authenticator, 802 CheckStreamModel: self.create_check_stream, 803 DynamicStreamCheckConfigModel: self.create_dynamic_stream_check_config, 804 CheckDynamicStreamModel: self.create_check_dynamic_stream, 805 CompositeErrorHandlerModel: self.create_composite_error_handler, 806 ConcurrencyLevelModel: self.create_concurrency_level, 807 ConfigMigrationModel: self.create_config_migration, 808 ConfigAddFieldsModel: self.create_config_add_fields, 809 ConfigRemapFieldModel: self.create_config_remap_field, 810 ConfigRemoveFieldsModel: self.create_config_remove_fields, 811 ConstantBackoffStrategyModel: self.create_constant_backoff_strategy, 812 CsvDecoderModel: self.create_csv_decoder, 813 CursorPaginationModel: self.create_cursor_pagination, 814 CustomAuthenticatorModel: self.create_custom_component, 815 CustomBackoffStrategyModel: self.create_custom_component, 816 CustomDecoderModel: self.create_custom_component, 817 CustomErrorHandlerModel: self.create_custom_component, 818 CustomRecordExtractorModel: self.create_custom_component, 819 CustomRecordFilterModel: self.create_custom_component, 820 CustomRequesterModel: self.create_custom_component, 821 CustomRetrieverModel: self.create_custom_component, 822 CustomSchemaLoader: self.create_custom_component, 823 CustomSchemaNormalizationModel: self.create_custom_component, 824 CustomStateMigration: self.create_custom_component, 825 CustomPaginationStrategyModel: self.create_custom_component, 826 CustomPartitionRouterModel: self.create_custom_component, 827 CustomTransformationModel: self.create_custom_component, 828 CustomValidationStrategyModel: self.create_custom_component, 829 CustomConfigTransformationModel: self.create_custom_component, 830 DeclarativeStreamModel: self.create_default_stream, 831 DefaultErrorHandlerModel: self.create_default_error_handler, 832 DefaultPaginatorModel: self.create_default_paginator, 833 CombinedExtractorModel: self.create_combined_extractor, 834 DpathExtractorModel: self.create_dpath_extractor, 835 DpathValidatorModel: self.create_dpath_validator, 836 ResponseToFileExtractorModel: self.create_response_to_file_extractor, 837 ExponentialBackoffStrategyModel: self.create_exponential_backoff_strategy, 838 SessionTokenAuthenticatorModel: self.create_session_token_authenticator, 839 GroupByKeyMergeStrategyModel: self.create_group_by_key, 840 HttpRequesterModel: self.create_http_requester, 841 HttpResponseFilterModel: self.create_http_response_filter, 842 InlineSchemaLoaderModel: self.create_inline_schema_loader, 843 JsonDecoderModel: self.create_json_decoder, 844 JsonItemsDecoderModel: self.create_json_items_decoder, 845 JsonlDecoderModel: self.create_jsonl_decoder, 846 JsonSchemaPropertySelectorModel: self.create_json_schema_property_selector, 847 GzipDecoderModel: self.create_gzip_decoder, 848 KeysToLowerModel: self.create_keys_to_lower_transformation, 849 KeysToSnakeCaseModel: self.create_keys_to_snake_transformation, 850 KeysReplaceModel: self.create_keys_replace_transformation, 851 FlattenFieldsModel: self.create_flatten_fields, 852 DpathFlattenFieldsModel: self.create_dpath_flatten_fields, 853 IterableDecoderModel: self.create_iterable_decoder, 854 XmlDecoderModel: self.create_xml_decoder, 855 JsonFileSchemaLoaderModel: self.create_json_file_schema_loader, 856 DynamicSchemaLoaderModel: self.create_dynamic_schema_loader, 857 SchemaTypeIdentifierModel: self.create_schema_type_identifier, 858 TypesMapModel: self.create_types_map, 859 ComplexFieldTypeModel: self.create_complex_field_type, 860 JwtAuthenticatorModel: self.create_jwt_authenticator, 861 LegacyToPerPartitionStateMigrationModel: self.create_legacy_to_per_partition_state_migration, 862 ListPartitionRouterModel: self.create_list_partition_router, 863 MinMaxDatetimeModel: self.create_min_max_datetime, 864 NoAuthModel: self.create_no_auth, 865 NoPaginationModel: self.create_no_pagination, 866 OAuthAuthenticatorModel: self.create_oauth_authenticator, 867 OffsetIncrementModel: self.create_offset_increment, 868 PageIncrementModel: self.create_page_increment, 869 ParentStreamConfigModel: self.create_parent_stream_config_with_substream_wrapper, 870 PredicateValidatorModel: self.create_predicate_validator, 871 PropertiesFromEndpointModel: self.create_properties_from_endpoint, 872 PropertyChunkingModel: self.create_property_chunking, 873 QueryPropertiesModel: self.create_query_properties, 874 ParentFieldPathModel: self.create_parent_field_path, 875 RecordExpanderModel: self.create_record_expander, 876 RecordFilterModel: self.create_record_filter, 877 RecordSelectorModel: self.create_record_selector, 878 RemoveFieldsModel: self.create_remove_fields, 879 RequestPathModel: self.create_request_path, 880 RequestOptionModel: self.create_request_option, 881 LegacySessionTokenAuthenticatorModel: self.create_legacy_session_token_authenticator, 882 SelectiveAuthenticatorModel: self.create_selective_authenticator, 883 SimpleRetrieverModel: self.create_simple_retriever, 884 StateDelegatingStreamModel: self.create_state_delegating_stream, 885 SpecModel: self.create_spec, 886 SubstreamPartitionRouterModel: self.create_substream_partition_router, 887 ValidateAdheresToSchemaModel: self.create_validate_adheres_to_schema, 888 WaitTimeFromHeaderModel: self.create_wait_time_from_header, 889 WaitUntilTimeFromHeaderModel: self.create_wait_until_time_from_header, 890 AsyncRetrieverModel: self.create_async_retriever, 891 HttpComponentsResolverModel: self.create_http_components_resolver, 892 ConfigComponentsResolverModel: self.create_config_components_resolver, 893 ParametrizedComponentsResolverModel: self.create_parametrized_components_resolver, 894 StreamConfigModel: self.create_stream_config, 895 ComponentMappingDefinitionModel: self.create_components_mapping_definition, 896 ZipfileDecoderModel: self.create_zipfile_decoder, 897 HTTPAPIBudgetModel: self.create_http_api_budget, 898 FileUploaderModel: self.create_file_uploader, 899 FixedWindowCallRatePolicyModel: self.create_fixed_window_call_rate_policy, 900 MovingWindowCallRatePolicyModel: self.create_moving_window_call_rate_policy, 901 UnlimitedCallRatePolicyModel: self.create_unlimited_call_rate_policy, 902 RateModel: self.create_rate, 903 HttpRequestRegexMatcherModel: self.create_http_request_matcher, 904 RateLimitedMultipleTokenAuthenticatorModel: self.create_rate_limited_multiple_token_authenticator, 905 GroupingPartitionRouterModel: self.create_grouping_partition_router, 906 UnionPartitionRouterModel: self.create_union_partition_router, 907 } 908 909 # Needed for the case where we need to perform a second parse on the fields of a custom component 910 self.TYPE_NAME_TO_MODEL = {cls.__name__: cls for cls in self.PYDANTIC_MODEL_TO_CONSTRUCTOR} 911 912 @staticmethod 913 def _create_stream_name_to_configured_stream( 914 configured_catalog: Optional[ConfiguredAirbyteCatalog], 915 ) -> Mapping[str, ConfiguredAirbyteStream]: 916 return ( 917 {stream.stream.name: stream for stream in configured_catalog.streams} 918 if configured_catalog 919 else {} 920 ) 921 922 def create_component( 923 self, 924 model_type: Type[BaseModel], 925 component_definition: ComponentDefinition, 926 config: Config, 927 **kwargs: Any, 928 ) -> Any: 929 """ 930 Takes a given Pydantic model type and Mapping representing a component definition and creates a declarative component and 931 subcomponents which will be used at runtime. This is done by first parsing the mapping into a Pydantic model and then creating 932 creating declarative components from that model. 933 934 :param model_type: The type of declarative component that is being initialized 935 :param component_definition: The mapping that represents a declarative component 936 :param config: The connector config that is provided by the customer 937 :return: The declarative component to be used at runtime 938 """ 939 940 component_type = component_definition.get("type") 941 if component_definition.get("type") != model_type.__name__: 942 raise ValueError( 943 f"Expected manifest component of type {model_type.__name__}, but received {component_type} instead" 944 ) 945 946 declarative_component_model = model_type.parse_obj(component_definition) 947 948 if not isinstance(declarative_component_model, model_type): 949 raise ValueError( 950 f"Expected {model_type.__name__} component, but received {declarative_component_model.__class__.__name__}" 951 ) 952 953 return self._create_component_from_model( 954 model=declarative_component_model, config=config, **kwargs 955 ) 956 957 def _create_component_from_model(self, model: BaseModel, config: Config, **kwargs: Any) -> Any: 958 if model.__class__ not in self.PYDANTIC_MODEL_TO_CONSTRUCTOR: 959 raise ValueError( 960 f"{model.__class__} with attributes {model} is not a valid component type" 961 ) 962 component_constructor = self.PYDANTIC_MODEL_TO_CONSTRUCTOR.get(model.__class__) 963 if not component_constructor: 964 raise ValueError(f"Could not find constructor for {model.__class__}") 965 966 # collect deprecation warnings for supported models. 967 if isinstance(model, BaseModelWithDeprecations): 968 self._collect_model_deprecations(model) 969 970 return component_constructor(model=model, config=config, **kwargs) 971 972 def get_model_deprecations(self) -> List[ConnectorBuilderLogMessage]: 973 """ 974 Returns the deprecation warnings that were collected during the creation of components. 975 """ 976 return self._collected_deprecation_logs 977 978 def _collect_model_deprecations(self, model: BaseModelWithDeprecations) -> None: 979 """ 980 Collects deprecation logs from the given model and appends any new logs to the internal collection. 981 982 This method checks if the provided model has deprecation logs (identified by the presence of the DEPRECATION_LOGS_TAG attribute and a non-None `_deprecation_logs` property). It iterates through each deprecation log in the model and appends it to the `_collected_deprecation_logs` list if it has not already been collected, ensuring that duplicate logs are avoided. 983 984 Args: 985 model (BaseModelWithDeprecations): The model instance from which to collect deprecation logs. 986 """ 987 if hasattr(model, DEPRECATION_LOGS_TAG) and model._deprecation_logs is not None: 988 for log in model._deprecation_logs: 989 # avoid duplicates for deprecation logs observed. 990 if log not in self._collected_deprecation_logs: 991 self._collected_deprecation_logs.append(log) 992 993 def create_config_migration( 994 self, model: ConfigMigrationModel, config: Config 995 ) -> ConfigMigration: 996 transformations: List[ConfigTransformation] = [ 997 self._create_component_from_model(transformation, config) 998 for transformation in model.transformations 999 ] 1000 1001 return ConfigMigration( 1002 description=model.description, 1003 transformations=transformations, 1004 ) 1005 1006 def create_config_add_fields( 1007 self, model: ConfigAddFieldsModel, config: Config, **kwargs: Any 1008 ) -> ConfigAddFields: 1009 fields = [self._create_component_from_model(field, config) for field in model.fields] 1010 return ConfigAddFields( 1011 fields=fields, 1012 condition=model.condition or "", 1013 ) 1014 1015 @staticmethod 1016 def create_config_remove_fields( 1017 model: ConfigRemoveFieldsModel, config: Config, **kwargs: Any 1018 ) -> ConfigRemoveFields: 1019 return ConfigRemoveFields( 1020 field_pointers=model.field_pointers, 1021 condition=model.condition or "", 1022 ) 1023 1024 @staticmethod 1025 def create_config_remap_field( 1026 model: ConfigRemapFieldModel, config: Config, **kwargs: Any 1027 ) -> ConfigRemapField: 1028 mapping = cast(Mapping[str, Any], model.map) 1029 return ConfigRemapField( 1030 map=mapping, 1031 field_path=model.field_path, 1032 config=config, 1033 ) 1034 1035 def create_dpath_validator(self, model: DpathValidatorModel, config: Config) -> DpathValidator: 1036 strategy = self._create_component_from_model(model.validation_strategy, config) 1037 1038 return DpathValidator( 1039 field_path=model.field_path, 1040 strategy=strategy, 1041 ) 1042 1043 def create_predicate_validator( 1044 self, model: PredicateValidatorModel, config: Config 1045 ) -> PredicateValidator: 1046 strategy = self._create_component_from_model(model.validation_strategy, config) 1047 1048 return PredicateValidator( 1049 value=model.value, 1050 strategy=strategy, 1051 ) 1052 1053 @staticmethod 1054 def create_validate_adheres_to_schema( 1055 model: ValidateAdheresToSchemaModel, config: Config, **kwargs: Any 1056 ) -> ValidateAdheresToSchema: 1057 base_schema = cast(Mapping[str, Any], model.base_schema) 1058 return ValidateAdheresToSchema( 1059 schema=base_schema, 1060 ) 1061 1062 @staticmethod 1063 def create_added_field_definition( 1064 model: AddedFieldDefinitionModel, config: Config, **kwargs: Any 1065 ) -> AddedFieldDefinition: 1066 interpolated_value = InterpolatedString.create( 1067 model.value, parameters=model.parameters or {} 1068 ) 1069 return AddedFieldDefinition( 1070 path=model.path, 1071 value=interpolated_value, 1072 value_type=ModelToComponentFactory._json_schema_type_name_to_type(model.value_type), 1073 parameters=model.parameters or {}, 1074 ) 1075 1076 def create_add_fields(self, model: AddFieldsModel, config: Config, **kwargs: Any) -> AddFields: 1077 added_field_definitions = [ 1078 self._create_component_from_model( 1079 model=added_field_definition_model, 1080 value_type=ModelToComponentFactory._json_schema_type_name_to_type( 1081 added_field_definition_model.value_type 1082 ), 1083 config=config, 1084 ) 1085 for added_field_definition_model in model.fields 1086 ] 1087 return AddFields( 1088 fields=added_field_definitions, 1089 condition=model.condition or "", 1090 parameters=model.parameters or {}, 1091 ) 1092 1093 def create_keys_to_lower_transformation( 1094 self, model: KeysToLowerModel, config: Config, **kwargs: Any 1095 ) -> KeysToLowerTransformation: 1096 return KeysToLowerTransformation() 1097 1098 def create_keys_to_snake_transformation( 1099 self, model: KeysToSnakeCaseModel, config: Config, **kwargs: Any 1100 ) -> KeysToSnakeCaseTransformation: 1101 return KeysToSnakeCaseTransformation() 1102 1103 def create_keys_replace_transformation( 1104 self, model: KeysReplaceModel, config: Config, **kwargs: Any 1105 ) -> KeysReplaceTransformation: 1106 return KeysReplaceTransformation( 1107 old=model.old, new=model.new, parameters=model.parameters or {} 1108 ) 1109 1110 def create_flatten_fields( 1111 self, model: FlattenFieldsModel, config: Config, **kwargs: Any 1112 ) -> FlattenFields: 1113 return FlattenFields( 1114 flatten_lists=model.flatten_lists if model.flatten_lists is not None else True 1115 ) 1116 1117 def create_dpath_flatten_fields( 1118 self, model: DpathFlattenFieldsModel, config: Config, **kwargs: Any 1119 ) -> DpathFlattenFields: 1120 model_field_path: List[Union[InterpolatedString, str]] = [x for x in model.field_path] 1121 key_transformation = ( 1122 KeyTransformation( 1123 config=config, 1124 prefix=model.key_transformation.prefix, 1125 suffix=model.key_transformation.suffix, 1126 parameters=model.parameters or {}, 1127 ) 1128 if model.key_transformation is not None 1129 else None 1130 ) 1131 return DpathFlattenFields( 1132 config=config, 1133 field_path=model_field_path, 1134 delete_origin_value=model.delete_origin_value 1135 if model.delete_origin_value is not None 1136 else False, 1137 replace_record=model.replace_record if model.replace_record is not None else False, 1138 key_transformation=key_transformation, 1139 parameters=model.parameters or {}, 1140 ) 1141 1142 @staticmethod 1143 def _json_schema_type_name_to_type(value_type: Optional[ValueType]) -> Optional[Type[Any]]: 1144 if not value_type: 1145 return None 1146 names_to_types = { 1147 ValueType.string: str, 1148 ValueType.number: float, 1149 ValueType.integer: int, 1150 ValueType.boolean: bool, 1151 } 1152 return names_to_types[value_type] 1153 1154 def create_api_key_authenticator( 1155 self, 1156 model: ApiKeyAuthenticatorModel, 1157 config: Config, 1158 token_provider: Optional[TokenProvider] = None, 1159 **kwargs: Any, 1160 ) -> ApiKeyAuthenticator: 1161 if model.inject_into is None and model.header is None: 1162 raise ValueError( 1163 "Expected either inject_into or header to be set for ApiKeyAuthenticator" 1164 ) 1165 1166 if model.inject_into is not None and model.header is not None: 1167 raise ValueError( 1168 "inject_into and header cannot be set both for ApiKeyAuthenticator - remove the deprecated header option" 1169 ) 1170 1171 if token_provider is not None and model.api_token != "": 1172 raise ValueError( 1173 "If token_provider is set, api_token is ignored and has to be set to empty string." 1174 ) 1175 1176 request_option = ( 1177 self._create_component_from_model( 1178 model.inject_into, config, parameters=model.parameters or {} 1179 ) 1180 if model.inject_into 1181 else RequestOption( 1182 inject_into=RequestOptionType.header, 1183 field_name=model.header or "", 1184 parameters=model.parameters or {}, 1185 ) 1186 ) 1187 1188 return ApiKeyAuthenticator( 1189 token_provider=( 1190 token_provider 1191 if token_provider is not None 1192 else InterpolatedStringTokenProvider( 1193 api_token=model.api_token or "", 1194 config=config, 1195 parameters=model.parameters or {}, 1196 ) 1197 ), 1198 request_option=request_option, 1199 config=config, 1200 parameters=model.parameters or {}, 1201 ) 1202 1203 def create_legacy_to_per_partition_state_migration( 1204 self, 1205 model: LegacyToPerPartitionStateMigrationModel, 1206 config: Mapping[str, Any], 1207 declarative_stream: DeclarativeStreamModel, 1208 ) -> LegacyToPerPartitionStateMigration: 1209 retriever = declarative_stream.retriever 1210 if not isinstance(retriever, (SimpleRetrieverModel, AsyncRetrieverModel)): 1211 raise ValueError( 1212 f"LegacyToPerPartitionStateMigrations can only be applied on a DeclarativeStream with a SimpleRetriever or AsyncRetriever. Got {type(retriever)}" 1213 ) 1214 partition_router = retriever.partition_router 1215 if not isinstance( 1216 partition_router, 1217 ( 1218 SubstreamPartitionRouterModel, 1219 CustomPartitionRouterModel, 1220 UnionPartitionRouterModel, 1221 ), 1222 ): 1223 raise ValueError( 1224 f"LegacyToPerPartitionStateMigrations can only be applied on a SimpleRetriever with a SubstreamPartitionRouter, UnionPartitionRouter or CustomPartitionRouter. Got {type(partition_router)}" 1225 ) 1226 if not isinstance(partition_router, UnionPartitionRouterModel) and not hasattr( 1227 partition_router, "parent_stream_configs" 1228 ): 1229 raise ValueError( 1230 "LegacyToPerPartitionStateMigrations can only be applied with a parent stream configuration." 1231 ) 1232 1233 if not hasattr(declarative_stream, "incremental_sync"): 1234 raise ValueError( 1235 "LegacyToPerPartitionStateMigrations can only be applied with an incremental_sync configuration." 1236 ) 1237 1238 return LegacyToPerPartitionStateMigration( 1239 partition_router, # type: ignore # was already checked above 1240 declarative_stream.incremental_sync, # type: ignore # was already checked. Migration can be applied only to incremental streams. 1241 config, 1242 declarative_stream.parameters, # type: ignore # different type is expected here Mapping[str, Any], got Dict[str, Any] 1243 ) 1244 1245 def create_session_token_authenticator( 1246 self, model: SessionTokenAuthenticatorModel, config: Config, *, name: str, **kwargs: Any 1247 ) -> Union[ApiKeyAuthenticator, BearerAuthenticator]: 1248 self._reject_reduce_page_size_action( 1249 model.login_requester, f"`login_requester` of the SessionTokenAuthenticator of {name}" 1250 ) 1251 self._reject_split_request_window_action( 1252 model.login_requester, f"`login_requester` of the SessionTokenAuthenticator of {name}" 1253 ) 1254 decoder = ( 1255 self._create_component_from_model(model=model.decoder, config=config) 1256 if model.decoder 1257 else JsonDecoder(parameters={}) 1258 ) 1259 login_requester = self._create_component_from_model( 1260 model=model.login_requester, 1261 config=config, 1262 name=f"{name}_login_requester", 1263 decoder=decoder, 1264 ) 1265 token_provider = SessionTokenProvider( 1266 login_requester=login_requester, 1267 session_token_path=model.session_token_path, 1268 expiration_duration=parse_duration(model.expiration_duration) 1269 if model.expiration_duration 1270 else None, 1271 parameters=model.parameters or {}, 1272 message_repository=self._message_repository, 1273 decoder=decoder, 1274 ) 1275 if model.request_authentication.type == "Bearer": 1276 return ModelToComponentFactory.create_bearer_authenticator( 1277 BearerAuthenticatorModel(type="BearerAuthenticator", api_token=""), # type: ignore # $parameters has a default value 1278 config, 1279 token_provider=token_provider, 1280 ) 1281 else: 1282 # Get the api_token template if specified, default to just the session token 1283 api_token_template = ( 1284 getattr(model.request_authentication, "api_token", None) or "{{ session_token }}" 1285 ) 1286 final_token_provider: TokenProvider = InterpolatedSessionTokenProvider( 1287 config=config, 1288 api_token=api_token_template, 1289 session_token_provider=token_provider, 1290 parameters=model.parameters or {}, 1291 ) 1292 return self.create_api_key_authenticator( 1293 ApiKeyAuthenticatorModel( 1294 type="ApiKeyAuthenticator", 1295 api_token="", 1296 inject_into=model.request_authentication.inject_into, 1297 ), # type: ignore # $parameters and headers default to None 1298 config=config, 1299 token_provider=final_token_provider, 1300 ) 1301 1302 @staticmethod 1303 def create_basic_http_authenticator( 1304 model: BasicHttpAuthenticatorModel, config: Config, **kwargs: Any 1305 ) -> BasicHttpAuthenticator: 1306 return BasicHttpAuthenticator( 1307 password=model.password or "", 1308 username=model.username, 1309 config=config, 1310 parameters=model.parameters or {}, 1311 ) 1312 1313 @staticmethod 1314 def create_bearer_authenticator( 1315 model: BearerAuthenticatorModel, 1316 config: Config, 1317 token_provider: Optional[TokenProvider] = None, 1318 **kwargs: Any, 1319 ) -> BearerAuthenticator: 1320 if token_provider is not None and model.api_token != "": 1321 raise ValueError( 1322 "If token_provider is set, api_token is ignored and has to be set to empty string." 1323 ) 1324 return BearerAuthenticator( 1325 token_provider=( 1326 token_provider 1327 if token_provider is not None 1328 else InterpolatedStringTokenProvider( 1329 api_token=model.api_token or "", 1330 config=config, 1331 parameters=model.parameters or {}, 1332 ) 1333 ), 1334 config=config, 1335 parameters=model.parameters or {}, 1336 ) 1337 1338 @staticmethod 1339 def create_dynamic_stream_check_config( 1340 model: DynamicStreamCheckConfigModel, config: Config, **kwargs: Any 1341 ) -> DynamicStreamCheckConfig: 1342 return DynamicStreamCheckConfig( 1343 dynamic_stream_name=model.dynamic_stream_name, 1344 stream_count=model.stream_count, 1345 ) 1346 1347 def create_check_stream( 1348 self, model: CheckStreamModel, config: Config, **kwargs: Any 1349 ) -> CheckStream: 1350 if model.dynamic_streams_check_configs is None and model.stream_names is None: 1351 raise ValueError( 1352 "Expected either stream_names or dynamic_streams_check_configs to be set for CheckStream" 1353 ) 1354 1355 dynamic_streams_check_configs = ( 1356 [ 1357 self._create_component_from_model(model=dynamic_stream_check_config, config=config) 1358 for dynamic_stream_check_config in model.dynamic_streams_check_configs 1359 ] 1360 if model.dynamic_streams_check_configs 1361 else [] 1362 ) 1363 1364 # `model.config_overrides` is deliberately not read here. The source applies it around the whole 1365 # check operation (`ConcurrentDeclarativeSource._config_overridden_for_check`), which is what makes 1366 # it work for every checker type rather than only this one. Do not wire it in a second time. 1367 return CheckStream( 1368 stream_names=model.stream_names or [], 1369 dynamic_streams_check_configs=dynamic_streams_check_configs, 1370 parameters={}, 1371 ) 1372 1373 @staticmethod 1374 def create_check_dynamic_stream( 1375 model: CheckDynamicStreamModel, config: Config, **kwargs: Any 1376 ) -> CheckDynamicStream: 1377 assert model.use_check_availability is not None # for mypy 1378 1379 use_check_availability = model.use_check_availability 1380 1381 # See `create_check_stream`: `model.config_overrides` is applied by the source, not here. 1382 return CheckDynamicStream( 1383 stream_count=model.stream_count, 1384 use_check_availability=use_check_availability, 1385 parameters={}, 1386 ) 1387 1388 def create_composite_error_handler( 1389 self, model: CompositeErrorHandlerModel, config: Config, **kwargs: Any 1390 ) -> CompositeErrorHandler: 1391 error_handlers = [ 1392 self._create_component_from_model(model=error_handler_model, config=config) 1393 for error_handler_model in model.error_handlers 1394 ] 1395 return CompositeErrorHandler( 1396 error_handlers=error_handlers, parameters=model.parameters or {} 1397 ) 1398 1399 @staticmethod 1400 def create_concurrency_level( 1401 model: ConcurrencyLevelModel, config: Config, **kwargs: Any 1402 ) -> ConcurrencyLevel: 1403 return ConcurrencyLevel( 1404 default_concurrency=model.default_concurrency, 1405 max_concurrency=model.max_concurrency, 1406 config=config, 1407 parameters={}, 1408 ) 1409 1410 @staticmethod 1411 def apply_stream_state_migrations( 1412 stream_state_migrations: List[Any] | None, stream_state: MutableMapping[str, Any] 1413 ) -> MutableMapping[str, Any]: 1414 if stream_state_migrations: 1415 for state_migration in stream_state_migrations: 1416 if state_migration.should_migrate(stream_state): 1417 # The state variable is expected to be mutable but the migrate method returns an immutable mapping. 1418 stream_state = dict(state_migration.migrate(stream_state)) 1419 return stream_state 1420 1421 def create_concurrent_cursor_from_datetime_based_cursor( 1422 self, 1423 model_type: Type[BaseModel], 1424 component_definition: ComponentDefinition, 1425 stream_name: str, 1426 stream_namespace: Optional[str], 1427 stream_state: MutableMapping[str, Any], 1428 config: Config, 1429 message_repository: Optional[MessageRepository] = None, 1430 runtime_lookback_window: Optional[datetime.timedelta] = None, 1431 **kwargs: Any, 1432 ) -> ConcurrentCursor: 1433 component_type = component_definition.get("type") 1434 if component_definition.get("type") != model_type.__name__: 1435 raise ValueError( 1436 f"Expected manifest component of type {model_type.__name__}, but received {component_type} instead" 1437 ) 1438 1439 # FIXME the interfaces of the concurrent cursor are kind of annoying as they take a `ComponentDefinition` instead of the actual model. This was done because the ConcurrentDeclarativeSource didn't have access to the models [here for example](https://github.com/airbytehq/airbyte-python-cdk/blob/f525803b3fec9329e4cc8478996a92bf884bfde9/airbyte_cdk/sources/declarative/concurrent_declarative_source.py#L354C54-L354C91). So now we have two cases: 1440 # * The ComponentDefinition comes from model.__dict__ in which case we have `parameters` 1441 # * The ComponentDefinition comes from the manifest as a dict in which case we have `$parameters` 1442 # We should change those interfaces to use the model once we clean up the code in CDS at which point the parameter propagation should happen as part of the ModelToComponentFactory. 1443 if "$parameters" not in component_definition and "parameters" in component_definition: 1444 component_definition["$parameters"] = component_definition.get("parameters") # type: ignore # This is a dict 1445 datetime_based_cursor_model = model_type.parse_obj(component_definition) 1446 1447 if not isinstance(datetime_based_cursor_model, DatetimeBasedCursorModel): 1448 raise ValueError( 1449 f"Expected {model_type.__name__} component, but received {datetime_based_cursor_model.__class__.__name__}" 1450 ) 1451 1452 model_parameters = datetime_based_cursor_model.parameters or {} 1453 1454 cursor_field = self._get_catalog_defined_cursor_field( 1455 stream_name=stream_name, 1456 allow_catalog_defined_cursor_field=datetime_based_cursor_model.allow_catalog_defined_cursor_field 1457 or False, 1458 ) 1459 1460 if not cursor_field: 1461 interpolated_cursor_field = InterpolatedString.create( 1462 datetime_based_cursor_model.cursor_field, 1463 parameters=model_parameters, 1464 ) 1465 cursor_field = CursorField( 1466 cursor_field_key=interpolated_cursor_field.eval(config=config), 1467 supports_catalog_defined_cursor_field=datetime_based_cursor_model.allow_catalog_defined_cursor_field 1468 or False, 1469 ) 1470 1471 interpolated_partition_field_start = InterpolatedString.create( 1472 datetime_based_cursor_model.partition_field_start or "start_time", 1473 parameters=model_parameters, 1474 ) 1475 interpolated_partition_field_end = InterpolatedString.create( 1476 datetime_based_cursor_model.partition_field_end or "end_time", 1477 parameters=model_parameters, 1478 ) 1479 1480 slice_boundary_fields = ( 1481 interpolated_partition_field_start.eval(config=config), 1482 interpolated_partition_field_end.eval(config=config), 1483 ) 1484 1485 datetime_format = datetime_based_cursor_model.datetime_format 1486 1487 cursor_granularity = ( 1488 parse_duration(datetime_based_cursor_model.cursor_granularity) 1489 if datetime_based_cursor_model.cursor_granularity 1490 else None 1491 ) 1492 1493 lookback_window = None 1494 interpolated_lookback_window = ( 1495 InterpolatedString.create( 1496 datetime_based_cursor_model.lookback_window, 1497 parameters=model_parameters, 1498 ) 1499 if datetime_based_cursor_model.lookback_window 1500 else None 1501 ) 1502 if interpolated_lookback_window: 1503 evaluated_lookback_window = interpolated_lookback_window.eval(config=config) 1504 if evaluated_lookback_window: 1505 lookback_window = parse_duration(evaluated_lookback_window) 1506 1507 connector_state_converter: DateTimeStreamStateConverter 1508 connector_state_converter = CustomFormatConcurrentStreamStateConverter( 1509 datetime_format=datetime_format, 1510 input_datetime_formats=datetime_based_cursor_model.cursor_datetime_formats, 1511 is_sequential_state=True, # ConcurrentPerPartitionCursor only works with sequential state 1512 cursor_granularity=cursor_granularity, 1513 ) 1514 1515 # Adjusts the stream state by applying the runtime lookback window. 1516 # This is used to ensure correct state handling in case of failed partitions. 1517 stream_state_value = stream_state.get(cursor_field.cursor_field_key) 1518 if runtime_lookback_window and stream_state_value: 1519 new_stream_state = ( 1520 connector_state_converter.parse_timestamp(stream_state_value) 1521 - runtime_lookback_window 1522 ) 1523 stream_state[cursor_field.cursor_field_key] = connector_state_converter.output_format( 1524 new_stream_state 1525 ) 1526 1527 start_date_runtime_value: Union[InterpolatedString, str, MinMaxDatetime] 1528 if isinstance(datetime_based_cursor_model.start_datetime, MinMaxDatetimeModel): 1529 start_date_runtime_value = self.create_min_max_datetime( 1530 model=datetime_based_cursor_model.start_datetime, config=config 1531 ) 1532 else: 1533 start_date_runtime_value = datetime_based_cursor_model.start_datetime 1534 1535 end_date_runtime_value: Optional[Union[InterpolatedString, str, MinMaxDatetime]] 1536 if isinstance(datetime_based_cursor_model.end_datetime, MinMaxDatetimeModel): 1537 end_date_runtime_value = self.create_min_max_datetime( 1538 model=datetime_based_cursor_model.end_datetime, config=config 1539 ) 1540 else: 1541 end_date_runtime_value = datetime_based_cursor_model.end_datetime 1542 1543 interpolated_start_date = MinMaxDatetime.create( 1544 interpolated_string_or_min_max_datetime=start_date_runtime_value, 1545 parameters=datetime_based_cursor_model.parameters, 1546 ) 1547 interpolated_end_date = ( 1548 None 1549 if not end_date_runtime_value 1550 else MinMaxDatetime.create( 1551 end_date_runtime_value, datetime_based_cursor_model.parameters 1552 ) 1553 ) 1554 1555 # If datetime format is not specified then start/end datetime should inherit it from the stream slicer 1556 if not interpolated_start_date.datetime_format: 1557 interpolated_start_date.datetime_format = datetime_format 1558 if interpolated_end_date and not interpolated_end_date.datetime_format: 1559 interpolated_end_date.datetime_format = datetime_format 1560 1561 start_date = interpolated_start_date.get_datetime(config=config) 1562 end_date_provider = ( 1563 partial(interpolated_end_date.get_datetime, config) 1564 if interpolated_end_date 1565 else connector_state_converter.get_end_provider() 1566 ) 1567 1568 if ( 1569 datetime_based_cursor_model.step and not datetime_based_cursor_model.cursor_granularity 1570 ) or ( 1571 not datetime_based_cursor_model.step and datetime_based_cursor_model.cursor_granularity 1572 ): 1573 raise ValueError( 1574 f"If step is defined, cursor_granularity should be as well and vice-versa. " 1575 f"Right now, step is `{datetime_based_cursor_model.step}` and cursor_granularity is `{datetime_based_cursor_model.cursor_granularity}`" 1576 ) 1577 1578 # When step is not defined, default to a step size from the starting date to the present moment 1579 step_length = datetime.timedelta.max 1580 interpolated_step = ( 1581 InterpolatedString.create( 1582 datetime_based_cursor_model.step, 1583 parameters=model_parameters, 1584 ) 1585 if datetime_based_cursor_model.step 1586 else None 1587 ) 1588 if interpolated_step: 1589 evaluated_step = interpolated_step.eval(config) 1590 if evaluated_step: 1591 step_length = parse_duration(evaluated_step) 1592 1593 clamping_strategy: ClampingStrategy = NoClamping() 1594 if datetime_based_cursor_model.clamping: 1595 # While it is undesirable to interpolate within the model factory (as opposed to at runtime), 1596 # it is still better than shifting interpolation low-code concept into the ConcurrentCursor runtime 1597 # object which we want to keep agnostic of being low-code 1598 target = InterpolatedString( 1599 string=datetime_based_cursor_model.clamping.target, 1600 parameters=model_parameters, 1601 ) 1602 evaluated_target = target.eval(config=config) 1603 match evaluated_target: 1604 case "DAY": 1605 clamping_strategy = DayClampingStrategy() 1606 end_date_provider = ClampingEndProvider( 1607 DayClampingStrategy(is_ceiling=False), 1608 end_date_provider, # type: ignore # Having issues w/ inspection for GapType and CursorValueType as shown in existing tests. Confirmed functionality is working in practice 1609 granularity=cursor_granularity or datetime.timedelta(seconds=1), 1610 ) 1611 case "WEEK": 1612 if ( 1613 not datetime_based_cursor_model.clamping.target_details 1614 or "weekday" not in datetime_based_cursor_model.clamping.target_details 1615 ): 1616 raise ValueError( 1617 "Given WEEK clamping, weekday needs to be provided as target_details" 1618 ) 1619 weekday = self._assemble_weekday( 1620 datetime_based_cursor_model.clamping.target_details["weekday"] 1621 ) 1622 clamping_strategy = WeekClampingStrategy(weekday) 1623 end_date_provider = ClampingEndProvider( 1624 WeekClampingStrategy(weekday, is_ceiling=False), 1625 end_date_provider, # type: ignore # Having issues w/ inspection for GapType and CursorValueType as shown in existing tests. Confirmed functionality is working in practice 1626 granularity=cursor_granularity or datetime.timedelta(days=1), 1627 ) 1628 case "MONTH": 1629 clamping_strategy = MonthClampingStrategy() 1630 end_date_provider = ClampingEndProvider( 1631 MonthClampingStrategy(is_ceiling=False), 1632 end_date_provider, # type: ignore # Having issues w/ inspection for GapType and CursorValueType as shown in existing tests. Confirmed functionality is working in practice 1633 granularity=cursor_granularity or datetime.timedelta(days=1), 1634 ) 1635 case _: 1636 raise ValueError( 1637 f"Invalid clamping target {evaluated_target}, expected DAY, WEEK, MONTH" 1638 ) 1639 1640 return ConcurrentCursor( 1641 stream_name=stream_name, 1642 stream_namespace=stream_namespace, 1643 stream_state=stream_state, 1644 message_repository=message_repository or self._message_repository, 1645 connector_state_manager=self._connector_state_manager, 1646 connector_state_converter=connector_state_converter, 1647 cursor_field=cursor_field, 1648 slice_boundary_fields=slice_boundary_fields, 1649 start=start_date, # type: ignore # Having issues w/ inspection for GapType and CursorValueType as shown in existing tests. Confirmed functionality is working in practice 1650 end_provider=end_date_provider, # type: ignore # Having issues w/ inspection for GapType and CursorValueType as shown in existing tests. Confirmed functionality is working in practice 1651 lookback_window=lookback_window, 1652 slice_range=step_length, 1653 cursor_granularity=cursor_granularity, 1654 clamping_strategy=clamping_strategy, 1655 is_compare_strictly=datetime_based_cursor_model.is_compare_strictly or False, 1656 ) 1657 1658 def create_concurrent_cursor_from_incrementing_count_cursor( 1659 self, 1660 model_type: Type[BaseModel], 1661 component_definition: ComponentDefinition, 1662 stream_name: str, 1663 stream_namespace: Optional[str], 1664 stream_state: MutableMapping[str, Any], 1665 config: Config, 1666 message_repository: Optional[MessageRepository] = None, 1667 **kwargs: Any, 1668 ) -> ConcurrentCursor: 1669 component_type = component_definition.get("type") 1670 if component_definition.get("type") != model_type.__name__: 1671 raise ValueError( 1672 f"Expected manifest component of type {model_type.__name__}, but received {component_type} instead" 1673 ) 1674 1675 incrementing_count_cursor_model = model_type.parse_obj(component_definition) 1676 1677 if not isinstance(incrementing_count_cursor_model, IncrementingCountCursorModel): 1678 raise ValueError( 1679 f"Expected {model_type.__name__} component, but received {incrementing_count_cursor_model.__class__.__name__}" 1680 ) 1681 1682 start_value: Union[int, str, None] = incrementing_count_cursor_model.start_value 1683 # Pydantic Union type coercion can convert int 0 to string '0' depending on Union order. 1684 # We need to handle both int and str representations of numeric values. 1685 # Evaluate the InterpolatedString and convert to int for the ConcurrentCursor. 1686 if start_value is not None: 1687 interpolated_start_value = InterpolatedString.create( 1688 str(start_value), # Ensure we pass a string to InterpolatedString.create 1689 parameters=incrementing_count_cursor_model.parameters or {}, 1690 ) 1691 evaluated_start_value: int = int(interpolated_start_value.eval(config=config)) 1692 else: 1693 evaluated_start_value = 0 1694 1695 cursor_field = self._get_catalog_defined_cursor_field( 1696 stream_name=stream_name, 1697 allow_catalog_defined_cursor_field=incrementing_count_cursor_model.allow_catalog_defined_cursor_field 1698 or False, 1699 ) 1700 1701 if not cursor_field: 1702 interpolated_cursor_field = InterpolatedString.create( 1703 incrementing_count_cursor_model.cursor_field, 1704 parameters=incrementing_count_cursor_model.parameters or {}, 1705 ) 1706 cursor_field = CursorField( 1707 cursor_field_key=interpolated_cursor_field.eval(config=config), 1708 supports_catalog_defined_cursor_field=incrementing_count_cursor_model.allow_catalog_defined_cursor_field 1709 or False, 1710 ) 1711 1712 connector_state_converter = IncrementingCountStreamStateConverter( 1713 is_sequential_state=True, # ConcurrentPerPartitionCursor only works with sequential state 1714 ) 1715 1716 return ConcurrentCursor( 1717 stream_name=stream_name, 1718 stream_namespace=stream_namespace, 1719 stream_state=stream_state, 1720 message_repository=message_repository or self._message_repository, 1721 connector_state_manager=self._connector_state_manager, 1722 connector_state_converter=connector_state_converter, 1723 cursor_field=cursor_field, 1724 slice_boundary_fields=None, 1725 start=evaluated_start_value, # type: ignore # Having issues w/ inspection for GapType and CursorValueType as shown in existing tests. Confirmed functionality is working in practice 1726 end_provider=connector_state_converter.get_end_provider(), # type: ignore # Having issues w/ inspection for GapType and CursorValueType as shown in existing tests. Confirmed functionality is working in practice 1727 ) 1728 1729 def _assemble_weekday(self, weekday: str) -> Weekday: 1730 match weekday: 1731 case "MONDAY": 1732 return Weekday.MONDAY 1733 case "TUESDAY": 1734 return Weekday.TUESDAY 1735 case "WEDNESDAY": 1736 return Weekday.WEDNESDAY 1737 case "THURSDAY": 1738 return Weekday.THURSDAY 1739 case "FRIDAY": 1740 return Weekday.FRIDAY 1741 case "SATURDAY": 1742 return Weekday.SATURDAY 1743 case "SUNDAY": 1744 return Weekday.SUNDAY 1745 case _: 1746 raise ValueError(f"Unknown weekday {weekday}") 1747 1748 def create_concurrent_cursor_from_perpartition_cursor( 1749 self, 1750 state_manager: ConnectorStateManager, 1751 model_type: Type[BaseModel], 1752 component_definition: ComponentDefinition, 1753 stream_name: str, 1754 stream_namespace: Optional[str], 1755 config: Config, 1756 stream_state: MutableMapping[str, Any], 1757 partition_router: PartitionRouter, 1758 attempt_to_create_cursor_if_not_provided: bool = False, 1759 **kwargs: Any, 1760 ) -> ConcurrentPerPartitionCursor: 1761 component_type = component_definition.get("type") 1762 if component_definition.get("type") != model_type.__name__: 1763 raise ValueError( 1764 f"Expected manifest component of type {model_type.__name__}, but received {component_type} instead" 1765 ) 1766 1767 # FIXME the interfaces of the concurrent cursor are kind of annoying as they take a `ComponentDefinition` instead of the actual model. This was done because the ConcurrentDeclarativeSource didn't have access to the models [here for example](https://github.com/airbytehq/airbyte-python-cdk/blob/f525803b3fec9329e4cc8478996a92bf884bfde9/airbyte_cdk/sources/declarative/concurrent_declarative_source.py#L354C54-L354C91). So now we have two cases: 1768 # * The ComponentDefinition comes from model.__dict__ in which case we have `parameters` 1769 # * The ComponentDefinition comes from the manifest as a dict in which case we have `$parameters` 1770 # We should change those interfaces to use the model once we clean up the code in CDS at which point the parameter propagation should happen as part of the ModelToComponentFactory. 1771 if "$parameters" not in component_definition and "parameters" in component_definition: 1772 component_definition["$parameters"] = component_definition.get("parameters") # type: ignore # This is a dict 1773 datetime_based_cursor_model = model_type.parse_obj(component_definition) 1774 1775 if not isinstance(datetime_based_cursor_model, DatetimeBasedCursorModel): 1776 raise ValueError( 1777 f"Expected {model_type.__name__} component, but received {datetime_based_cursor_model.__class__.__name__}" 1778 ) 1779 1780 cursor_field = self._get_catalog_defined_cursor_field( 1781 stream_name=stream_name, 1782 allow_catalog_defined_cursor_field=datetime_based_cursor_model.allow_catalog_defined_cursor_field 1783 or False, 1784 ) 1785 1786 if not cursor_field: 1787 interpolated_cursor_field = InterpolatedString.create( 1788 datetime_based_cursor_model.cursor_field, 1789 # FIXME the interfaces of the concurrent cursor are kind of annoying as they take a `ComponentDefinition` instead of the actual model. This was done because the ConcurrentDeclarativeSource didn't have access to the models [here for example](https://github.com/airbytehq/airbyte-python-cdk/blob/f525803b3fec9329e4cc8478996a92bf884bfde9/airbyte_cdk/sources/declarative/concurrent_declarative_source.py#L354C54-L354C91). So now we have two cases: 1790 # * The ComponentDefinition comes from model.__dict__ in which case we have `parameters` 1791 # * The ComponentDefinition comes from the manifest as a dict in which case we have `$parameters` 1792 # We should change those interfaces to use the model once we clean up the code in CDS at which point the parameter propagation should happen as part of the ModelToComponentFactory. 1793 parameters=datetime_based_cursor_model.parameters or {}, 1794 ) 1795 cursor_field = CursorField( 1796 cursor_field_key=interpolated_cursor_field.eval(config=config), 1797 supports_catalog_defined_cursor_field=datetime_based_cursor_model.allow_catalog_defined_cursor_field 1798 or False, 1799 ) 1800 1801 datetime_format = datetime_based_cursor_model.datetime_format 1802 1803 cursor_granularity = ( 1804 parse_duration(datetime_based_cursor_model.cursor_granularity) 1805 if datetime_based_cursor_model.cursor_granularity 1806 else None 1807 ) 1808 1809 connector_state_converter: DateTimeStreamStateConverter 1810 connector_state_converter = CustomFormatConcurrentStreamStateConverter( 1811 datetime_format=datetime_format, 1812 input_datetime_formats=datetime_based_cursor_model.cursor_datetime_formats, 1813 is_sequential_state=True, # ConcurrentPerPartitionCursor only works with sequential state 1814 cursor_granularity=cursor_granularity, 1815 ) 1816 1817 # Create the cursor factory 1818 cursor_factory = ConcurrentCursorFactory( 1819 partial( 1820 self.create_concurrent_cursor_from_datetime_based_cursor, 1821 state_manager=state_manager, 1822 model_type=model_type, 1823 component_definition=component_definition, 1824 stream_name=stream_name, 1825 stream_namespace=stream_namespace, 1826 config=config, 1827 message_repository=NoopMessageRepository(), 1828 ) 1829 ) 1830 1831 # Per-partition state doesn't make sense for GroupingPartitionRouter, so force the global state 1832 use_global_cursor = isinstance( 1833 partition_router, GroupingPartitionRouter 1834 ) or component_definition.get("global_substream_cursor", False) 1835 1836 # Return the concurrent cursor and state converter 1837 return ConcurrentPerPartitionCursor( 1838 cursor_factory=cursor_factory, 1839 partition_router=partition_router, 1840 stream_name=stream_name, 1841 stream_namespace=stream_namespace, 1842 stream_state=stream_state, 1843 message_repository=self._message_repository, # type: ignore 1844 connector_state_manager=state_manager, 1845 connector_state_converter=connector_state_converter, 1846 cursor_field=cursor_field, 1847 use_global_cursor=use_global_cursor, 1848 attempt_to_create_cursor_if_not_provided=attempt_to_create_cursor_if_not_provided, 1849 ) 1850 1851 @staticmethod 1852 def create_constant_backoff_strategy( 1853 model: ConstantBackoffStrategyModel, config: Config, **kwargs: Any 1854 ) -> ConstantBackoffStrategy: 1855 ModelToComponentFactory._validate_jitter_range(model.jitter_range_in_seconds) 1856 return ConstantBackoffStrategy( 1857 backoff_time_in_seconds=model.backoff_time_in_seconds, 1858 jitter_range_in_seconds=model.jitter_range_in_seconds, 1859 config=config, 1860 parameters=model.parameters or {}, 1861 ) 1862 1863 @staticmethod 1864 def _validate_jitter_range(jitter_range_in_seconds: Optional[float]) -> None: 1865 if jitter_range_in_seconds is not None and jitter_range_in_seconds < 0: 1866 raise ValueError("jitter_range_in_seconds must be greater than or equal to 0") 1867 1868 def create_cursor_pagination( 1869 self, 1870 model: CursorPaginationModel, 1871 config: Config, 1872 decoder: Optional[Decoder] = None, 1873 **kwargs: Any, 1874 ) -> CursorPaginationStrategy: 1875 # None when nested under a custom component, which cannot pass a decoder 1876 if decoder is None: 1877 decoder = JsonDecoder(parameters={}) 1878 if isinstance(decoder, PaginationDecoderDecorator): 1879 inner_decoder = decoder.decoder 1880 else: 1881 inner_decoder = decoder 1882 decoder = PaginationDecoderDecorator(decoder=decoder) 1883 1884 if self._is_supported_decoder_for_pagination(inner_decoder): 1885 decoder_to_use = decoder 1886 else: 1887 raise ValueError( 1888 self._UNSUPPORTED_DECODER_ERROR.format(decoder_type=type(inner_decoder)) 1889 ) 1890 1891 # Pydantic v1 Union type coercion can convert int to string depending on Union order. 1892 # If page_size is a string that represents an integer (not an interpolation), convert it back. 1893 page_size = model.page_size 1894 if isinstance(page_size, str) and page_size.isdigit(): 1895 page_size = int(page_size) 1896 1897 return CursorPaginationStrategy( 1898 cursor_value=model.cursor_value, 1899 decoder=decoder_to_use, 1900 page_size=page_size, 1901 stop_condition=model.stop_condition, 1902 config=config, 1903 parameters=model.parameters or {}, 1904 ) 1905 1906 def create_custom_component(self, model: Any, config: Config, **kwargs: Any) -> Any: 1907 """ 1908 Generically creates a custom component based on the model type and a class_name reference to the custom Python class being 1909 instantiated. Only the model's additional properties that match the custom class definition are passed to the constructor 1910 :param model: The Pydantic model of the custom component being created 1911 :param config: The custom defined connector config 1912 :return: The declarative component built from the Pydantic model to be used at runtime 1913 """ 1914 # Instantiating a custom component means importing and executing arbitrary code referenced 1915 # by `class_name`. Manifests supplied by a caller, whether through the config or directly to 1916 # the manifest server, are untrusted input and could point `class_name` at any importable 1917 # callable, so they honor the same `AIRBYTE_ENABLE_UNSAFE_CODE` gate as injected 1918 # `components.py` code. Manifests bundled in a published connector image are trusted and may 1919 # always use their bundled custom components. 1920 manifest_is_untrusted = not self._custom_components_trusted or bool( 1921 config.get(INJECTED_MANIFEST) 1922 ) 1923 if manifest_is_untrusted and not custom_code_execution_permitted(): 1924 raise AirbyteCustomCodeNotPermittedError 1925 1926 custom_component_class = self._get_class_from_fully_qualified_class_name(model.class_name) 1927 component_fields = get_type_hints(custom_component_class) 1928 model_args = model.dict() 1929 model_args["config"] = config 1930 1931 # There are cases where a parent component will pass arguments to a child component via kwargs. When there are field collisions 1932 # we defer to these arguments over the component's definition 1933 for key, arg in kwargs.items(): 1934 model_args[key] = arg 1935 1936 # Pydantic is unable to parse a custom component's fields that are subcomponents into models because their fields and types are not 1937 # defined in the schema. The fields and types are defined within the Python class implementation. Pydantic can only parse down to 1938 # the custom component and this code performs a second parse to convert the sub-fields first into models, then declarative components 1939 for model_field, model_value in model_args.items(): 1940 # If a custom component field doesn't have a type set, we try to use the type hints to infer the type 1941 if ( 1942 isinstance(model_value, dict) 1943 and "type" not in model_value 1944 and model_field in component_fields 1945 ): 1946 derived_type = self._derive_component_type_from_type_hints( 1947 component_fields.get(model_field) 1948 ) 1949 if derived_type: 1950 model_value["type"] = derived_type 1951 1952 if self._is_component(model_value): 1953 model_args[model_field] = self._create_nested_component( 1954 model, 1955 model_field, 1956 model_value, 1957 config, 1958 **kwargs, 1959 ) 1960 elif isinstance(model_value, list): 1961 vals = [] 1962 for v in model_value: 1963 if isinstance(v, dict) and "type" not in v and model_field in component_fields: 1964 derived_type = self._derive_component_type_from_type_hints( 1965 component_fields.get(model_field) 1966 ) 1967 if derived_type: 1968 v["type"] = derived_type 1969 if self._is_component(v): 1970 vals.append( 1971 self._create_nested_component( 1972 model, 1973 model_field, 1974 v, 1975 config, 1976 **kwargs, 1977 ) 1978 ) 1979 else: 1980 vals.append(v) 1981 model_args[model_field] = vals 1982 1983 kwargs = { 1984 class_field: model_args[class_field] 1985 for class_field in component_fields.keys() 1986 if class_field in model_args 1987 } 1988 1989 if "api_budget" in component_fields and kwargs.get("api_budget") is None: 1990 kwargs["api_budget"] = self._api_budget 1991 1992 return custom_component_class(**kwargs) 1993 1994 @staticmethod 1995 def _get_class_from_fully_qualified_class_name( 1996 full_qualified_class_name: str, 1997 ) -> Any: 1998 """Get a class from its fully qualified name. 1999 2000 If a custom components module is needed, we assume it is already registered - probably 2001 as `source_declarative_manifest.components` or `components`. 2002 2003 Args: 2004 full_qualified_class_name (str): The fully qualified name of the class (e.g., "module.ClassName"). 2005 2006 Returns: 2007 Any: The class object. 2008 2009 Raises: 2010 ValueError: If the class cannot be loaded. 2011 """ 2012 split = full_qualified_class_name.split(".") 2013 module_name_full = ".".join(split[:-1]) 2014 class_name = split[-1] 2015 2016 try: 2017 module_ref = importlib.import_module(module_name_full) 2018 except ModuleNotFoundError as e: 2019 if split[0] == "source_declarative_manifest": 2020 # During testing, the modules containing the custom components are not moved to source_declarative_manifest. In order to run the test, add the source folder to your PYTHONPATH or add it runtime using sys.path.append 2021 try: 2022 import os 2023 2024 module_name_with_source_declarative_manifest = ".".join(split[1:-1]) 2025 module_ref = importlib.import_module( 2026 module_name_with_source_declarative_manifest 2027 ) 2028 except ModuleNotFoundError: 2029 raise ValueError(f"Could not load module `{module_name_full}`.") from e 2030 else: 2031 raise ValueError(f"Could not load module `{module_name_full}`.") from e 2032 2033 try: 2034 return getattr(module_ref, class_name) 2035 except AttributeError as e: 2036 raise ValueError( 2037 f"Could not load class `{class_name}` from module `{module_name_full}`.", 2038 ) from e 2039 2040 @staticmethod 2041 def _derive_component_type_from_type_hints(field_type: Any) -> Optional[str]: 2042 interface = field_type 2043 while True: 2044 origin = get_origin(interface) 2045 if origin: 2046 # Unnest types until we reach the raw type 2047 # List[T] -> T 2048 # Optional[List[T]] -> T 2049 args = get_args(interface) 2050 interface = args[0] 2051 else: 2052 break 2053 if isinstance(interface, type) and not ModelToComponentFactory.is_builtin_type(interface): 2054 return interface.__name__ 2055 return None 2056 2057 @staticmethod 2058 def is_builtin_type(cls: Optional[Type[Any]]) -> bool: 2059 if not cls: 2060 return False 2061 return cls.__module__ == "builtins" 2062 2063 @staticmethod 2064 def _extract_missing_parameters(error: TypeError) -> List[str]: 2065 parameter_search = re.search(r"keyword-only.*:\s(.*)", str(error)) 2066 if parameter_search: 2067 return re.findall(r"\'(.+?)\'", parameter_search.group(1)) 2068 else: 2069 return [] 2070 2071 def _create_nested_component( 2072 self, model: Any, model_field: str, model_value: Any, config: Config, **kwargs: Any 2073 ) -> Any: 2074 type_name = model_value.get("type", None) 2075 if not type_name: 2076 # If no type is specified, we can assume this is a dictionary object which can be returned instead of a subcomponent 2077 return model_value 2078 2079 model_type = self.TYPE_NAME_TO_MODEL.get(type_name, None) 2080 if model_type: 2081 parsed_model = model_type.parse_obj(model_value) 2082 try: 2083 # To improve usability of the language, certain fields are shared between components. This can come in the form of 2084 # a parent component passing some of its fields to a child component or the parent extracting fields from other child 2085 # components and passing it to others. One example is the DefaultPaginator referencing the HttpRequester url_base 2086 # while constructing a SimpleRetriever. However, custom components don't support this behavior because they are created 2087 # generically in create_custom_component(). This block allows developers to specify extra arguments in $parameters that 2088 # are needed by a component and could not be shared. 2089 model_constructor = self.PYDANTIC_MODEL_TO_CONSTRUCTOR.get(parsed_model.__class__) 2090 constructor_kwargs = inspect.getfullargspec(model_constructor).kwonlyargs 2091 model_parameters = model_value.get("$parameters", {}) 2092 matching_parameters = { 2093 kwarg: model_parameters[kwarg] 2094 for kwarg in constructor_kwargs 2095 if kwarg in model_parameters 2096 } 2097 matching_kwargs = { 2098 kwarg: kwargs[kwarg] for kwarg in constructor_kwargs if kwarg in kwargs 2099 } 2100 return self._create_component_from_model( 2101 model=parsed_model, config=config, **(matching_parameters | matching_kwargs) 2102 ) 2103 except TypeError as error: 2104 missing_parameters = self._extract_missing_parameters(error) 2105 if missing_parameters: 2106 raise ValueError( 2107 f"Error creating component '{type_name}' with parent custom component {model.class_name}: Please provide " 2108 + ", ".join( 2109 ( 2110 f"{type_name}.$parameters.{parameter}" 2111 for parameter in missing_parameters 2112 ) 2113 ) 2114 ) 2115 raise TypeError( 2116 f"Error creating component '{type_name}' with parent custom component {model.class_name}: {error}" 2117 ) 2118 else: 2119 raise ValueError( 2120 f"Error creating custom component {model.class_name}. Subcomponent creation has not been implemented for '{type_name}'" 2121 ) 2122 2123 @staticmethod 2124 def _is_component(model_value: Any) -> bool: 2125 return isinstance(model_value, dict) and model_value.get("type") is not None 2126 2127 def create_default_stream( 2128 self, model: DeclarativeStreamModel, config: Config, is_parent: bool = False, **kwargs: Any 2129 ) -> AbstractStream: 2130 primary_key = model.primary_key.__root__ if model.primary_key else None 2131 self._migrate_state(model, config) 2132 self._warn_on_ineffective_incremental_dependency(model) 2133 2134 partition_router = self._build_stream_slicer_from_partition_router( 2135 model.retriever, 2136 config, 2137 stream_name=model.name, 2138 **kwargs, 2139 ) 2140 concurrent_cursor = self._build_concurrent_cursor(model, partition_router, config) 2141 if model.incremental_sync and isinstance(model.incremental_sync, DatetimeBasedCursorModel): 2142 cursor_model: DatetimeBasedCursorModel = model.incremental_sync 2143 2144 end_time_option = ( 2145 self._create_component_from_model( 2146 cursor_model.end_time_option, config, parameters=cursor_model.parameters or {} 2147 ) 2148 if cursor_model.end_time_option 2149 else None 2150 ) 2151 start_time_option = ( 2152 self._create_component_from_model( 2153 cursor_model.start_time_option, config, parameters=cursor_model.parameters or {} 2154 ) 2155 if cursor_model.start_time_option 2156 else None 2157 ) 2158 2159 datetime_request_options_provider = DatetimeBasedRequestOptionsProvider( 2160 start_time_option=start_time_option, 2161 end_time_option=end_time_option, 2162 partition_field_start=cursor_model.partition_field_start, 2163 partition_field_end=cursor_model.partition_field_end, 2164 config=config, 2165 parameters=model.parameters or {}, 2166 ) 2167 request_options_provider = ( 2168 datetime_request_options_provider 2169 if not isinstance(concurrent_cursor, ConcurrentPerPartitionCursor) 2170 else PerPartitionRequestOptionsProvider( 2171 partition_router, datetime_request_options_provider 2172 ) 2173 ) 2174 elif model.incremental_sync and isinstance( 2175 model.incremental_sync, IncrementingCountCursorModel 2176 ): 2177 if isinstance(concurrent_cursor, ConcurrentPerPartitionCursor): 2178 raise ValueError( 2179 "PerPartition does not support per partition states because switching to global state is time based" 2180 ) 2181 2182 cursor_model: IncrementingCountCursorModel = model.incremental_sync # type: ignore 2183 2184 start_time_option = ( 2185 self._create_component_from_model( 2186 cursor_model.start_value_option, # type: ignore # mypy still thinks cursor_model of type DatetimeBasedCursor 2187 config, 2188 parameters=cursor_model.parameters or {}, 2189 ) 2190 if cursor_model.start_value_option # type: ignore # mypy still thinks cursor_model of type DatetimeBasedCursor 2191 else None 2192 ) 2193 2194 # The concurrent engine defaults the start/end fields on the slice to "start" and "end", but 2195 # the default DatetimeBasedRequestOptionsProvider() sets them to start_time/end_time 2196 partition_field_start = "start" 2197 2198 request_options_provider = DatetimeBasedRequestOptionsProvider( 2199 start_time_option=start_time_option, 2200 partition_field_start=partition_field_start, 2201 config=config, 2202 parameters=model.parameters or {}, 2203 ) 2204 else: 2205 request_options_provider = None 2206 2207 transformations = [] 2208 if model.transformations: 2209 for transformation_model in model.transformations: 2210 transformations.append( 2211 self._create_component_from_model(model=transformation_model, config=config) 2212 ) 2213 file_uploader = None 2214 if model.file_uploader: 2215 file_uploader = self._create_component_from_model( 2216 model=model.file_uploader, config=config 2217 ) 2218 2219 stream_slicer: ConcurrentStreamSlicer = ( 2220 partition_router 2221 if isinstance(concurrent_cursor, FinalStateCursor) 2222 else concurrent_cursor 2223 ) 2224 2225 retriever = self._create_component_from_model( 2226 model=model.retriever, 2227 config=config, 2228 name=model.name, 2229 primary_key=primary_key, 2230 request_options_provider=request_options_provider, 2231 stream_slicer=stream_slicer, 2232 partition_router=partition_router, 2233 has_stop_condition_cursor=self._is_stop_condition_on_cursor(model), 2234 is_client_side_incremental_sync=self._is_client_side_filtering_enabled(model), 2235 cursor=concurrent_cursor, 2236 transformations=transformations, 2237 file_uploader=file_uploader, 2238 incremental_sync=model.incremental_sync, 2239 ) 2240 if isinstance(retriever, AsyncRetriever): 2241 stream_slicer = retriever.stream_slicer 2242 2243 schema_loader: SchemaLoader 2244 if model.schema_loader and isinstance(model.schema_loader, list): 2245 nested_schema_loaders = [ 2246 self._create_component_from_model(model=nested_schema_loader, config=config) 2247 for nested_schema_loader in model.schema_loader 2248 ] 2249 schema_loader = CompositeSchemaLoader( 2250 schema_loaders=nested_schema_loaders, parameters={} 2251 ) 2252 elif model.schema_loader: 2253 schema_loader = self._create_component_from_model( 2254 model=model.schema_loader, # type: ignore # If defined, schema_loader is guaranteed not to be a list and will be one of the existing base models 2255 config=config, 2256 ) 2257 else: 2258 options = model.parameters or {} 2259 if "name" not in options: 2260 options["name"] = model.name 2261 schema_loader = DefaultSchemaLoader(config=config, parameters=options) 2262 schema_loader = CachingSchemaLoaderDecorator(schema_loader) 2263 2264 stream_name = model.name or "" 2265 return DefaultStream( 2266 partition_generator=StreamSlicerPartitionGenerator( 2267 DeclarativePartitionFactory( 2268 stream_name, 2269 schema_loader, 2270 retriever, 2271 self._message_repository, 2272 ), 2273 stream_slicer, 2274 slice_limit=self._limit_slices_fetched, 2275 ), 2276 name=stream_name, 2277 json_schema=schema_loader.get_json_schema, 2278 primary_key=get_primary_key_from_stream(primary_key), 2279 cursor_field=( 2280 concurrent_cursor.cursor_field 2281 if hasattr(concurrent_cursor, "cursor_field") 2282 else None 2283 ), 2284 logger=logging.getLogger(f"airbyte.{stream_name}"), 2285 cursor=concurrent_cursor, 2286 supports_file_transfer=hasattr(model, "file_uploader") and bool(model.file_uploader), 2287 ) 2288 2289 def _warn_on_ineffective_incremental_dependency(self, model: DeclarativeStreamModel) -> None: 2290 """ 2291 `incremental_dependency: true` only takes effect when the substream defines its own 2292 `incremental_sync`: the parent cursor is persisted under the `parent_state` key of the 2293 substream's state, which is only emitted by incremental substreams. On a stream without 2294 `incremental_sync`, the setting is silently ignored and all parent records are re-read on 2295 every sync, so we warn about the misconfiguration instead. 2296 """ 2297 if model.incremental_sync: 2298 return 2299 2300 partition_router = getattr(model.retriever, "partition_router", None) 2301 if not partition_router: 2302 return 2303 2304 routers = partition_router if isinstance(partition_router, list) else [partition_router] 2305 for router in routers: 2306 if isinstance(router, GroupingPartitionRouterModel): 2307 router = router.underlying_partition_router 2308 if isinstance(router, SubstreamPartitionRouterModel) and any( 2309 parent_stream_config.incremental_dependency 2310 for parent_stream_config in router.parent_stream_configs 2311 ): 2312 LOGGER.warning( 2313 f"Stream `{model.name}` has `incremental_dependency: true` in its parent stream configuration but does not define `incremental_sync`. " 2314 "The parent stream's cursor is only persisted in the state of an incremental substream, so this setting has no effect and all parent records will be re-read on every sync. " 2315 "Define `incremental_sync` on this stream or remove `incremental_dependency`." 2316 ) 2317 return 2318 2319 def _migrate_state(self, model: DeclarativeStreamModel, config: Config) -> None: 2320 stream_name = model.name or "" 2321 stream_state = self._connector_state_manager.get_stream_state( 2322 stream_name=stream_name, namespace=None 2323 ) 2324 if model.state_migrations: 2325 state_transformations = [ 2326 self._create_component_from_model(state_migration, config, declarative_stream=model) 2327 for state_migration in model.state_migrations 2328 ] 2329 else: 2330 state_transformations = [] 2331 stream_state = self.apply_stream_state_migrations(state_transformations, stream_state) 2332 self._connector_state_manager.update_state_for_stream( 2333 stream_name=stream_name, namespace=None, value=stream_state 2334 ) 2335 2336 def _is_stop_condition_on_cursor(self, model: DeclarativeStreamModel) -> bool: 2337 return bool( 2338 model.incremental_sync 2339 and hasattr(model.incremental_sync, "is_data_feed") 2340 and model.incremental_sync.is_data_feed 2341 ) 2342 2343 def _is_client_side_filtering_enabled(self, model: DeclarativeStreamModel) -> bool: 2344 return bool( 2345 model.incremental_sync 2346 and hasattr(model.incremental_sync, "is_client_side_incremental") 2347 and model.incremental_sync.is_client_side_incremental 2348 ) 2349 2350 def _build_stream_slicer_from_partition_router( 2351 self, 2352 model: Union[ 2353 AsyncRetrieverModel, 2354 CustomRetrieverModel, 2355 SimpleRetrieverModel, 2356 ], 2357 config: Config, 2358 stream_name: Optional[str] = None, 2359 **kwargs: Any, 2360 ) -> PartitionRouter: 2361 if ( 2362 hasattr(model, "partition_router") 2363 and isinstance(model, (SimpleRetrieverModel, AsyncRetrieverModel, CustomRetrieverModel)) 2364 and model.partition_router 2365 ): 2366 stream_slicer_model = model.partition_router 2367 if isinstance(stream_slicer_model, list): 2368 return CartesianProductStreamSlicer( 2369 [ 2370 self._create_component_from_model( 2371 model=slicer, config=config, stream_name=stream_name or "" 2372 ) 2373 for slicer in stream_slicer_model 2374 ], 2375 parameters={}, 2376 ) 2377 elif isinstance(stream_slicer_model, dict): 2378 # partition router comes from CustomRetrieverModel therefore has not been parsed as a model 2379 params = stream_slicer_model.get("$parameters") 2380 if not isinstance(params, dict): 2381 params = {} 2382 stream_slicer_model["$parameters"] = params 2383 2384 if stream_name is not None: 2385 params["stream_name"] = stream_name 2386 2387 return self._create_nested_component( # type: ignore[no-any-return] # There is no guarantee that this will return a stream slicer. If not, we expect an AttributeError during the call to `stream_slices` 2388 model, 2389 "partition_router", 2390 stream_slicer_model, 2391 config, 2392 **kwargs, 2393 ) 2394 else: 2395 return self._create_component_from_model( # type: ignore[no-any-return] # Will be created PartitionRouter as stream_slicer_model is model.partition_router 2396 model=stream_slicer_model, config=config, stream_name=stream_name or "" 2397 ) 2398 return SinglePartitionRouter(parameters={}) 2399 2400 def _build_concurrent_cursor( 2401 self, 2402 model: DeclarativeStreamModel, 2403 stream_slicer: Optional[PartitionRouter], 2404 config: Config, 2405 ) -> Cursor: 2406 stream_name = model.name or "" 2407 stream_state = self._connector_state_manager.get_stream_state(stream_name, None) 2408 2409 if ( 2410 model.incremental_sync 2411 and stream_slicer 2412 and not isinstance(stream_slicer, SinglePartitionRouter) 2413 ): 2414 if isinstance(model.incremental_sync, IncrementingCountCursorModel): 2415 # We don't currently support usage of partition routing and IncrementingCountCursor at the 2416 # same time because we didn't solve for design questions like what the lookback window would 2417 # be as well as global cursor fall backs. We have not seen customers that have needed both 2418 # at the same time yet and are currently punting on this until we need to solve it. 2419 raise ValueError( 2420 f"The low-code framework does not currently support usage of a PartitionRouter and an IncrementingCountCursor at the same time. Please specify only one of these options for stream {stream_name}." 2421 ) 2422 return self.create_concurrent_cursor_from_perpartition_cursor( # type: ignore # This is a known issue that we are creating and returning a ConcurrentCursor which does not technically implement the (low-code) StreamSlicer. However, (low-code) StreamSlicer and ConcurrentCursor both implement StreamSlicer.stream_slices() which is the primary method needed for checkpointing 2423 state_manager=self._connector_state_manager, 2424 model_type=DatetimeBasedCursorModel, 2425 component_definition=model.incremental_sync.__dict__, 2426 stream_name=stream_name, 2427 stream_state=stream_state, 2428 stream_namespace=None, 2429 config=config or {}, 2430 partition_router=stream_slicer, 2431 attempt_to_create_cursor_if_not_provided=True, # FIXME can we remove that now? 2432 ) 2433 elif model.incremental_sync: 2434 if type(model.incremental_sync) == IncrementingCountCursorModel: 2435 return self.create_concurrent_cursor_from_incrementing_count_cursor( # type: ignore # This is a known issue that we are creating and returning a ConcurrentCursor which does not technically implement the (low-code) StreamSlicer. However, (low-code) StreamSlicer and ConcurrentCursor both implement StreamSlicer.stream_slices() which is the primary method needed for checkpointing 2436 model_type=IncrementingCountCursorModel, 2437 component_definition=model.incremental_sync.__dict__, 2438 stream_name=stream_name, 2439 stream_namespace=None, 2440 stream_state=stream_state, 2441 config=config or {}, 2442 ) 2443 elif type(model.incremental_sync) == DatetimeBasedCursorModel: 2444 return self.create_concurrent_cursor_from_datetime_based_cursor( # type: ignore # This is a known issue that we are creating and returning a ConcurrentCursor which does not technically implement the (low-code) StreamSlicer. However, (low-code) StreamSlicer and ConcurrentCursor both implement StreamSlicer.stream_slices() which is the primary method needed for checkpointing 2445 model_type=type(model.incremental_sync), 2446 component_definition=model.incremental_sync.__dict__, 2447 stream_name=stream_name, 2448 stream_namespace=None, 2449 stream_state=stream_state, 2450 config=config or {}, 2451 attempt_to_create_cursor_if_not_provided=True, 2452 ) 2453 else: 2454 raise ValueError( 2455 f"Incremental sync of type {type(model.incremental_sync)} is not supported" 2456 ) 2457 return FinalStateCursor(stream_name, None, self._message_repository) 2458 2459 def create_default_error_handler( 2460 self, model: DefaultErrorHandlerModel, config: Config, **kwargs: Any 2461 ) -> DefaultErrorHandler: 2462 backoff_strategies = [] 2463 if model.backoff_strategies: 2464 for backoff_strategy_model in model.backoff_strategies: 2465 backoff_strategies.append( 2466 self._create_component_from_model(model=backoff_strategy_model, config=config) 2467 ) 2468 2469 response_filters = [] 2470 if model.response_filters: 2471 for response_filter_model in model.response_filters: 2472 response_filters.append( 2473 self._create_component_from_model(model=response_filter_model, config=config) 2474 ) 2475 response_filters.append( 2476 HttpResponseFilter(config=config, parameters=model.parameters or {}) 2477 ) 2478 2479 return DefaultErrorHandler( 2480 backoff_strategies=backoff_strategies, 2481 max_retries=model.max_retries, 2482 response_filters=response_filters, 2483 config=config, 2484 parameters=model.parameters or {}, 2485 ) 2486 2487 def create_default_paginator( 2488 self, 2489 model: DefaultPaginatorModel, 2490 config: Config, 2491 *, 2492 url_base: str, 2493 extractor_model: Optional[ 2494 Union[CustomRecordExtractorModel, DpathExtractorModel, CombinedExtractorModel] 2495 ] = None, 2496 decoder: Optional[Decoder] = None, 2497 cursor_used_for_stop_condition: Optional[Cursor] = None, 2498 ) -> Union[DefaultPaginator, PaginatorTestReadDecorator]: 2499 if decoder: 2500 if self._is_supported_decoder_for_pagination(decoder): 2501 decoder_to_use = PaginationDecoderDecorator(decoder=decoder) 2502 else: 2503 raise ValueError(self._UNSUPPORTED_DECODER_ERROR.format(decoder_type=type(decoder))) 2504 else: 2505 decoder_to_use = PaginationDecoderDecorator(decoder=JsonDecoder(parameters={})) 2506 page_size_option = ( 2507 self._create_component_from_model(model=model.page_size_option, config=config) 2508 if model.page_size_option 2509 else None 2510 ) 2511 page_token_option = ( 2512 self._create_component_from_model(model=model.page_token_option, config=config) 2513 if model.page_token_option 2514 else None 2515 ) 2516 pagination_strategy = self._create_component_from_model( 2517 model=model.pagination_strategy, 2518 config=config, 2519 decoder=decoder_to_use, 2520 extractor_model=extractor_model, 2521 ) 2522 if cursor_used_for_stop_condition: 2523 pagination_strategy = StopConditionPaginationStrategyDecorator( 2524 pagination_strategy, CursorStopCondition(cursor_used_for_stop_condition) 2525 ) 2526 paginator = DefaultPaginator( 2527 decoder=decoder_to_use, 2528 page_size_option=page_size_option, 2529 page_token_option=page_token_option, 2530 pagination_strategy=pagination_strategy, 2531 url_base=url_base, 2532 config=config, 2533 parameters=model.parameters or {}, 2534 ) 2535 if self._limit_pages_fetched_per_slice: 2536 return PaginatorTestReadDecorator(paginator, self._limit_pages_fetched_per_slice) 2537 return paginator 2538 2539 @staticmethod 2540 def _is_decoder_downgraded_by_connector_builder(decoder: Decoder) -> bool: 2541 """Is this the buffered stand-in the Connector Builder builds for a streaming decoder? 2542 2543 `create_csv_decoder`, `create_jsonl_decoder`, `create_json_items_decoder` and 2544 `create_gzip_decoder` build a `CompositeRawDecoder` with `stream_response=False` when 2545 `_emit_connector_builder_messages` is set, so a Builder test read can be replayed. Those 2546 four instances are the only ones that stream in production while reporting 2547 `is_stream_response() == False` in the Builder. 2548 2549 The match is on the exact class and on the parser, not on `isinstance`: a `CustomDecoder` 2550 subclassing `CompositeRawDecoder` with `stream_response=False`, or a buffered 2551 `CompositeRawDecoder` built around a `JsonParser`, reads the body from `response.content` 2552 in production too and must not be rejected. 2553 """ 2554 return type(decoder) is CompositeRawDecoder and isinstance( 2555 decoder.parser, (CsvParser, JsonLineParser, JsonItemsParser, GzipParser) 2556 ) 2557 2558 def _reject_combined_extractor_over_streaming_decoder(self, decoder: Optional[Decoder]) -> None: 2559 """Refuse to build a `CombinedExtractor` whose response body can only be read once. 2560 2561 Every sub-extractor is handed the same `requests.Response`. A streaming decoder consumes 2562 and closes `response.raw`, so every sub-extractor after the first reads a closed 2563 `urllib3.HTTPResponse`, which returns an empty body instead of raising: `union` emits only 2564 the first sub-extractor's records and `first_match` emits nothing when the first path 2565 misses. Rejecting the manifest is the only way to make that loud. 2566 2567 The Connector Builder downgrades those same decoders to `stream_response=False` so a test 2568 read can be replayed, which would let a manifest look correct in the Builder and lose 2569 records once published. The downgraded decoders are therefore rejected in the Builder too. 2570 """ 2571 if decoder is None: 2572 return 2573 inner_decoder = ( 2574 decoder.decoder if isinstance(decoder, PaginationDecoderDecorator) else decoder 2575 ) 2576 streams_in_production = inner_decoder.is_stream_response() or ( 2577 self._emit_connector_builder_messages 2578 and self._is_decoder_downgraded_by_connector_builder(inner_decoder) 2579 ) 2580 if not streams_in_production: 2581 return 2582 raise AirbyteTracedException( 2583 message="CombinedExtractor is not supported with a streaming decoder.", 2584 internal_message=( 2585 f"CombinedExtractor was configured with {type(inner_decoder).__name__}, which " 2586 f"streams the response and can only be read once. Sub-extractors after the first " 2587 f"would read a closed response and silently return no records. The streaming " 2588 f"decoders are CsvDecoder, JsonlDecoder, JsonItemsDecoder, GzipDecoder and " 2589 f"IterableDecoder; use JsonDecoder, XmlDecoder or ZipfileDecoder, or declare a " 2590 f"single extractor." 2591 ), 2592 failure_type=FailureType.config_error, 2593 ) 2594 2595 def create_combined_extractor( 2596 self, 2597 model: CombinedExtractorModel, 2598 config: Config, 2599 decoder: Optional[Decoder] = None, 2600 **kwargs: Any, 2601 ) -> CombinedExtractor: 2602 self._reject_combined_extractor_over_streaming_decoder(decoder) 2603 extractors = [ 2604 self._create_component_from_model(model=sub_extractor, config=config, decoder=decoder) 2605 for sub_extractor in model.extractors 2606 ] 2607 return CombinedExtractor( 2608 extractors=extractors, 2609 mode=CombineMode(model.mode.value) if model.mode else CombineMode.union, 2610 skip_empty_records=bool(model.skip_empty_records), 2611 parameters=model.parameters or {}, 2612 ) 2613 2614 def create_dpath_extractor( 2615 self, 2616 model: DpathExtractorModel, 2617 config: Config, 2618 decoder: Optional[Decoder] = None, 2619 **kwargs: Any, 2620 ) -> DpathExtractor: 2621 if decoder: 2622 decoder_to_use = decoder 2623 else: 2624 decoder_to_use = JsonDecoder(parameters={}) 2625 model_field_path: List[Union[InterpolatedString, str]] = [x for x in model.field_path] 2626 2627 record_expander = None 2628 if model.record_expander: 2629 record_expander = self._create_component_from_model( 2630 model=model.record_expander, 2631 config=config, 2632 ) 2633 2634 return DpathExtractor( 2635 decoder=decoder_to_use, 2636 field_path=model_field_path, 2637 config=config, 2638 parameters=model.parameters or {}, 2639 record_expander=record_expander, 2640 ) 2641 2642 def create_record_expander( 2643 self, 2644 model: RecordExpanderModel, 2645 config: Config, 2646 **kwargs: Any, 2647 ) -> RecordExpander: 2648 truncated_list_retriever = None 2649 suppress_incomplete_fetch_warning = False 2650 if model.truncated_list_retriever: 2651 retriever_model = model.truncated_list_retriever 2652 name = "record_expander_truncated_list" 2653 # `CustomRetriever` allows extra fields, so read from the dumped model to cover both types. 2654 retriever_fields = retriever_model.dict() 2655 for unsupported_option in ("partition_router", "pagination_reset"): 2656 if retriever_fields.get(unsupported_option): 2657 raise ValueError( 2658 f"`{unsupported_option}` is not supported on `truncated_list_retriever`." 2659 ) 2660 log_formatter = lambda response: format_http_message( 2661 response, 2662 f"Record expander '{name}' request", 2663 "Request performed in order to fetch the complete nested list of a truncated record.", 2664 name, 2665 is_auxiliary=True, 2666 ) 2667 # `name`/`primary_key`/`transformations` are also what a `CustomRetriever` forwards to its 2668 # nested `requester`/`record_selector`, so they are passed for both retriever types. 2669 truncated_list_retriever = self._create_component_from_model( 2670 model=retriever_model, 2671 config=config, 2672 name=name, 2673 primary_key=None, 2674 transformations=[], 2675 log_formatter=log_formatter, 2676 ) 2677 # Only a capped paginator makes a shortfall expected; `NoPagination` and retrievers 2678 # without a paginator are never capped. 2679 suppress_incomplete_fetch_warning = isinstance( 2680 truncated_list_retriever, SimpleRetriever 2681 ) and isinstance(truncated_list_retriever.paginator, PaginatorTestReadDecorator) 2682 return RecordExpander( 2683 expand_records_from_field=model.expand_records_from_field, 2684 config=config, 2685 parameters=model.parameters or {}, 2686 remain_original_record=model.remain_original_record or False, 2687 parent_fields=[ 2688 self._create_component_from_model(model=parent_field, config=config) 2689 for parent_field in model.parent_fields 2690 ] 2691 if model.parent_fields 2692 else None, 2693 merge_parent=model.merge_parent or False, 2694 on_no_records=OnNoRecords(model.on_no_records.value) 2695 if model.on_no_records 2696 else OnNoRecords.skip, 2697 truncation_indicator_path=model.truncation_indicator_path, 2698 truncated_list_retriever=truncated_list_retriever, 2699 message_repository=self._message_repository, 2700 suppress_incomplete_fetch_warning=suppress_incomplete_fetch_warning, 2701 ) 2702 2703 @staticmethod 2704 def create_parent_field_path( 2705 model: ParentFieldPathModel, 2706 config: Config, 2707 **kwargs: Any, 2708 ) -> ParentFieldPath: 2709 return ParentFieldPath( 2710 parent_path=model.parent_path, 2711 record_path=model.record_path, 2712 config=config, 2713 parameters=model.parameters or {}, 2714 ) 2715 2716 @staticmethod 2717 def create_response_to_file_extractor( 2718 model: ResponseToFileExtractorModel, 2719 **kwargs: Any, 2720 ) -> ResponseToFileExtractor: 2721 return ResponseToFileExtractor( 2722 parameters=model.parameters or {}, 2723 preserve_na_values=model.preserve_na_values or False, 2724 ) 2725 2726 @staticmethod 2727 def create_exponential_backoff_strategy( 2728 model: ExponentialBackoffStrategyModel, config: Config 2729 ) -> ExponentialBackoffStrategy: 2730 ModelToComponentFactory._validate_jitter_range(model.jitter_range_in_seconds) 2731 return ExponentialBackoffStrategy( 2732 factor=model.factor or 5, 2733 jitter_range_in_seconds=model.jitter_range_in_seconds, 2734 parameters=model.parameters or {}, 2735 config=config, 2736 ) 2737 2738 @staticmethod 2739 def create_group_by_key(model: GroupByKeyMergeStrategyModel, config: Config) -> GroupByKey: 2740 return GroupByKey(model.key, config=config, parameters=model.parameters or {}) 2741 2742 def create_http_requester( 2743 self, 2744 model: HttpRequesterModel, 2745 config: Config, 2746 decoder: Decoder = JsonDecoder(parameters={}), 2747 query_properties_key: Optional[str] = None, 2748 use_cache: Optional[bool] = None, 2749 *, 2750 name: str, 2751 ) -> HttpRequester: 2752 authenticator = ( 2753 self._create_component_from_model( 2754 model=model.authenticator, 2755 config=config, 2756 url_base=model.url or model.url_base, 2757 name=name, 2758 decoder=decoder, 2759 ) 2760 if model.authenticator 2761 else None 2762 ) 2763 error_handler = ( 2764 self._create_component_from_model(model=model.error_handler, config=config) 2765 if model.error_handler 2766 else DefaultErrorHandler( 2767 backoff_strategies=[], 2768 response_filters=[], 2769 config=config, 2770 parameters=model.parameters or {}, 2771 ) 2772 ) 2773 2774 api_budget = self._api_budget 2775 2776 request_options_provider = InterpolatedRequestOptionsProvider( 2777 request_body=model.request_body, 2778 request_body_data=model.request_body_data, 2779 request_body_json=model.request_body_json, 2780 request_headers=model.request_headers, 2781 request_parameters=model.request_parameters, # type: ignore # QueryProperties have been removed in `create_simple_retriever` 2782 query_properties_key=query_properties_key, 2783 config=config, 2784 parameters=model.parameters or {}, 2785 ) 2786 2787 assert model.use_cache is not None # for mypy 2788 assert model.http_method is not None # for mypy 2789 2790 should_use_cache = (model.use_cache or bool(use_cache)) and not self._disable_cache 2791 2792 return HttpRequester( 2793 name=name, 2794 url=model.url, 2795 url_base=model.url_base, 2796 path=model.path, 2797 authenticator=authenticator, 2798 error_handler=error_handler, 2799 api_budget=api_budget, 2800 http_method=HttpMethod[model.http_method.value], 2801 request_options_provider=request_options_provider, 2802 config=config, 2803 disable_retries=self._disable_retries, 2804 parameters=model.parameters or {}, 2805 message_repository=self._message_repository, 2806 use_cache=should_use_cache, 2807 connect_timeout_in_seconds=model.connect_timeout_in_seconds, 2808 read_timeout_in_seconds=model.read_timeout_in_seconds, 2809 use_tcp_keepalive=model.use_tcp_keepalive or False, 2810 decoder=decoder, 2811 stream_response=decoder.is_stream_response() if decoder else False, 2812 ) 2813 2814 @staticmethod 2815 def create_http_response_filter( 2816 model: HttpResponseFilterModel, config: Config, **kwargs: Any 2817 ) -> HttpResponseFilter: 2818 if model.action: 2819 action = ResponseAction(model.action.value) 2820 else: 2821 action = None 2822 2823 failure_type = FailureType(model.failure_type.value) if model.failure_type else None 2824 2825 http_codes = ( 2826 set(model.http_codes) if model.http_codes else set() 2827 ) # JSON schema notation has no set data type. The schema enforces an array of unique elements 2828 2829 return HttpResponseFilter( 2830 action=action, 2831 failure_type=failure_type, 2832 error_message=model.error_message or "", 2833 error_message_contains=model.error_message_contains or "", 2834 http_codes=http_codes, 2835 predicate=model.predicate or "", 2836 config=config, 2837 parameters=model.parameters or {}, 2838 ) 2839 2840 @staticmethod 2841 def create_inline_schema_loader( 2842 model: InlineSchemaLoaderModel, config: Config, **kwargs: Any 2843 ) -> InlineSchemaLoader: 2844 return InlineSchemaLoader(schema=model.schema_ or {}, parameters={}) 2845 2846 def create_complex_field_type( 2847 self, model: ComplexFieldTypeModel, config: Config, **kwargs: Any 2848 ) -> ComplexFieldType: 2849 items = ( 2850 self._create_component_from_model(model=model.items, config=config) 2851 if isinstance(model.items, ComplexFieldTypeModel) 2852 else model.items 2853 ) 2854 2855 return ComplexFieldType(field_type=model.field_type, items=items) 2856 2857 def create_types_map(self, model: TypesMapModel, config: Config, **kwargs: Any) -> TypesMap: 2858 target_type = ( 2859 self._create_component_from_model(model=model.target_type, config=config) 2860 if isinstance(model.target_type, ComplexFieldTypeModel) 2861 else model.target_type 2862 ) 2863 2864 return TypesMap( 2865 target_type=target_type, 2866 current_type=model.current_type, 2867 condition=model.condition if model.condition is not None else "True", 2868 ) 2869 2870 def create_schema_type_identifier( 2871 self, model: SchemaTypeIdentifierModel, config: Config, **kwargs: Any 2872 ) -> SchemaTypeIdentifier: 2873 types_mapping = [] 2874 if model.types_mapping: 2875 types_mapping.extend( 2876 [ 2877 self._create_component_from_model(types_map, config=config) 2878 for types_map in model.types_mapping 2879 ] 2880 ) 2881 model_schema_pointer: List[Union[InterpolatedString, str]] = ( 2882 [x for x in model.schema_pointer] if model.schema_pointer else [] 2883 ) 2884 model_key_pointer: List[Union[InterpolatedString, str]] = [x for x in model.key_pointer] 2885 model_type_pointer: Optional[List[Union[InterpolatedString, str]]] = ( 2886 [x for x in model.type_pointer] if model.type_pointer else None 2887 ) 2888 2889 return SchemaTypeIdentifier( 2890 schema_pointer=model_schema_pointer, 2891 key_pointer=model_key_pointer, 2892 type_pointer=model_type_pointer, 2893 types_mapping=types_mapping, 2894 parameters=model.parameters or {}, 2895 ) 2896 2897 def create_dynamic_schema_loader( 2898 self, model: DynamicSchemaLoaderModel, config: Config, **kwargs: Any 2899 ) -> DynamicSchemaLoader: 2900 schema_transformations = [] 2901 if model.schema_transformations: 2902 for transformation_model in model.schema_transformations: 2903 schema_transformations.append( 2904 self._create_component_from_model(model=transformation_model, config=config) 2905 ) 2906 name = "dynamic_properties" 2907 partition_router = self._build_stream_slicer_from_partition_router(model.retriever, config) 2908 retriever_kwargs: Dict[str, Any] = {} 2909 if isinstance(model.retriever, AsyncRetrieverModel): 2910 if model.retriever.partition_router: 2911 raise ValueError( 2912 "DynamicSchemaLoader does not support an AsyncRetriever with a partition_router: only the first record is read, so jobs for other partitions would be orphaned. Remove the partition_router from the schema loader's AsyncRetriever." 2913 ) 2914 # create_async_retriever requires stream_slicer; passing it to a CustomRetriever would override its own 2915 retriever_kwargs["stream_slicer"] = partition_router 2916 retriever = self._create_component_from_model( 2917 model=model.retriever, 2918 config=config, 2919 name=name, 2920 primary_key=None, 2921 partition_router=partition_router, 2922 **retriever_kwargs, 2923 transformations=[], 2924 use_cache=True, 2925 log_formatter=( 2926 lambda response: format_http_message( 2927 response, 2928 f"Schema loader '{name}' request", 2929 f"Request performed in order to extract schema.", 2930 name, 2931 is_auxiliary=True, 2932 ) 2933 ), 2934 ) 2935 schema_type_identifier = self._create_component_from_model( 2936 model.schema_type_identifier, config=config, parameters=model.parameters or {} 2937 ) 2938 schema_filter = ( 2939 self._create_component_from_model( 2940 model.schema_filter, config=config, parameters=model.parameters or {} 2941 ) 2942 if model.schema_filter is not None 2943 else None 2944 ) 2945 2946 return DynamicSchemaLoader( 2947 retriever=retriever, 2948 config=config, 2949 schema_transformations=schema_transformations, 2950 schema_filter=schema_filter, 2951 schema_type_identifier=schema_type_identifier, 2952 parameters=model.parameters or {}, 2953 ) 2954 2955 @staticmethod 2956 def create_json_decoder(model: JsonDecoderModel, config: Config, **kwargs: Any) -> Decoder: 2957 return JsonDecoder(parameters={}) 2958 2959 def create_csv_decoder(self, model: CsvDecoderModel, config: Config, **kwargs: Any) -> Decoder: 2960 return CompositeRawDecoder( 2961 parser=ModelToComponentFactory._get_parser(model, config), 2962 stream_response=False if self._emit_connector_builder_messages else True, 2963 ) 2964 2965 def create_jsonl_decoder( 2966 self, model: JsonlDecoderModel, config: Config, **kwargs: Any 2967 ) -> Decoder: 2968 return CompositeRawDecoder( 2969 parser=ModelToComponentFactory._get_parser(model, config), 2970 stream_response=False if self._emit_connector_builder_messages else True, 2971 ) 2972 2973 def create_json_items_decoder( 2974 self, model: JsonItemsDecoderModel, config: Config, **kwargs: Any 2975 ) -> Decoder: 2976 return CompositeRawDecoder( 2977 parser=ModelToComponentFactory._get_parser(model, config), 2978 stream_response=False if self._emit_connector_builder_messages else True, 2979 ) 2980 2981 def create_gzip_decoder( 2982 self, model: GzipDecoderModel, config: Config, **kwargs: Any 2983 ) -> Decoder: 2984 _compressed_response_types = { 2985 "gzip", 2986 "x-gzip", 2987 "gzip, deflate", 2988 "x-gzip, deflate", 2989 "application/zip", 2990 "application/gzip", 2991 "application/x-gzip", 2992 "application/x-zip-compressed", 2993 } 2994 2995 gzip_parser: GzipParser = ModelToComponentFactory._get_parser(model, config) # type: ignore # based on the model, we know this will be a GzipParser 2996 2997 if self._emit_connector_builder_messages: 2998 return CompositeRawDecoder(gzip_parser, False) 2999 3000 transport_gzip_parser = GzipParser(inner_parser=gzip_parser) 3001 return CompositeRawDecoder.by_headers( 3002 [ 3003 ({"Content-Encoding"}, {"gzip"}, transport_gzip_parser), 3004 ({"Content-Type"}, _compressed_response_types, gzip_parser), 3005 ], 3006 stream_response=True, 3007 fallback_parser=gzip_parser, 3008 ) 3009 3010 @staticmethod 3011 def create_iterable_decoder( 3012 model: IterableDecoderModel, config: Config, **kwargs: Any 3013 ) -> IterableDecoder: 3014 return IterableDecoder(parameters={}) 3015 3016 @staticmethod 3017 def create_xml_decoder(model: XmlDecoderModel, config: Config, **kwargs: Any) -> XmlDecoder: 3018 return XmlDecoder(parameters={}) 3019 3020 def create_zipfile_decoder( 3021 self, model: ZipfileDecoderModel, config: Config, **kwargs: Any 3022 ) -> ZipfileDecoder: 3023 return ZipfileDecoder(parser=ModelToComponentFactory._get_parser(model.decoder, config)) 3024 3025 @staticmethod 3026 def _get_parser(model: BaseModel, config: Config) -> Parser: 3027 if isinstance(model, JsonDecoderModel): 3028 # Note that the logic is a bit different from the JsonDecoder as there is some legacy that is maintained to return {} on error cases 3029 return JsonParser() 3030 elif isinstance(model, JsonItemsDecoderModel): 3031 return JsonItemsParser( 3032 items_path=model.items_path, 3033 encoding=model.encoding, 3034 ) 3035 elif isinstance(model, JsonlDecoderModel): 3036 return JsonLineParser() 3037 elif isinstance(model, CsvDecoderModel): 3038 return CsvParser( 3039 encoding=model.encoding, 3040 delimiter=model.delimiter, 3041 set_values_to_none=model.set_values_to_none, 3042 ) 3043 elif isinstance(model, GzipDecoderModel): 3044 return GzipParser( 3045 inner_parser=ModelToComponentFactory._get_parser(model.decoder, config) 3046 ) 3047 elif isinstance( 3048 model, (CustomDecoderModel, IterableDecoderModel, XmlDecoderModel, ZipfileDecoderModel) 3049 ): 3050 raise ValueError(f"Decoder type {model} does not have parser associated to it") 3051 3052 raise ValueError(f"Unknown decoder type {model}") 3053 3054 @staticmethod 3055 def create_json_file_schema_loader( 3056 model: JsonFileSchemaLoaderModel, config: Config, **kwargs: Any 3057 ) -> JsonFileSchemaLoader: 3058 return JsonFileSchemaLoader( 3059 file_path=model.file_path or "", config=config, parameters=model.parameters or {} 3060 ) 3061 3062 def create_jwt_authenticator( 3063 self, model: JwtAuthenticatorModel, config: Config, **kwargs: Any 3064 ) -> JwtAuthenticator: 3065 jwt_headers = model.jwt_headers or JwtHeadersModel(kid=None, typ="JWT", cty=None) 3066 jwt_payload = model.jwt_payload or JwtPayloadModel(iss=None, sub=None, aud=None) 3067 request_option = ( 3068 self._create_component_from_model(model.request_option, config) 3069 if model.request_option 3070 else None 3071 ) 3072 return JwtAuthenticator( 3073 config=config, 3074 parameters=model.parameters or {}, 3075 algorithm=JwtAlgorithm(model.algorithm.value), 3076 secret_key=model.secret_key, 3077 base64_encode_secret_key=model.base64_encode_secret_key, 3078 token_duration=model.token_duration, 3079 header_prefix=model.header_prefix, 3080 kid=jwt_headers.kid, 3081 typ=jwt_headers.typ, 3082 cty=jwt_headers.cty, 3083 iss=jwt_payload.iss, 3084 sub=jwt_payload.sub, 3085 aud=jwt_payload.aud, 3086 additional_jwt_headers=model.additional_jwt_headers, 3087 additional_jwt_payload=model.additional_jwt_payload, 3088 passphrase=model.passphrase, 3089 request_option=request_option, 3090 ) 3091 3092 def create_list_partition_router( 3093 self, model: ListPartitionRouterModel, config: Config, **kwargs: Any 3094 ) -> ListPartitionRouter: 3095 request_option = ( 3096 self._create_component_from_model(model.request_option, config) 3097 if model.request_option 3098 else None 3099 ) 3100 return ListPartitionRouter( 3101 cursor_field=model.cursor_field, 3102 request_option=request_option, 3103 values=model.values, 3104 config=config, 3105 parameters=model.parameters or {}, 3106 ) 3107 3108 @staticmethod 3109 def create_min_max_datetime( 3110 model: MinMaxDatetimeModel, config: Config, **kwargs: Any 3111 ) -> MinMaxDatetime: 3112 return MinMaxDatetime( 3113 datetime=model.datetime, 3114 datetime_format=model.datetime_format or "", 3115 max_datetime=model.max_datetime or "", 3116 min_datetime=model.min_datetime or "", 3117 parameters=model.parameters or {}, 3118 ) 3119 3120 @staticmethod 3121 def create_no_auth(model: NoAuthModel, config: Config, **kwargs: Any) -> NoAuth: 3122 return NoAuth(parameters=model.parameters or {}) 3123 3124 @staticmethod 3125 def create_no_pagination( 3126 model: NoPaginationModel, config: Config, **kwargs: Any 3127 ) -> NoPagination: 3128 return NoPagination(parameters={}) 3129 3130 def create_oauth_authenticator( 3131 self, model: OAuthAuthenticatorModel, config: Config, **kwargs: Any 3132 ) -> DeclarativeOauth2Authenticator: 3133 profile_assertion = ( 3134 self._create_component_from_model(model.profile_assertion, config=config) 3135 if model.profile_assertion 3136 else None 3137 ) 3138 3139 refresh_token_error_status_codes, refresh_token_error_key, refresh_token_error_values = ( 3140 self._get_refresh_token_error_information(model) 3141 ) 3142 if model.refresh_token_updater: 3143 # ignore type error because fixing it would have a lot of dependencies, revisit later 3144 return DeclarativeSingleUseRefreshTokenOauth2Authenticator( # type: ignore 3145 config, 3146 InterpolatedString.create( 3147 model.token_refresh_endpoint, # type: ignore 3148 parameters=model.parameters or {}, 3149 ).eval(config), 3150 access_token_name=InterpolatedString.create( 3151 model.access_token_name or "access_token", parameters=model.parameters or {} 3152 ).eval(config), 3153 refresh_token_name=model.refresh_token_updater.refresh_token_name, 3154 expires_in_name=InterpolatedString.create( 3155 model.expires_in_name or "expires_in", parameters=model.parameters or {} 3156 ).eval(config), 3157 client_id_name=InterpolatedString.create( 3158 model.client_id_name or "client_id", parameters=model.parameters or {} 3159 ).eval(config), 3160 client_id=InterpolatedString.create( 3161 model.client_id, parameters=model.parameters or {} 3162 ).eval(config) 3163 if model.client_id 3164 else model.client_id, 3165 client_secret_name=InterpolatedString.create( 3166 model.client_secret_name or "client_secret", parameters=model.parameters or {} 3167 ).eval(config), 3168 client_secret=InterpolatedString.create( 3169 model.client_secret, parameters=model.parameters or {} 3170 ).eval(config) 3171 if model.client_secret 3172 else model.client_secret, 3173 access_token_config_path=model.refresh_token_updater.access_token_config_path, 3174 refresh_token_config_path=model.refresh_token_updater.refresh_token_config_path, 3175 token_expiry_date_config_path=model.refresh_token_updater.token_expiry_date_config_path, 3176 grant_type_name=InterpolatedString.create( 3177 model.grant_type_name or "grant_type", parameters=model.parameters or {} 3178 ).eval(config), 3179 grant_type=InterpolatedString.create( 3180 model.grant_type or "refresh_token", parameters=model.parameters or {} 3181 ).eval(config), 3182 refresh_request_body=InterpolatedMapping( 3183 model.refresh_request_body or {}, parameters=model.parameters or {} 3184 ).eval(config), 3185 refresh_request_headers=InterpolatedMapping( 3186 model.refresh_request_headers or {}, parameters=model.parameters or {} 3187 ).eval(config), 3188 send_refresh_request_as_query_params=bool( 3189 model.send_refresh_request_as_query_params 3190 ), 3191 scopes=model.scopes, 3192 token_expiry_date_format=model.token_expiry_date_format, 3193 token_expiry_is_time_of_expiration=bool(model.token_expiry_date_format), 3194 message_repository=self._message_repository, 3195 refresh_token_error_status_codes=refresh_token_error_status_codes, 3196 refresh_token_error_key=refresh_token_error_key, 3197 refresh_token_error_values=refresh_token_error_values, 3198 ) 3199 # ignore type error because fixing it would have a lot of dependencies, revisit later 3200 return DeclarativeOauth2Authenticator( # type: ignore 3201 access_token_name=model.access_token_name or "access_token", 3202 access_token_value=model.access_token_value, 3203 client_id_name=model.client_id_name or "client_id", 3204 client_id=model.client_id, 3205 client_secret_name=model.client_secret_name or "client_secret", 3206 client_secret=model.client_secret, 3207 expires_in_name=model.expires_in_name or "expires_in", 3208 grant_type_name=model.grant_type_name or "grant_type", 3209 grant_type=model.grant_type or "refresh_token", 3210 refresh_request_body=model.refresh_request_body, 3211 refresh_request_headers=model.refresh_request_headers, 3212 send_refresh_request_as_query_params=bool(model.send_refresh_request_as_query_params), 3213 refresh_token_name=model.refresh_token_name or "refresh_token", 3214 refresh_token=model.refresh_token, 3215 scopes=model.scopes, 3216 token_expiry_date=model.token_expiry_date, 3217 token_expiry_date_format=model.token_expiry_date_format, 3218 token_expiry_is_time_of_expiration=bool(model.token_expiry_date_format), 3219 token_refresh_endpoint=model.token_refresh_endpoint, 3220 config=config, 3221 parameters=model.parameters or {}, 3222 message_repository=self._message_repository, 3223 profile_assertion=profile_assertion, 3224 use_profile_assertion=model.use_profile_assertion, 3225 refresh_token_error_status_codes=refresh_token_error_status_codes, 3226 refresh_token_error_key=refresh_token_error_key, 3227 refresh_token_error_values=refresh_token_error_values, 3228 ) 3229 3230 @staticmethod 3231 def _get_refresh_token_error_information( 3232 model: OAuthAuthenticatorModel, 3233 ) -> Tuple[Tuple[int, ...], str, Tuple[str, ...]]: 3234 """ 3235 In a previous version of the CDK, the auth error as config_error was only done if a refresh token updater was 3236 defined. As a transition, we added those fields on the OAuthAuthenticatorModel. This method ensures that the 3237 information is defined only once and return the right fields. 3238 """ 3239 refresh_token_updater = model.refresh_token_updater 3240 is_defined_on_refresh_token_updated = refresh_token_updater and ( 3241 refresh_token_updater.refresh_token_error_status_codes 3242 or refresh_token_updater.refresh_token_error_key 3243 or refresh_token_updater.refresh_token_error_values 3244 ) 3245 is_defined_on_oauth_authenticator = ( 3246 model.refresh_token_error_status_codes 3247 or model.refresh_token_error_key 3248 or model.refresh_token_error_values 3249 ) 3250 if is_defined_on_refresh_token_updated and is_defined_on_oauth_authenticator: 3251 raise ValueError( 3252 "refresh_token_error should either be defined on the OAuthAuthenticatorModel or the RefreshTokenUpdaterModel, not both" 3253 ) 3254 3255 if is_defined_on_refresh_token_updated: 3256 not_optional_refresh_token_updater: RefreshTokenUpdaterModel = refresh_token_updater # type: ignore # we know from the condition that this is not None 3257 return ( 3258 tuple(not_optional_refresh_token_updater.refresh_token_error_status_codes) 3259 if not_optional_refresh_token_updater.refresh_token_error_status_codes 3260 else (), 3261 not_optional_refresh_token_updater.refresh_token_error_key or "", 3262 tuple(not_optional_refresh_token_updater.refresh_token_error_values) 3263 if not_optional_refresh_token_updater.refresh_token_error_values 3264 else (), 3265 ) 3266 elif is_defined_on_oauth_authenticator: 3267 return ( 3268 tuple(model.refresh_token_error_status_codes) 3269 if model.refresh_token_error_status_codes 3270 else (), 3271 model.refresh_token_error_key or "", 3272 tuple(model.refresh_token_error_values) if model.refresh_token_error_values else (), 3273 ) 3274 3275 # returning default values we think cover most cases 3276 return (400,), "error", ("invalid_grant", "invalid_permissions") 3277 3278 @staticmethod 3279 def _reject_union_combined_extractor_for_offset_increment( 3280 extractor_model: Optional[BaseModel], 3281 ) -> None: 3282 """Refuse an `OffsetIncrement` paginator driven by a `union` `CombinedExtractor`. 3283 3284 `OffsetIncrement.next_page_token` advances the offset by the number of records its 3285 extractor returns for the page. Under `union` that number is the sum over all 3286 sub-extractors, so the offset overshoots the API page size and every page after the first 3287 starts past the records that were never read: with two sub-extractors returning two records 3288 each and `page_size: 2`, the requested offsets are 0, 4, 8 instead of 0, 2, 4 and two 3289 thirds of the records are silently dropped. 3290 3291 `first_match` returns the winning sub-extractor's count, which does not inflate the count, 3292 and is left alone. A `union` nested anywhere in the tree inflates the count of the node 3293 above it, so the whole tree is walked. 3294 """ 3295 if not isinstance(extractor_model, CombinedExtractorModel): 3296 return 3297 if not ModelToComponentFactory._combined_extractor_tree_contains_union(extractor_model): 3298 return 3299 raise AirbyteTracedException( 3300 message=( 3301 'CombinedExtractor mode "union" is not supported with an OffsetIncrement paginator.' 3302 ), 3303 internal_message=( 3304 "OffsetIncrement counts the records of its extractor to advance the offset. A " 3305 "`union` CombinedExtractor returns the sum of its sub-extractors' records, which " 3306 "overshoots the page the API returned, so records would be skipped. Use the " 3307 "`first_match` mode, a CursorPagination or PageIncrement paginator, or a single " 3308 "extractor." 3309 ), 3310 failure_type=FailureType.config_error, 3311 ) 3312 3313 @staticmethod 3314 def _combined_extractor_tree_contains_union(model: CombinedExtractorModel) -> bool: 3315 mode = CombineMode(model.mode.value) if model.mode else CombineMode.union 3316 if mode == CombineMode.union: 3317 return True 3318 return any( 3319 isinstance(sub_extractor, CombinedExtractorModel) 3320 and ModelToComponentFactory._combined_extractor_tree_contains_union(sub_extractor) 3321 for sub_extractor in model.extractors 3322 ) 3323 3324 @staticmethod 3325 def _count_dropped_empty_records(extractor: RecordExtractor) -> None: 3326 """Make the record selector count the empty records a `CombinedExtractor` drops. 3327 3328 `OffsetIncrement` and `PageIncrement` stop when that count falls below the page size, so a 3329 full page holding a null would otherwise look short and silently end pagination. 3330 """ 3331 if not isinstance(extractor, CombinedExtractor): 3332 return 3333 extractor.count_dropped_empty_records = True 3334 for sub_extractor in extractor.extractors: 3335 ModelToComponentFactory._count_dropped_empty_records(sub_extractor) 3336 3337 def create_offset_increment( 3338 self, 3339 model: OffsetIncrementModel, 3340 config: Config, 3341 decoder: Optional[Decoder] = None, 3342 extractor_model: Optional[ 3343 Union[CustomRecordExtractorModel, DpathExtractorModel, CombinedExtractorModel] 3344 ] = None, 3345 **kwargs: Any, 3346 ) -> OffsetIncrement: 3347 # None when nested under a custom component, which cannot pass a decoder 3348 if decoder is None: 3349 decoder = JsonDecoder(parameters={}) 3350 if isinstance(decoder, PaginationDecoderDecorator): 3351 inner_decoder = decoder.decoder 3352 else: 3353 inner_decoder = decoder 3354 decoder = PaginationDecoderDecorator(decoder=decoder) 3355 3356 if self._is_supported_decoder_for_pagination(inner_decoder): 3357 decoder_to_use = decoder 3358 else: 3359 raise ValueError( 3360 self._UNSUPPORTED_DECODER_ERROR.format(decoder_type=type(inner_decoder)) 3361 ) 3362 3363 self._reject_union_combined_extractor_for_offset_increment(extractor_model) 3364 3365 # Pydantic v1 Union type coercion can convert int to string depending on Union order. 3366 # If page_size is a string that represents an integer (not an interpolation), convert it back. 3367 page_size = model.page_size 3368 if isinstance(page_size, str) and page_size.isdigit(): 3369 page_size = int(page_size) 3370 3371 return OffsetIncrement( 3372 page_size=page_size, 3373 config=config, 3374 decoder=decoder_to_use, 3375 inject_on_first_request=model.inject_on_first_request or False, 3376 parameters=model.parameters or {}, 3377 ) 3378 3379 @staticmethod 3380 def create_page_increment( 3381 model: PageIncrementModel, config: Config, **kwargs: Any 3382 ) -> PageIncrement: 3383 # Pydantic v1 Union type coercion can convert int to string depending on Union order. 3384 # If page_size is a string that represents an integer (not an interpolation), convert it back. 3385 page_size = model.page_size 3386 if isinstance(page_size, str) and page_size.isdigit(): 3387 page_size = int(page_size) 3388 3389 return PageIncrement( 3390 page_size=page_size, 3391 config=config, 3392 start_from_page=model.start_from_page or 0, 3393 inject_on_first_request=model.inject_on_first_request or False, 3394 parameters=model.parameters or {}, 3395 ) 3396 3397 def create_parent_stream_config( 3398 self, model: ParentStreamConfigModel, config: Config, *, stream_name: str, **kwargs: Any 3399 ) -> ParentStreamConfig: 3400 declarative_stream = self._create_component_from_model( 3401 model.stream, 3402 config=config, 3403 is_parent=True, 3404 **kwargs, 3405 ) 3406 request_option = ( 3407 self._create_component_from_model(model.request_option, config=config) 3408 if model.request_option 3409 else None 3410 ) 3411 3412 if model.lazy_read_pointer and any("*" in pointer for pointer in model.lazy_read_pointer): 3413 raise ValueError( 3414 "The '*' wildcard in 'lazy_read_pointer' is not supported — only direct paths are allowed." 3415 ) 3416 3417 model_lazy_read_pointer: List[Union[InterpolatedString, str]] = ( 3418 [x for x in model.lazy_read_pointer] if model.lazy_read_pointer else [] 3419 ) 3420 3421 return ParentStreamConfig( 3422 parent_key=model.parent_key, 3423 request_option=request_option, 3424 stream=declarative_stream, 3425 partition_field=model.partition_field, 3426 config=config, 3427 incremental_dependency=model.incremental_dependency or False, 3428 parameters=model.parameters or {}, 3429 extra_fields=model.extra_fields, 3430 lazy_read_pointer=model_lazy_read_pointer, 3431 ) 3432 3433 def create_properties_from_endpoint( 3434 self, model: PropertiesFromEndpointModel, config: Config, **kwargs: Any 3435 ) -> PropertiesFromEndpoint: 3436 retriever = self._create_component_from_model( 3437 model=model.retriever, 3438 config=config, 3439 name="dynamic_properties", 3440 primary_key=None, 3441 stream_slicer=None, 3442 transformations=[], 3443 use_cache=True, # Enable caching on the HttpRequester/HttpClient because the properties endpoint will be called for every slice being processed, and it is highly unlikely for the response to different 3444 ) 3445 return PropertiesFromEndpoint( 3446 property_field_path=model.property_field_path, 3447 retriever=retriever, 3448 config=config, 3449 parameters=model.parameters or {}, 3450 ) 3451 3452 def create_property_chunking( 3453 self, model: PropertyChunkingModel, config: Config, **kwargs: Any 3454 ) -> PropertyChunking: 3455 record_merge_strategy = ( 3456 self._create_component_from_model( 3457 model=model.record_merge_strategy, config=config, **kwargs 3458 ) 3459 if model.record_merge_strategy 3460 else None 3461 ) 3462 3463 property_limit_type: PropertyLimitType 3464 match model.property_limit_type: 3465 case PropertyLimitTypeModel.property_count: 3466 property_limit_type = PropertyLimitType.property_count 3467 case PropertyLimitTypeModel.characters: 3468 property_limit_type = PropertyLimitType.characters 3469 case _: 3470 raise ValueError(f"Invalid PropertyLimitType {property_limit_type}") 3471 3472 return PropertyChunking( 3473 property_limit_type=property_limit_type, 3474 property_limit=model.property_limit, 3475 record_merge_strategy=record_merge_strategy, 3476 config=config, 3477 parameters=model.parameters or {}, 3478 ) 3479 3480 def create_query_properties( 3481 self, model: QueryPropertiesModel, config: Config, *, stream_name: str, **kwargs: Any 3482 ) -> QueryProperties: 3483 if isinstance(model.property_list, list): 3484 property_list = model.property_list 3485 else: 3486 property_list = self._create_component_from_model( 3487 model=model.property_list, config=config, **kwargs 3488 ) 3489 3490 property_chunking = ( 3491 self._create_component_from_model( 3492 model=model.property_chunking, config=config, **kwargs 3493 ) 3494 if model.property_chunking 3495 else None 3496 ) 3497 3498 property_selector = ( 3499 self._create_component_from_model( 3500 model=model.property_selector, config=config, stream_name=stream_name, **kwargs 3501 ) 3502 if model.property_selector 3503 else None 3504 ) 3505 3506 return QueryProperties( 3507 property_list=property_list, 3508 always_include_properties=model.always_include_properties, 3509 property_chunking=property_chunking, 3510 property_selector=property_selector, 3511 config=config, 3512 parameters=model.parameters or {}, 3513 ) 3514 3515 def create_json_schema_property_selector( 3516 self, 3517 model: JsonSchemaPropertySelectorModel, 3518 config: Config, 3519 *, 3520 stream_name: str, 3521 **kwargs: Any, 3522 ) -> JsonSchemaPropertySelector: 3523 configured_stream = self._stream_name_to_configured_stream.get(stream_name) 3524 3525 transformations = [] 3526 if model.transformations: 3527 for transformation_model in model.transformations: 3528 transformations.append( 3529 self._create_component_from_model(model=transformation_model, config=config) 3530 ) 3531 3532 return JsonSchemaPropertySelector( 3533 configured_stream=configured_stream, 3534 properties_transformations=transformations, 3535 config=config, 3536 parameters=model.parameters or {}, 3537 ) 3538 3539 @staticmethod 3540 def create_record_filter( 3541 model: RecordFilterModel, config: Config, **kwargs: Any 3542 ) -> RecordFilter: 3543 return RecordFilter( 3544 condition=model.condition or "", config=config, parameters=model.parameters or {} 3545 ) 3546 3547 @staticmethod 3548 def create_request_path(model: RequestPathModel, config: Config, **kwargs: Any) -> RequestPath: 3549 return RequestPath(parameters={}) 3550 3551 @staticmethod 3552 def create_request_option( 3553 model: RequestOptionModel, config: Config, **kwargs: Any 3554 ) -> RequestOption: 3555 inject_into = RequestOptionType(model.inject_into.value) 3556 field_path: Optional[List[Union[InterpolatedString, str]]] = ( 3557 [ 3558 InterpolatedString.create(segment, parameters=kwargs.get("parameters", {})) 3559 for segment in model.field_path 3560 ] 3561 if model.field_path 3562 else None 3563 ) 3564 field_name = ( 3565 InterpolatedString.create(model.field_name, parameters=kwargs.get("parameters", {})) 3566 if model.field_name 3567 else None 3568 ) 3569 return RequestOption( 3570 field_name=field_name, 3571 field_path=field_path, 3572 inject_into=inject_into, 3573 parameters=kwargs.get("parameters", {}), 3574 ) 3575 3576 def create_record_selector( 3577 self, 3578 model: RecordSelectorModel, 3579 config: Config, 3580 *, 3581 name: str, 3582 transformations: List[RecordTransformation] | None = None, 3583 decoder: Decoder | None = None, 3584 client_side_incremental_sync_cursor: Optional[Cursor] = None, 3585 is_client_side_incremental_sync: bool = False, 3586 file_uploader: Optional[DefaultFileUploader] = None, 3587 **kwargs: Any, 3588 ) -> RecordSelector: 3589 extractor = self._create_component_from_model( 3590 model=model.extractor, decoder=decoder, config=config 3591 ) 3592 self._count_dropped_empty_records(extractor) 3593 record_filter = ( 3594 self._create_component_from_model(model.record_filter, config=config) 3595 if model.record_filter 3596 else None 3597 ) 3598 3599 # A client-side incremental stream transforms before filtering by default. That default belongs to the flag, 3600 # not to the component that ends up doing the cursor comparison: a data feed does it in the retriever and 3601 # receives no cursor here, but its `record_filter` condition must keep running after the transformations. 3602 default_transform_before_filtering = bool( 3603 client_side_incremental_sync_cursor or is_client_side_incremental_sync 3604 ) 3605 transform_before_filtering = ( 3606 default_transform_before_filtering 3607 if model.transform_before_filtering is None 3608 else model.transform_before_filtering 3609 ) 3610 if client_side_incremental_sync_cursor: 3611 record_filter = ClientSideIncrementalRecordFilterDecorator( 3612 config=config, 3613 parameters=model.parameters, 3614 condition=model.record_filter.condition 3615 if (model.record_filter and hasattr(model.record_filter, "condition")) 3616 else None, 3617 cursor=client_side_incremental_sync_cursor, 3618 ) 3619 3620 if model.schema_normalization is None: 3621 # default to no schema normalization if not set 3622 model.schema_normalization = SchemaNormalizationModel.None_ 3623 3624 schema_normalization = ( 3625 TypeTransformer(SCHEMA_TRANSFORMER_TYPE_MAPPING[model.schema_normalization]) 3626 if isinstance(model.schema_normalization, SchemaNormalizationModel) 3627 else self._create_component_from_model(model.schema_normalization, config=config) # type: ignore[arg-type] # custom normalization model expected here 3628 ) 3629 3630 return RecordSelector( 3631 extractor=extractor, 3632 name=name, 3633 config=config, 3634 record_filter=record_filter, 3635 transformations=transformations or [], 3636 file_uploader=file_uploader, 3637 schema_normalization=schema_normalization, 3638 parameters=model.parameters or {}, 3639 transform_before_filtering=transform_before_filtering, 3640 ) 3641 3642 @staticmethod 3643 def create_remove_fields( 3644 model: RemoveFieldsModel, config: Config, **kwargs: Any 3645 ) -> RemoveFields: 3646 return RemoveFields( 3647 field_pointers=model.field_pointers, condition=model.condition or "", parameters={} 3648 ) 3649 3650 def create_selective_authenticator( 3651 self, 3652 model: SelectiveAuthenticatorModel, 3653 config: Config, 3654 *, 3655 name: Optional[str] = None, 3656 url_base: Optional[str] = None, 3657 **kwargs: Any, 3658 ) -> DeclarativeAuthenticator: 3659 # Keyword-only so that _create_nested_component also fills them under a custom component, 3660 # from the parent's kwargs or $parameters. None is not forwarded, so a missing required one 3661 # keeps its hint. 3662 nested_kwargs = { 3663 key: value 3664 for key, value in {"name": name, "url_base": url_base}.items() 3665 if value is not None 3666 } 3667 authenticators = { 3668 key: self._create_component_from_model( 3669 model=auth, config=config, **nested_kwargs, **kwargs 3670 ) 3671 for key, auth in model.authenticators.items() 3672 } 3673 # SelectiveAuthenticator will return instance of DeclarativeAuthenticator or raise ValueError error 3674 return SelectiveAuthenticator( # type: ignore[abstract] 3675 config=config, 3676 authenticators=authenticators, 3677 authenticator_selection_path=model.authenticator_selection_path, 3678 **kwargs, 3679 ) 3680 3681 @staticmethod 3682 def create_legacy_session_token_authenticator( 3683 model: LegacySessionTokenAuthenticatorModel, config: Config, *, url_base: str, **kwargs: Any 3684 ) -> LegacySessionTokenAuthenticator: 3685 return LegacySessionTokenAuthenticator( 3686 api_url=url_base, 3687 header=model.header, 3688 login_url=model.login_url, 3689 password=model.password or "", 3690 session_token=model.session_token or "", 3691 session_token_response_key=model.session_token_response_key or "", 3692 username=model.username or "", 3693 validate_session_url=model.validate_session_url, 3694 config=config, 3695 parameters=model.parameters or {}, 3696 ) 3697 3698 def create_simple_retriever( 3699 self, 3700 model: SimpleRetrieverModel, 3701 config: Config, 3702 *, 3703 name: str, 3704 primary_key: Optional[Union[str, List[str], List[List[str]]]], 3705 request_options_provider: Optional[RequestOptionsProvider] = None, 3706 cursor: Optional[Cursor] = None, 3707 has_stop_condition_cursor: bool = False, 3708 is_client_side_incremental_sync: bool = False, 3709 transformations: List[RecordTransformation], 3710 file_uploader: Optional[DefaultFileUploader] = None, 3711 incremental_sync: Optional[ 3712 Union[IncrementingCountCursorModel, DatetimeBasedCursorModel] 3713 ] = None, 3714 use_cache: Optional[bool] = None, 3715 log_formatter: Optional[Callable[[Response], Any]] = None, 3716 partition_router: Optional[PartitionRouter] = None, 3717 **kwargs: Any, 3718 ) -> SimpleRetriever: 3719 def _get_url(req: Requester) -> str: 3720 """ 3721 Closure to get the URL from the requester. This is used to get the URL in the case of a lazy retriever. 3722 This is needed because the URL is not set until the requester is created. 3723 """ 3724 3725 _url: str = ( 3726 model.requester.url 3727 if hasattr(model.requester, "url") and model.requester.url is not None 3728 else req.get_url(stream_state=None, stream_slice=None, next_page_token=None) 3729 ) 3730 _url_base: str = ( 3731 model.requester.url_base 3732 if hasattr(model.requester, "url_base") and model.requester.url_base is not None 3733 else req.get_url_base(stream_state=None, stream_slice=None, next_page_token=None) 3734 ) 3735 3736 return _url or _url_base 3737 3738 if cursor is None: 3739 cursor = FinalStateCursor(name, None, self._message_repository) 3740 3741 # A data feed drops the records the cursor considers already synced in the retriever, which sits downstream of 3742 # the paginator. Letting the record selector drop them as well would be redundant and would hide them from the 3743 # pagination stop condition, so a data feed never delegates that filtering to the record selector, whether 3744 # `is_client_side_incremental` is set or not. The `condition` from `record_filter` is intentionally left out of 3745 # the post-pagination filter and stays in the record selector, which preserves the existing behaviour: the 3746 # selector runs inside the page loop, so the records the condition rejects never reach the paginator's 3747 # accounting. Moving it downstream would start counting them. 3748 post_pagination_filter = ( 3749 ClientSideIncrementalRecordFilterDecorator( 3750 config=config, 3751 parameters=model.parameters or {}, 3752 condition=None, 3753 cursor=cursor, 3754 ) 3755 if has_stop_condition_cursor 3756 else None 3757 ) 3758 client_side_incremental_cursor = ( 3759 cursor if is_client_side_incremental_sync and not post_pagination_filter else None 3760 ) 3761 if post_pagination_filter and is_client_side_incremental_sync: 3762 LOGGER.warning( 3763 f"Stream {name}: `is_client_side_incremental` adds no record filtering when `is_data_feed` is set, " 3764 "as a data feed already filters on the cursor value. It still makes the record selector apply the " 3765 "transformations before the `record_filter` condition." 3766 ) 3767 3768 decoder = ( 3769 self._create_component_from_model(model=model.decoder, config=config) 3770 if model.decoder 3771 else JsonDecoder(parameters={}) 3772 ) 3773 record_selector = self._create_component_from_model( 3774 model=model.record_selector, 3775 name=name, 3776 config=config, 3777 decoder=decoder, 3778 transformations=transformations, 3779 client_side_incremental_sync_cursor=client_side_incremental_cursor, 3780 is_client_side_incremental_sync=is_client_side_incremental_sync, 3781 file_uploader=file_uploader, 3782 ) 3783 3784 query_properties: Optional[QueryProperties] = None 3785 query_properties_key: Optional[str] = None 3786 self._ensure_query_properties_to_model(model.requester) 3787 if self._has_query_properties_in_request_parameters(model.requester): 3788 # It is better to be explicit about an error if PropertiesFromEndpoint is defined in multiple 3789 # places instead of default to request_parameters which isn't clearly documented 3790 if ( 3791 hasattr(model.requester, "fetch_properties_from_endpoint") 3792 and model.requester.fetch_properties_from_endpoint 3793 ): 3794 raise ValueError( 3795 f"PropertiesFromEndpoint should only be specified once per stream, but found in {model.requester.type}.fetch_properties_from_endpoint and {model.requester.type}.request_parameters" 3796 ) 3797 3798 query_properties_definitions = [] 3799 for key, request_parameter in model.requester.request_parameters.items(): # type: ignore # request_parameters is already validated to be a Mapping using _has_query_properties_in_request_parameters() 3800 if isinstance(request_parameter, QueryPropertiesModel): 3801 query_properties_key = key 3802 query_properties_definitions.append(request_parameter) 3803 3804 if len(query_properties_definitions) > 1: 3805 raise ValueError( 3806 f"request_parameters only supports defining one QueryProperties field, but found {len(query_properties_definitions)} usages" 3807 ) 3808 3809 if len(query_properties_definitions) == 1: 3810 query_properties = self._create_component_from_model( 3811 model=query_properties_definitions[0], stream_name=name, config=config 3812 ) 3813 3814 # Removes QueryProperties components from the interpolated mappings because it has been designed 3815 # to be used by the SimpleRetriever and will be resolved from the provider from the slice directly 3816 # instead of through jinja interpolation 3817 if hasattr(model.requester, "request_parameters") and isinstance( 3818 model.requester.request_parameters, Mapping 3819 ): 3820 model.requester.request_parameters = self._remove_query_properties( 3821 model.requester.request_parameters 3822 ) 3823 elif ( 3824 hasattr(model.requester, "fetch_properties_from_endpoint") 3825 and model.requester.fetch_properties_from_endpoint 3826 ): 3827 # todo: Deprecate this condition once dependent connectors migrate to query_properties 3828 query_properties_definition = QueryPropertiesModel( 3829 type="QueryProperties", 3830 property_list=model.requester.fetch_properties_from_endpoint, 3831 always_include_properties=None, 3832 property_chunking=None, 3833 ) # type: ignore # $parameters has a default value 3834 3835 query_properties = self.create_query_properties( 3836 model=query_properties_definition, 3837 stream_name=name, 3838 config=config, 3839 ) 3840 elif hasattr(model.requester, "query_properties") and model.requester.query_properties: 3841 query_properties = self.create_query_properties( 3842 model=model.requester.query_properties, 3843 stream_name=name, 3844 config=config, 3845 ) 3846 3847 requester = self._create_component_from_model( 3848 model=model.requester, 3849 decoder=decoder, 3850 name=name, 3851 query_properties_key=query_properties_key, 3852 use_cache=use_cache, 3853 config=config, 3854 ) 3855 3856 if not request_options_provider: 3857 request_options_provider = DefaultRequestOptionsProvider(parameters={}) 3858 if isinstance(request_options_provider, DefaultRequestOptionsProvider) and isinstance( 3859 partition_router, PartitionRouter 3860 ): 3861 request_options_provider = partition_router 3862 3863 paginator = ( 3864 self._create_component_from_model( 3865 model=model.paginator, 3866 config=config, 3867 url_base=_get_url(requester), 3868 extractor_model=model.record_selector.extractor, 3869 decoder=decoder, 3870 cursor_used_for_stop_condition=cursor if has_stop_condition_cursor else None, 3871 ) 3872 if model.paginator 3873 else NoPagination(parameters={}) 3874 ) 3875 3876 ignore_stream_slicer_parameters_on_paginated_requests = ( 3877 model.ignore_stream_slicer_parameters_on_paginated_requests or False 3878 ) 3879 3880 reads_parent_stream_lazily = bool( 3881 model.partition_router 3882 and isinstance(model.partition_router, SubstreamPartitionRouterModel) 3883 and any( 3884 parent_stream_config.lazy_read_pointer 3885 for parent_stream_config in model.partition_router.parent_stream_configs 3886 ) 3887 ) 3888 if reads_parent_stream_lazily and ( 3889 model.page_size_reduction 3890 or self._uses_reduce_page_size_action(getattr(model.requester, "error_handler", None)) 3891 ): 3892 # Checked outside of the LazySimpleRetriever branch below, which only applies on the first 3893 # sync of a stream: gating it on the absence of state would accept the same manifest from the 3894 # second sync onwards. LazySimpleRetriever paginates the parent's embedded pages, so there is 3895 # no page of its own to re-issue with a smaller page size. 3896 raise ValueError( 3897 f"`page_size_reduction` and the REDUCE_PAGE_SIZE response action are not supported when " 3898 f"reading a parent stream lazily. Remove either the page size reduction or the parent " 3899 f"stream's `lazy_read_pointer` for stream {name}." 3900 ) 3901 3902 if reads_parent_stream_lazily and ( 3903 model.request_window_splitting 3904 or self._uses_split_request_window_action( 3905 getattr(model.requester, "error_handler", None) 3906 ) 3907 ): 3908 # Same reasoning as the page_size_reduction check above: LazySimpleRetriever reads records 3909 # embedded in the parent's own pages rather than requesting a window of its own, so there is 3910 # nothing here for a split child window to be read from. 3911 raise ValueError( 3912 f"`request_window_splitting` and the SPLIT_REQUEST_WINDOW response action are not " 3913 f"supported when reading a parent stream lazily. Remove either the request window " 3914 f"splitting or the parent stream's `lazy_read_pointer` for stream {name}." 3915 ) 3916 3917 if reads_parent_stream_lazily and not bool( 3918 self._connector_state_manager.get_stream_state(name, None) 3919 ): 3920 if incremental_sync: 3921 if incremental_sync.type != "DatetimeBasedCursor": 3922 raise ValueError( 3923 f"LazySimpleRetriever only supports DatetimeBasedCursor. Found: {incremental_sync.type}." 3924 ) 3925 3926 elif incremental_sync.step or incremental_sync.cursor_granularity: 3927 raise ValueError( 3928 f"Found more that one slice per parent. LazySimpleRetriever only supports single slice read for stream - {name}." 3929 ) 3930 3931 if model.decoder and model.decoder.type != "JsonDecoder": 3932 raise ValueError( 3933 f"LazySimpleRetriever only supports JsonDecoder. Found: {model.decoder.type}." 3934 ) 3935 3936 return LazySimpleRetriever( 3937 name=name, 3938 paginator=paginator, 3939 primary_key=primary_key, 3940 requester=requester, 3941 record_selector=record_selector, 3942 stream_slicer=_NO_STREAM_SLICING, 3943 request_option_provider=request_options_provider, 3944 config=config, 3945 ignore_stream_slicer_parameters_on_paginated_requests=ignore_stream_slicer_parameters_on_paginated_requests, 3946 post_pagination_filter=post_pagination_filter, 3947 parameters=model.parameters or {}, 3948 ) 3949 3950 if ( 3951 model.record_selector.record_filter 3952 and model.pagination_reset 3953 and model.pagination_reset.limits 3954 ): 3955 raise ValueError("PaginationResetLimits are not supported while having record filter.") 3956 3957 request_window_splitting = self._create_request_window_splitting( 3958 model, name, cursor, incremental_sync, query_properties, file_uploader 3959 ) 3960 return SimpleRetriever( 3961 name=name, 3962 paginator=paginator, 3963 primary_key=primary_key, 3964 requester=requester, 3965 record_selector=record_selector, 3966 stream_slicer=_NO_STREAM_SLICING, 3967 request_option_provider=request_options_provider, 3968 config=config, 3969 ignore_stream_slicer_parameters_on_paginated_requests=ignore_stream_slicer_parameters_on_paginated_requests, 3970 additional_query_properties=query_properties, 3971 log_formatter=self._get_log_formatter(log_formatter, name), 3972 pagination_tracker_factory=self._create_pagination_tracker_factory( 3973 model.pagination_reset, cursor 3974 ), 3975 page_size_reduction=self._create_page_size_reduction( 3976 model, name, query_properties, file_uploader 3977 ), 3978 request_window_splitting=request_window_splitting, 3979 request_window_splitter=( 3980 cursor.split_request_window 3981 if request_window_splitting and hasattr(cursor, "split_request_window") 3982 else None 3983 ), 3984 post_pagination_filter=post_pagination_filter, 3985 parameters=model.parameters or {}, 3986 ) 3987 3988 def _create_page_size_reduction( 3989 self, 3990 model: SimpleRetrieverModel, 3991 name: str, 3992 query_properties: Optional[QueryProperties], 3993 file_uploader: Optional[DefaultFileUploader] = None, 3994 ) -> Optional[PageSizeReduction]: 3995 # A CustomRequester does not necessarily define an error handler. A CustomRequester that does define 3996 # one keeps it as a raw dict rather than a typed model, so this returns False for it as well. 3997 error_handler = getattr(model.requester, "error_handler", None) 3998 uses_action = self._uses_reduce_page_size_action(error_handler) 3999 if uses_action and not model.page_size_reduction: 4000 raise ValueError( 4001 f"Stream {name} has a response filter with the REDUCE_PAGE_SIZE action but the retriever does not " 4002 f"define `page_size_reduction`. Add a `page_size_reduction` block to the retriever." 4003 ) 4004 4005 if not model.page_size_reduction: 4006 return None 4007 4008 if not uses_action: 4009 # Not raised: a CustomErrorHandler can resolve to REDUCE_PAGE_SIZE without us being able to see 4010 # it, so the only safe reaction to a block we cannot tie to an action is a warning. Without it a 4011 # misspelled action leaves the feature silently dead on a stream that only exists because it 4012 # would otherwise fail. 4013 LOGGER.warning( 4014 f"Stream {name} defines `page_size_reduction` but no response filter with the " 4015 f"REDUCE_PAGE_SIZE action was found on its requester. The page size will never be reduced " 4016 f"unless a custom error handler resolves to that action." 4017 ) 4018 4019 self._validate_page_size_reduction_is_supported( 4020 model, name, query_properties, file_uploader 4021 ) 4022 4023 reset_policy = model.page_size_reduction.reset_policy 4024 return PageSizeReduction( 4025 reduction_factor=model.page_size_reduction.reduction_factor, # type: ignore[arg-type] # the schema defines a default 4026 minimum_page_size=model.page_size_reduction.minimum_page_size, # type: ignore[arg-type] # the schema defines a default 4027 max_attempts=model.page_size_reduction.max_attempts, # type: ignore[arg-type] # the schema defines a default 4028 backoff_seconds=model.page_size_reduction.backoff_seconds, # type: ignore[arg-type] # the schema defines a default 4029 retries_at_minimum_page_size=model.page_size_reduction.retries_at_minimum_page_size, # type: ignore[arg-type] # the schema defines a default 4030 failure_message=model.page_size_reduction.failure_message, 4031 reset_policy=PageSizeResetPolicy(reset_policy.value) 4032 if reset_policy is not None 4033 else PageSizeResetPolicy.NEVER, 4034 ) 4035 4036 def _validate_page_size_reduction_is_supported( 4037 self, 4038 model: SimpleRetrieverModel, 4039 name: str, 4040 query_properties: Optional[QueryProperties], 4041 file_uploader: Optional[DefaultFileUploader] = None, 4042 ) -> None: 4043 """ 4044 Page size reduction re-issues the same page with a smaller page size. That is only correct when the next 4045 page does not depend on the page size, and it only has an effect when the paginator injects the page size 4046 in the request. A custom pagination strategy is accepted when it can receive the reduced page size, which 4047 is checked by inspecting its signature rather than by recognizing its type. 4048 """ 4049 if query_properties: 4050 raise ValueError( 4051 f"`page_size_reduction` cannot be used together with query properties on stream {name}. Records " 4052 f"from the earlier property chunks have already been emitted when a chunk asks for a smaller page, " 4053 f"so retrying the page would emit them twice." 4054 ) 4055 4056 if file_uploader: 4057 raise ValueError( 4058 f"`page_size_reduction` cannot be used together with a `file_uploader` on stream {name}. The " 4059 f"file uploader sends one request per record from inside the page's record generator, so a " 4060 f"reduction asked for halfway through a page would re-emit the records already yielded by it." 4061 ) 4062 4063 if not isinstance(model.paginator, DefaultPaginatorModel): 4064 raise ValueError( 4065 f"`page_size_reduction` requires a DefaultPaginator on stream {name} so that the connector can " 4066 f"send a smaller page size." 4067 ) 4068 4069 if not model.paginator.page_size_option: 4070 raise ValueError( 4071 f"`page_size_reduction` requires `page_size_option` on the paginator of stream {name}: without it " 4072 f"the connector cannot tell the API to send a smaller page." 4073 ) 4074 4075 if isinstance(model.paginator.page_token_option, RequestPathModel): 4076 # A RequestPath page token is a full URL built by the API, and it already carries the page size the 4077 # API echoed back. The reduced page size is injected as a request option on top of that URL, so the 4078 # request goes out with the page size twice - the original one from the URL and the reduced one - 4079 # and which of the two the API honors is up to the API. Every page after the first would then keep 4080 # asking for the page size that just failed. 4081 raise ValueError( 4082 f"`page_size_reduction` does not support a `page_token_option` of type RequestPath on stream " 4083 f"{name}. The next page is then requested through a URL returned by the API, which already " 4084 f"carries the page size, so the reduced page size would be sent alongside the original one. Use " 4085 f"a CursorPagination strategy with a `page_token_option` of type RequestOption instead." 4086 ) 4087 4088 strategy = model.paginator.pagination_strategy 4089 if isinstance(strategy, PageIncrementModel): 4090 raise ValueError( 4091 f"`page_size_reduction` does not support the PageIncrement pagination strategy used by stream " 4092 f"{name}. Pages are addressed as page number * page size, so a smaller page size shifts every " 4093 f"following page boundary and would skip records. Use OffsetIncrement or CursorPagination." 4094 ) 4095 if isinstance(strategy, CustomPaginationStrategyModel): 4096 # A custom strategy is written by the same person enabling the reduction, so the 4097 # question is not whether we recognize it but whether it can be told the reduced 4098 # page size. Checking the signature keeps a strategy that would raise TypeError 4099 # mid-sync from being accepted at config time. 4100 custom_class = self._get_class_from_fully_qualified_class_name(strategy.class_name) 4101 if isinstance(custom_class, type) and issubclass(custom_class, PageIncrement): 4102 # `PageIncrement.next_page_token` declares `page_size_override` only to reject it, so a 4103 # subclass that does not override the method would pass the signature check below and only 4104 # fail once the first reduction is requested, mid-sync. 4105 raise ValueError( 4106 f"`page_size_reduction` does not support the PageIncrement pagination strategy that the " 4107 f"custom pagination strategy {strategy.class_name} used by stream {name} inherits from. " 4108 f"Pages are addressed as page number * page size, so a smaller page size shifts every " 4109 f"following page boundary and would skip records. Use OffsetIncrement or CursorPagination." 4110 ) 4111 try: 4112 parameters = inspect.signature(custom_class.next_page_token).parameters 4113 except (AttributeError, TypeError, ValueError) as exception: 4114 raise ValueError( 4115 f"`page_size_reduction` could not check the signature of `next_page_token` on the custom " 4116 f"pagination strategy {strategy.class_name} used by stream {name}: {exception}. Make sure " 4117 f"`class_name` points at a PaginationStrategy subclass whose `next_page_token` accepts a " 4118 f"`page_size_override` keyword argument." 4119 ) 4120 accepts_override = "page_size_override" in parameters or any( 4121 parameter.kind == inspect.Parameter.VAR_KEYWORD for parameter in parameters.values() 4122 ) 4123 if not accepts_override: 4124 raise ValueError( 4125 f"`page_size_reduction` requires the custom pagination strategy " 4126 f"{strategy.class_name} used by stream {name} to accept a `page_size_override` keyword " 4127 f"argument in `next_page_token`, so that it can honor the reduced page size. Add " 4128 f"`page_size_override: Optional[int] = None` to its signature; a strategy that does not " 4129 f"use its page size as a stop condition can ignore the value." 4130 ) 4131 elif not isinstance(strategy, (CursorPaginationModel, OffsetIncrementModel)): 4132 raise ValueError( 4133 f"`page_size_reduction` only supports the CursorPagination, OffsetIncrement and " 4134 f"CustomPaginationStrategy pagination strategies. Stream {name} uses " 4135 f"{type(strategy).__name__}." 4136 ) 4137 else: 4138 # A CustomPaginationStrategy carries its page size in its own code, so this is only checkable 4139 # for the strategies the CDK defines. `page_size` is optional on both of them, and without it 4140 # `get_page_size` returns None, the paginator injects nothing, and the reduction is a dead end 4141 # that only surfaces on the first failing response - after records have been emitted. 4142 self._validate_page_size_is_reducible(strategy, model.page_size_reduction, name) 4143 if isinstance(strategy, CursorPaginationModel): 4144 self._validate_stop_condition_is_reduction_aware( 4145 strategy, model.page_size_reduction, name 4146 ) 4147 4148 @staticmethod 4149 def _validate_stop_condition_is_reduction_aware( 4150 strategy: CursorPaginationModel, 4151 page_size_reduction: Optional[PageSizeReductionModel], 4152 name: str, 4153 ) -> None: 4154 """ 4155 A `stop_condition` comparing `last_page_size` to a hardcoded page size reads a full reduced page as a 4156 short page and ends the pagination early, dropping the rest of the partition without failing. The 4157 strategy exposes the page size that was actually requested as `page_size`, so the condition can be 4158 written correctly - but only if it is, which is what this checks. 4159 4160 The condition is parsed as a Jinja expression rather than matched as a string: only the AST tells 4161 `page_size`, which follows the reduction, apart from `config['page_size']`, which does not, and only the 4162 AST tells an inequality, which a reduction can invalidate, apart from `last_page_size == 0`, which it 4163 cannot. A condition that never names `last_page_size` is not waved through: a count read from the 4164 response body - `{{ response['data'] | length < 100 }}` - truncates in exactly the same way, and which 4165 response field counts the records of a page is not knowable here. A shape the analysis does not 4166 understand is warned about rather than rejected - this runs at stream construction, so a false 4167 rejection takes `check`, `discover` and `read` down with it. 4168 """ 4169 stop_condition = strategy.stop_condition 4170 if not stop_condition: 4171 return 4172 4173 minimum_page_size = ( 4174 page_size_reduction.minimum_page_size if page_size_reduction else None 4175 ) or 1 4176 verdict, reason = classify_stop_condition(stop_condition, minimum_page_size) 4177 if verdict is StopConditionSafety.TRUNCATES: 4178 raise ValueError( 4179 f"`page_size_reduction` on stream {name} cannot be used with the `stop_condition` " 4180 f"{stop_condition!r}: {reason}. The pagination would then end early, silently dropping the " 4181 f"rest of the partition. Compare against the `page_size` interpolation variable instead, " 4182 f"which holds the page size that was actually requested (for example " 4183 f"`{{{{ last_page_size < page_size }}}}`), or test the page for emptiness with " 4184 f"`{{{{ last_page_size == 0 }}}}`." 4185 ) 4186 if verdict is StopConditionSafety.UNKNOWN: 4187 LOGGER.warning( 4188 f"Stream {name} uses `page_size_reduction` with the `stop_condition` {stop_condition!r}, " 4189 f"which could not be checked against the reduction because {reason}. Make sure a page that is " 4190 f"full at a reduced page size does not satisfy it, otherwise the pagination ends early and the " 4191 f"rest of the partition is silently dropped. Comparing against the `page_size` interpolation " 4192 f"variable, which holds the page size that was actually requested, is always safe." 4193 ) 4194 4195 @staticmethod 4196 def _validate_page_size_is_reducible( 4197 strategy: Union[CursorPaginationModel, OffsetIncrementModel], 4198 page_size_reduction: Optional[PageSizeReductionModel], 4199 name: str, 4200 ) -> None: 4201 page_size = strategy.page_size 4202 if page_size is None: 4203 raise ValueError( 4204 f"`page_size_reduction` requires `page_size` on the pagination strategy of stream {name}: " 4205 f"without it the paginator does not send a page size, so there is nothing to reduce." 4206 ) 4207 4208 # The schema allows a string so that the page size can be interpolated. `page_size` is 4209 # `Optional[Union[int, str]]` and pydantic v1 tries `int` first, so both `100` and `"100"` arrive as 4210 # an int and only a genuinely non-numeric template stays a str. Such a template is only known once 4211 # the config is available, so it is left to the runtime check in `PageSizeReducer.reduce`. 4212 try: 4213 configured_page_size: Optional[int] = int(page_size) 4214 except ValueError: 4215 configured_page_size = None 4216 4217 minimum_page_size = ( 4218 page_size_reduction.minimum_page_size if page_size_reduction else None 4219 ) or 1 4220 if configured_page_size is not None and configured_page_size <= minimum_page_size: 4221 raise ValueError( 4222 f"`page_size_reduction` on stream {name} can never reduce its page size: the pagination " 4223 f"strategy's `page_size` is {configured_page_size} and `minimum_page_size` is " 4224 f"{minimum_page_size}. Lower `minimum_page_size` or raise `page_size`." 4225 ) 4226 4227 # Each reduction divides the page size by `reduction_factor` and spends one attempt, so an unbroken 4228 # run of failures bottoms out at `page_size / reduction_factor ** max_attempts` whatever 4229 # `minimum_page_size` says. Only warned about, and not raised: pages that succeed in between restart 4230 # the budget while `NEVER` keeps the page size, so the floor is reachable over a partition even when 4231 # it is out of reach of a single run - and a manifest that deliberately gives up earlier than its 4232 # floor is not wrong, only worth pointing out. 4233 reduction_factor = ( 4234 page_size_reduction.reduction_factor if page_size_reduction else None 4235 ) or 2.0 4236 max_attempts = (page_size_reduction.max_attempts if page_size_reduction else None) or 5 4237 # Only when there is a floor worth reaching. The default of 1 is out of reach of the default budget on 4238 # any page size above 32, so this would otherwise fire on nearly every stream that opts in - and 4239 # `__fields_set__` would not help, since it records that a value was supplied and not that it differs 4240 # from the default, so spelling `minimum_page_size: 1` out longhand would earn the warning. 4241 if ( 4242 configured_page_size is not None 4243 and minimum_page_size > 1 4244 and reduction_factor**max_attempts < configured_page_size / minimum_page_size 4245 ): 4246 reachable_page_size = max( 4247 minimum_page_size, int(configured_page_size // reduction_factor**max_attempts) 4248 ) 4249 LOGGER.warning( 4250 f"Stream {name} sets `minimum_page_size` to {minimum_page_size}, which `page_size_reduction` " 4251 f"cannot reach in one run of failing pages: `max_attempts` is {max_attempts} and " 4252 f"`reduction_factor` is {reduction_factor}, so {max_attempts} reductions of a page size of " 4253 f"{configured_page_size} stop at {reachable_page_size} records per page and the sync then " 4254 f"fails with a transient error. Whichever of the two bounds is tighter wins; reaching " 4255 f"{minimum_page_size} in one run needs `max_attempts` of at least " 4256 f"{math.ceil(math.log(configured_page_size / minimum_page_size, reduction_factor))}." 4257 ) 4258 4259 def _reject_reduce_page_size_action(self, requester: Any, description: str) -> None: 4260 """ 4261 `error_handler` is defined on `HttpRequester`, which is referenced by requesters that have no page of 4262 their own, so REDUCE_PAGE_SIZE is schema-legal in places where nothing can honor it. Only 4263 `SimpleRetriever._read_pages` re-issues a page, so every other requester is rejected here rather than 4264 surfacing the exception mid-sync as a generic failure. 4265 """ 4266 if requester is None: 4267 return 4268 if self._uses_reduce_page_size_action(getattr(requester, "error_handler", None)): 4269 raise ValueError( 4270 f"The REDUCE_PAGE_SIZE response action is not supported on the {description}: only the main " 4271 f"requester of a SimpleRetriever can re-issue its page with a smaller page size. Use a " 4272 f"different action on that error handler." 4273 ) 4274 4275 def _uses_reduce_page_size_action(self, error_handler: Any) -> bool: 4276 if isinstance(error_handler, CompositeErrorHandlerModel): 4277 return any( 4278 self._uses_reduce_page_size_action(nested) 4279 for nested in error_handler.error_handlers 4280 ) 4281 if isinstance(error_handler, DefaultErrorHandlerModel): 4282 return any( 4283 response_filter.action == HttpResponseFilterActionModel.REDUCE_PAGE_SIZE 4284 for response_filter in error_handler.response_filters or [] 4285 ) 4286 # A CustomErrorHandler can return any action and we cannot inspect it, so we do not validate it. 4287 return False 4288 4289 def _create_request_window_splitting( 4290 self, 4291 model: SimpleRetrieverModel, 4292 name: str, 4293 cursor: Optional[Cursor], 4294 incremental_sync: Optional[ 4295 Union[IncrementingCountCursorModel, DatetimeBasedCursorModel] 4296 ] = None, 4297 query_properties: Optional[QueryProperties] = None, 4298 file_uploader: Optional[DefaultFileUploader] = None, 4299 ) -> Optional[RequestWindowSplitting]: 4300 # A CustomRequester does not necessarily define an error handler. A CustomRequester that does define 4301 # one keeps it as a raw dict rather than a typed model, so this returns False for it as well. 4302 error_handler = getattr(model.requester, "error_handler", None) 4303 uses_action = self._uses_split_request_window_action(error_handler) 4304 if uses_action and not model.request_window_splitting: 4305 raise ValueError( 4306 f"Stream {name} has a response filter with the SPLIT_REQUEST_WINDOW action but the " 4307 f"retriever does not define `request_window_splitting`. Add a `request_window_splitting` " 4308 f"block to the retriever." 4309 ) 4310 4311 if not model.request_window_splitting: 4312 return None 4313 4314 if not uses_action: 4315 # Not raised: a CustomErrorHandler can resolve to SPLIT_REQUEST_WINDOW without us being able to 4316 # see it, and custom code can raise RequestWindowSplitRequiredException directly without 4317 # going through any error handler at all - so the only safe reaction to a block we cannot tie to 4318 # a known trigger is a warning. 4319 LOGGER.warning( 4320 f"Stream {name} defines `request_window_splitting` but no response filter with the " 4321 f"SPLIT_REQUEST_WINDOW action was found on its requester. The request window will never " 4322 f"be split unless a custom error handler or custom code resolves to that action." 4323 ) 4324 4325 self._validate_request_window_splitting_is_supported( 4326 name, 4327 cursor, 4328 incremental_sync, 4329 model.request_window_splitting.min_split_window, 4330 query_properties, 4331 file_uploader, 4332 ) 4333 4334 return RequestWindowSplitting( 4335 failure_message=model.request_window_splitting.failure_message, 4336 min_split_window=parse_duration(model.request_window_splitting.min_split_window) 4337 if model.request_window_splitting.min_split_window 4338 else None, 4339 ) 4340 4341 def _validate_request_window_splitting_is_supported( 4342 self, 4343 name: str, 4344 cursor: Optional[Cursor], 4345 incremental_sync: Optional[ 4346 Union[IncrementingCountCursorModel, DatetimeBasedCursorModel] 4347 ] = None, 4348 min_split_window: Optional[str] = None, 4349 query_properties: Optional[QueryProperties] = None, 4350 file_uploader: Optional[DefaultFileUploader] = None, 4351 ) -> None: 4352 """ 4353 Request window splitting replaces a failing slice with smaller children derived from the stream's own 4354 cursor, so it only makes sense on a stream whose cursor exposes a `split_request_window` method and it 4355 is rejected for the same structural reasons `page_size_reduction` is: `additional_query_properties` and 4356 a `file_uploader` both mean records of the failing window may already have been emitted by the time the 4357 split is requested, from inside the record generator rather than from a page boundary this retriever 4358 controls. 4359 """ 4360 if not hasattr(cursor, "split_request_window"): 4361 raise ValueError( 4362 f"`request_window_splitting` requires an incremental cursor that supports window splitting on " 4363 f"stream {name}. Found {type(cursor).__name__ if cursor is not None else 'no cursor'}: this is " 4364 f"only supported today for a `DatetimeBasedCursor` with `cursor_granularity` set." 4365 ) 4366 4367 if not ( 4368 isinstance(incremental_sync, DatetimeBasedCursorModel) 4369 and incremental_sync.cursor_granularity 4370 ): 4371 # `cursor_granularity` is both the smallest window the connector will ever request and what keeps 4372 # two child windows from overlapping at their shared edge, so `split_request_window` cannot split 4373 # at all without it - checked here, on the manifest model, rather than by reaching into the 4374 # cursor's private state from outside the class it belongs to. 4375 raise ValueError( 4376 f"`request_window_splitting` requires `cursor_granularity` on the `DatetimeBasedCursor` of " 4377 f"stream {name}: without it there is no smallest window to stop splitting at, and no way to " 4378 f"keep two child windows from overlapping at their shared edge." 4379 ) 4380 4381 if incremental_sync.is_client_side_incremental: 4382 # A client-side-incremental cursor filters records after they are read rather than sending the 4383 # window as request parameters, so every child window sends the exact same request as its parent: 4384 # splitting changes nothing, and the sync would keep splitting all the way to `min_split_window` 4385 # or the safety-net depth before failing. 4386 raise ValueError( 4387 f"`request_window_splitting` cannot be used together with `is_client_side_incremental` on " 4388 f"stream {name}: the window is filtered client-side rather than sent to the API, so splitting " 4389 f"it would not change the request and could not resolve a SPLIT_REQUEST_WINDOW response." 4390 ) 4391 4392 parsed_cursor_granularity = parse_duration(incremental_sync.cursor_granularity) 4393 if isinstance( 4394 parsed_cursor_granularity, datetime.timedelta 4395 ) and parsed_cursor_granularity <= datetime.timedelta(0): 4396 # Passes the truthy check above (the string is non-empty), but a zero-or-negative granularity can 4397 # never produce a child strictly smaller than the parent. 4398 raise ValueError( 4399 f"`request_window_splitting` requires a `cursor_granularity` greater than zero on stream " 4400 f"{name}: `{incremental_sync.cursor_granularity}` parses to a zero-length duration, which can " 4401 f"never produce a smaller child window." 4402 ) 4403 4404 smallest_format_unit = self._smallest_datetime_format_unit(incremental_sync.datetime_format) 4405 if ( 4406 smallest_format_unit is not None 4407 and isinstance(parsed_cursor_granularity, datetime.timedelta) 4408 and parsed_cursor_granularity < smallest_format_unit 4409 ): 4410 # Two children split closer together than `datetime_format` can render would format to the same 4411 # boundary value instead of splitting. 4412 raise ValueError( 4413 f"`request_window_splitting` requires `cursor_granularity` to be no finer than what " 4414 f"`datetime_format` can represent on stream {name}: `datetime_format` " 4415 f"{incremental_sync.datetime_format!r} cannot represent a value finer than " 4416 f"{smallest_format_unit}, but `cursor_granularity` {incremental_sync.cursor_granularity!r} is " 4417 f"finer than that. Use a `datetime_format` precise enough to represent `cursor_granularity`, " 4418 f"or a coarser `cursor_granularity`." 4419 ) 4420 4421 if min_split_window: 4422 parsed_min_split_window = parse_duration(min_split_window) 4423 if not isinstance( 4424 parsed_min_split_window, datetime.timedelta 4425 ) or parsed_min_split_window <= datetime.timedelta(0): 4426 # A duration with years or months (e.g. `P1M`) parses to an `isodate.Duration`, which has no 4427 # fixed length to compare a window's span against - `split_request_window` would raise 4428 # mid-sync trying to. A zero-or-negative duration would never stop splitting. 4429 raise ValueError( 4430 f"`min_split_window` must be a positive duration expressible as a fixed number of " 4431 f"days/hours/minutes/seconds on stream {name}: `{min_split_window}` is not." 4432 ) 4433 4434 if query_properties: 4435 raise ValueError( 4436 f"`request_window_splitting` cannot be used together with query properties on stream {name}. " 4437 f"Records from the earlier property chunks may have already been emitted when a chunk asks to " 4438 f"split the window, so splitting and re-reading it could duplicate them." 4439 ) 4440 4441 if file_uploader: 4442 raise ValueError( 4443 f"`request_window_splitting` cannot be used together with a `file_uploader` on stream {name}. " 4444 f"The file uploader sends one request per record from inside the window's record generator, so " 4445 f"a split requested partway through could re-emit records already yielded by it." 4446 ) 4447 4448 @staticmethod 4449 def _smallest_datetime_format_unit(datetime_format: str) -> Optional[datetime.timedelta]: 4450 """ 4451 :return: the finest duration `datetime_format` can distinguish, per 4452 `_DATETIME_FORMAT_DIRECTIVE_GRANULARITY`, or `None` if it contains none of those directives. 4453 """ 4454 matches = [ 4455 granularity 4456 for directive, granularity in _DATETIME_FORMAT_DIRECTIVE_GRANULARITY 4457 if directive in datetime_format 4458 ] 4459 return min(matches) if matches else None 4460 4461 def _reject_split_request_window_action(self, requester: Any, description: str) -> None: 4462 """ 4463 `error_handler` is defined on `HttpRequester`, which is referenced by requesters that have no window of 4464 their own to split, so SPLIT_REQUEST_WINDOW is schema-legal in places where nothing can honor it. Only 4465 the main requester of a `SimpleRetriever` reads a cursor-sliced window, so every other requester is 4466 rejected here rather than surfacing the exception mid-sync as a generic failure. 4467 """ 4468 if requester is None: 4469 return 4470 if self._uses_split_request_window_action(getattr(requester, "error_handler", None)): 4471 raise ValueError( 4472 f"The SPLIT_REQUEST_WINDOW response action is not supported on the {description}: only the " 4473 f"main requester of a SimpleRetriever reads a window that can be split. Use a different " 4474 f"action on that error handler." 4475 ) 4476 4477 def _uses_split_request_window_action(self, error_handler: Any) -> bool: 4478 if isinstance(error_handler, CompositeErrorHandlerModel): 4479 return any( 4480 self._uses_split_request_window_action(nested) 4481 for nested in error_handler.error_handlers 4482 ) 4483 if isinstance(error_handler, DefaultErrorHandlerModel): 4484 return any( 4485 response_filter.action == HttpResponseFilterActionModel.SPLIT_REQUEST_WINDOW 4486 for response_filter in error_handler.response_filters or [] 4487 ) 4488 # A CustomErrorHandler can return any action and we cannot inspect it, so we do not validate it. 4489 return False 4490 4491 def _create_pagination_tracker_factory( 4492 self, model: Optional[PaginationResetModel], cursor: Cursor 4493 ) -> Callable[[], PaginationTracker]: 4494 if model is None: 4495 return lambda: PaginationTracker() 4496 4497 # Until we figure out a way to use any cursor for PaginationTracker, we will have to have this cursor selector logic 4498 cursor_factory: Callable[[], Optional[ConcurrentCursor]] = lambda: None 4499 if model.action == PaginationResetActionModel.RESET: 4500 # in that case, we will let cursor_factory to return None even if the stream has a cursor 4501 pass 4502 elif model.action == PaginationResetActionModel.SPLIT_USING_CURSOR: 4503 if isinstance(cursor, ConcurrentCursor): 4504 cursor_factory = lambda: cursor.copy_without_state() # type: ignore # the if condition validates that it is a ConcurrentCursor 4505 elif isinstance(cursor, ConcurrentPerPartitionCursor): 4506 cursor_factory = lambda: cursor._cursor_factory.create( # type: ignore # if this becomes a problem, we would need to extract the cursor_factory instantiation logic and make it accessible here 4507 {}, datetime.timedelta(0) 4508 ) 4509 elif not isinstance(cursor, FinalStateCursor): 4510 LOGGER.warning( 4511 "Unknown cursor for PaginationTracker. Pagination resets might not work properly" 4512 ) 4513 else: 4514 raise ValueError(f"Unknown PaginationReset action: {model.action}") 4515 4516 limit = model.limits.number_of_records if model and model.limits else None 4517 return lambda: PaginationTracker(cursor_factory(), limit) 4518 4519 def _get_log_formatter( 4520 self, log_formatter: Callable[[Response], Any] | None, name: str 4521 ) -> Callable[[Response], Any] | None: 4522 if self._should_limit_slices_fetched(): 4523 return ( 4524 ( 4525 lambda response: format_http_message( 4526 response, 4527 f"Stream '{name}' request", 4528 f"Request performed in order to extract records for stream '{name}'", 4529 name, 4530 ) 4531 ) 4532 if not log_formatter 4533 else log_formatter 4534 ) 4535 return None 4536 4537 def _should_limit_slices_fetched(self) -> bool: 4538 """ 4539 Returns True if the number of slices fetched should be limited, False otherwise. 4540 This is used to limit the number of slices fetched during tests. 4541 """ 4542 return bool(self._limit_slices_fetched or self._emit_connector_builder_messages) 4543 4544 @staticmethod 4545 def _has_query_properties_in_request_parameters( 4546 requester: Union[HttpRequesterModel, CustomRequesterModel], 4547 ) -> bool: 4548 if not hasattr(requester, "request_parameters"): 4549 return False 4550 request_parameters = requester.request_parameters 4551 if request_parameters and isinstance(request_parameters, Mapping): 4552 for request_parameter in request_parameters.values(): 4553 if isinstance(request_parameter, QueryPropertiesModel): 4554 return True 4555 return False 4556 4557 @staticmethod 4558 def _remove_query_properties( 4559 request_parameters: Mapping[str, Union[str, QueryPropertiesModel]], 4560 ) -> Mapping[str, str]: 4561 return { 4562 parameter_field: request_parameter 4563 for parameter_field, request_parameter in request_parameters.items() 4564 if not isinstance(request_parameter, QueryPropertiesModel) 4565 } 4566 4567 def create_state_delegating_stream( 4568 self, 4569 model: StateDelegatingStreamModel, 4570 config: Config, 4571 **kwargs: Any, 4572 ) -> DefaultStream: 4573 if ( 4574 model.full_refresh_stream.name != model.name 4575 or model.name != model.incremental_stream.name 4576 ): 4577 raise ValueError( 4578 f"state_delegating_stream, full_refresh_stream name and incremental_stream must have equal names. Instead has {model.name}, {model.full_refresh_stream.name} and {model.incremental_stream.name}." 4579 ) 4580 4581 # Resolve api_retention_period with config context (supports Jinja2 interpolation) 4582 resolved_retention_period: Optional[str] = None 4583 if model.api_retention_period: 4584 interpolated_retention = InterpolatedString.create( 4585 model.api_retention_period, parameters=model.parameters or {} 4586 ) 4587 resolved_value = interpolated_retention.eval(config=config) 4588 if resolved_value: 4589 resolved_retention_period = str(resolved_value) 4590 4591 if resolved_retention_period: 4592 for stream_model in (model.full_refresh_stream, model.incremental_stream): 4593 if isinstance(stream_model.incremental_sync, IncrementingCountCursorModel): 4594 raise ValueError( 4595 f"Stream '{model.name}' uses IncrementingCountCursor which is not supported " 4596 f"with api_retention_period. IncrementingCountCursor does not use datetime-based " 4597 f"cursors, so cursor age validation cannot be performed." 4598 ) 4599 4600 stream_state = self._connector_state_manager.get_stream_state(model.name, None) 4601 4602 if not stream_state: 4603 return self._create_component_from_model( # type: ignore[no-any-return] 4604 model.full_refresh_stream, config=config, **kwargs 4605 ) 4606 4607 incremental_stream: DefaultStream = self._create_component_from_model( 4608 model.incremental_stream, config=config, **kwargs 4609 ) # type: ignore[assignment] 4610 4611 # Only run cursor age validation for streams that are in the configured 4612 # catalog (or when no catalog was provided, e.g. during discover / connector 4613 # builder). Streams not selected by the user but instantiated as parent-stream 4614 # dependencies must not go through this path because it emits state messages 4615 # that the destination does not know about, causing "Stream not found" crashes. 4616 stream_is_in_catalog = ( 4617 not self._stream_name_to_configured_stream # no catalog → validate by default 4618 or model.name in self._stream_name_to_configured_stream 4619 ) 4620 if resolved_retention_period and stream_is_in_catalog: 4621 full_refresh_stream: DefaultStream = self._create_component_from_model( 4622 model.full_refresh_stream, config=config, **kwargs 4623 ) # type: ignore[assignment] 4624 if self._is_cursor_older_than_retention_period( 4625 stream_state, 4626 full_refresh_stream.cursor, 4627 incremental_stream.cursor, 4628 resolved_retention_period, 4629 model.name, 4630 ): 4631 # Clear state BEFORE constructing the full_refresh_stream so that 4632 # its cursor starts from start_date instead of the stale cursor. 4633 self._connector_state_manager.update_state_for_stream(model.name, None, {}) 4634 state_message = self._connector_state_manager.create_state_message(model.name, None) 4635 self._message_repository.emit_message(state_message) 4636 return self._create_component_from_model( # type: ignore[no-any-return] 4637 model.full_refresh_stream, config=config, **kwargs 4638 ) 4639 4640 return incremental_stream 4641 4642 @staticmethod 4643 def _is_cursor_older_than_retention_period( 4644 stream_state: Mapping[str, Any], 4645 full_refresh_cursor: Cursor, 4646 incremental_cursor: Cursor, 4647 api_retention_period: str, 4648 stream_name: str, 4649 ) -> bool: 4650 """Check if the cursor value in the state is older than the API's retention period. 4651 4652 Checks cursors in sequence: full refresh cursor first, then incremental cursor. 4653 FinalStateCursor returns now() for completed full refresh state (NO_CURSOR_STATE_KEY), 4654 which is always within retention, so we use incremental. For other states, it returns 4655 None and we fall back to checking the incremental cursor. 4656 4657 Returns True if the cursor is older than the retention period (should use full refresh). 4658 Returns False if the cursor is within the retention period (safe to use incremental). 4659 """ 4660 retention_duration = parse_duration(api_retention_period) 4661 retention_cutoff = datetime.datetime.now(datetime.timezone.utc) - retention_duration 4662 4663 # Check full refresh cursor first 4664 cursor_datetime = full_refresh_cursor.get_cursor_datetime_from_state(stream_state) 4665 4666 # If full refresh cursor returns None, check incremental cursor 4667 if cursor_datetime is None: 4668 cursor_datetime = incremental_cursor.get_cursor_datetime_from_state(stream_state) 4669 4670 if cursor_datetime is None: 4671 # Neither cursor could parse the state - fall back to full refresh to be safe 4672 return True 4673 4674 if cursor_datetime < retention_cutoff: 4675 logging.warning( 4676 f"Stream '{stream_name}' has a cursor value older than " 4677 f"the API's retention period of {api_retention_period} " 4678 f"(cutoff: {retention_cutoff.isoformat()}). " 4679 f"Falling back to full refresh to avoid data loss." 4680 ) 4681 return True 4682 4683 return False 4684 4685 def _get_state_delegating_stream_model( 4686 self, 4687 model: StateDelegatingStreamModel, 4688 parent_state: Optional[Mapping[str, Any]] = None, 4689 ) -> DeclarativeStreamModel: 4690 """Return the appropriate underlying stream model based on state.""" 4691 return ( 4692 model.incremental_stream 4693 if self._connector_state_manager.get_stream_state(model.name, None) or parent_state 4694 else model.full_refresh_stream 4695 ) 4696 4697 _OPTIONAL_ASYNC_STATUS_FIELDS = {"skipped"} 4698 4699 def _create_async_job_status_mapping( 4700 self, model: AsyncJobStatusMapModel, config: Config, **kwargs: Any 4701 ) -> Mapping[str, AsyncJobStatus]: 4702 api_status_to_cdk_status = {} 4703 for cdk_status, api_statuses in model.dict().items(): 4704 if cdk_status == "type": 4705 # This is an element of the dict because of the typing of the CDK but it is not a CDK status 4706 continue 4707 4708 if api_statuses is None: 4709 if cdk_status in self._OPTIONAL_ASYNC_STATUS_FIELDS: 4710 continue 4711 raise ValueError( 4712 f"Required CDK status '{cdk_status}' has no API statuses mapped. " 4713 f"Please provide at least an empty list for required status fields." 4714 ) 4715 4716 for status in api_statuses: 4717 if status in api_status_to_cdk_status: 4718 raise ValueError( 4719 f"API status {status} is already set for CDK status {cdk_status}. Please ensure API statuses are only provided once" 4720 ) 4721 api_status_to_cdk_status[status] = self._get_async_job_status(cdk_status) 4722 return api_status_to_cdk_status 4723 4724 def _get_async_job_status(self, status: str) -> AsyncJobStatus: 4725 match status: 4726 case "running": 4727 return AsyncJobStatus.RUNNING 4728 case "completed": 4729 return AsyncJobStatus.COMPLETED 4730 case "failed": 4731 return AsyncJobStatus.FAILED 4732 case "timeout": 4733 return AsyncJobStatus.TIMED_OUT 4734 case "skipped": 4735 return AsyncJobStatus.SKIPPED 4736 case _: 4737 raise ValueError(f"Unsupported CDK status {status}") 4738 4739 def create_async_retriever( 4740 self, 4741 model: AsyncRetrieverModel, 4742 config: Config, 4743 *, 4744 name: str, 4745 primary_key: Optional[ 4746 Union[str, List[str], List[List[str]]] 4747 ], # this seems to be needed to match create_simple_retriever 4748 stream_slicer: Optional[StreamSlicer], 4749 client_side_incremental_sync: Optional[Dict[str, Any]] = None, 4750 transformations: List[RecordTransformation], 4751 **kwargs: Any, 4752 ) -> AsyncRetriever: 4753 if model.download_target_requester and not model.download_target_extractor: 4754 raise ValueError( 4755 f"`download_target_extractor` required if using a `download_target_requester`" 4756 ) 4757 4758 for requester_field in ( 4759 "creation_requester", 4760 "polling_requester", 4761 "download_requester", 4762 "download_target_requester", 4763 "abort_requester", 4764 "delete_requester", 4765 ): 4766 self._reject_reduce_page_size_action( 4767 getattr(model, requester_field, None), 4768 f"`{requester_field}` of the AsyncRetriever of stream {name}", 4769 ) 4770 self._reject_split_request_window_action( 4771 getattr(model, requester_field, None), 4772 f"`{requester_field}` of the AsyncRetriever of stream {name}", 4773 ) 4774 4775 def _get_download_retriever( 4776 requester: Requester, extractor: RecordExtractor, _decoder: Decoder 4777 ) -> SimpleRetriever: 4778 self._count_dropped_empty_records(extractor) 4779 # We create a record selector for the download retriever 4780 # with no schema normalization and no transformations, neither record filter 4781 # as all this occurs in the record_selector of the AsyncRetriever 4782 record_selector = RecordSelector( 4783 extractor=extractor, 4784 name=name, 4785 record_filter=None, 4786 transformations=[], 4787 schema_normalization=TypeTransformer(TransformConfig.NoTransform), 4788 config=config, 4789 parameters={}, 4790 ) 4791 paginator = ( 4792 self._create_component_from_model( 4793 model=model.download_paginator, 4794 decoder=_decoder, 4795 config=config, 4796 url_base="", 4797 # Only read by the `union` rejection of `OffsetIncrement`. 4798 extractor_model=( 4799 model.download_extractor 4800 if isinstance(model.download_extractor, CombinedExtractorModel) 4801 else None 4802 ), 4803 ) 4804 if model.download_paginator 4805 else NoPagination(parameters={}) 4806 ) 4807 4808 return SimpleRetriever( 4809 requester=requester, 4810 record_selector=record_selector, 4811 primary_key=None, 4812 name=name, 4813 paginator=paginator, 4814 config=config, 4815 parameters={}, 4816 log_formatter=self._get_log_formatter(None, name), 4817 ) 4818 4819 def _get_job_timeout() -> datetime.timedelta: 4820 user_defined_timeout: Optional[int] = ( 4821 int( 4822 InterpolatedString.create( 4823 str(model.polling_job_timeout), 4824 parameters={}, 4825 ).eval(config) 4826 ) 4827 if model.polling_job_timeout 4828 else None 4829 ) 4830 4831 # check for user defined timeout during the test read or 15 minutes 4832 test_read_timeout = datetime.timedelta(minutes=user_defined_timeout or 15) 4833 # default value for non-connector builder is 60 minutes. 4834 default_sync_timeout = datetime.timedelta(minutes=user_defined_timeout or 60) 4835 4836 return ( 4837 test_read_timeout if self._emit_connector_builder_messages else default_sync_timeout 4838 ) 4839 4840 decoder = ( 4841 self._create_component_from_model(model=model.decoder, config=config) 4842 if model.decoder 4843 else JsonDecoder(parameters={}) 4844 ) 4845 record_selector = self._create_component_from_model( 4846 model=model.record_selector, 4847 config=config, 4848 decoder=decoder, 4849 name=name, 4850 transformations=transformations, 4851 client_side_incremental_sync=client_side_incremental_sync, 4852 ) 4853 4854 stream_slicer = stream_slicer or SinglePartitionRouter(parameters={}) 4855 if self._should_limit_slices_fetched(): 4856 stream_slicer = cast( 4857 StreamSlicer, 4858 StreamSlicerTestReadDecorator( 4859 wrapped_slicer=stream_slicer, 4860 maximum_number_of_slices=self._limit_slices_fetched or 5, 4861 ), 4862 ) 4863 4864 creation_requester = self._create_component_from_model( 4865 model=model.creation_requester, 4866 decoder=decoder, 4867 config=config, 4868 name=f"job creation - {name}", 4869 ) 4870 polling_requester = self._create_component_from_model( 4871 model=model.polling_requester, 4872 decoder=decoder, 4873 config=config, 4874 name=f"job polling - {name}", 4875 ) 4876 job_download_components_name = f"job download - {name}" 4877 download_decoder = ( 4878 self._create_component_from_model(model=model.download_decoder, config=config) 4879 if model.download_decoder 4880 else JsonDecoder(parameters={}) 4881 ) 4882 download_extractor = ( 4883 self._create_component_from_model( 4884 model=model.download_extractor, 4885 config=config, 4886 decoder=download_decoder, 4887 parameters=model.parameters, 4888 ) 4889 if model.download_extractor 4890 else DpathExtractor( 4891 [], 4892 config=config, 4893 decoder=download_decoder, 4894 parameters=model.parameters or {}, 4895 ) 4896 ) 4897 download_requester = self._create_component_from_model( 4898 model=model.download_requester, 4899 decoder=download_decoder, 4900 config=config, 4901 name=job_download_components_name, 4902 ) 4903 download_retriever = _get_download_retriever( 4904 download_requester, download_extractor, download_decoder 4905 ) 4906 abort_requester = ( 4907 self._create_component_from_model( 4908 model=model.abort_requester, 4909 decoder=decoder, 4910 config=config, 4911 name=f"job abort - {name}", 4912 ) 4913 if model.abort_requester 4914 else None 4915 ) 4916 delete_requester = ( 4917 self._create_component_from_model( 4918 model=model.delete_requester, 4919 decoder=decoder, 4920 config=config, 4921 name=f"job delete - {name}", 4922 ) 4923 if model.delete_requester 4924 else None 4925 ) 4926 download_target_requester = ( 4927 self._create_component_from_model( 4928 model=model.download_target_requester, 4929 decoder=decoder, 4930 config=config, 4931 name=f"job extract_url - {name}", 4932 ) 4933 if model.download_target_requester 4934 else None 4935 ) 4936 status_extractor = self._create_component_from_model( 4937 model=model.status_extractor, decoder=decoder, config=config, name=name 4938 ) 4939 download_target_extractor = ( 4940 self._create_component_from_model( 4941 model=model.download_target_extractor, 4942 decoder=decoder, 4943 config=config, 4944 name=name, 4945 ) 4946 if model.download_target_extractor 4947 else None 4948 ) 4949 4950 job_repository: AsyncJobRepository = AsyncHttpJobRepository( 4951 creation_requester=creation_requester, 4952 polling_requester=polling_requester, 4953 download_retriever=download_retriever, 4954 download_target_requester=download_target_requester, 4955 abort_requester=abort_requester, 4956 delete_requester=delete_requester, 4957 status_extractor=status_extractor, 4958 status_mapping=self._create_async_job_status_mapping(model.status_mapping, config), 4959 download_target_extractor=download_target_extractor, 4960 job_timeout=_get_job_timeout(), 4961 ) 4962 4963 failed_retry_wait_time_in_seconds: Optional[int] = ( 4964 int( 4965 InterpolatedString.create( 4966 str(model.failed_retry_wait_time_in_seconds), 4967 parameters={}, 4968 ).eval(config) 4969 ) 4970 if model.failed_retry_wait_time_in_seconds 4971 else None 4972 ) 4973 4974 async_job_partition_router = AsyncJobPartitionRouter( 4975 job_orchestrator_factory=lambda stream_slices: AsyncJobOrchestrator( 4976 job_repository, 4977 stream_slices, 4978 self._job_tracker, 4979 self._message_repository, 4980 # FIXME work would need to be done here in order to detect if a stream as a parent stream that is bulk 4981 has_bulk_parent=False, 4982 # set the `job_max_retry` to 1 for the `Connector Builder`` use-case. 4983 # `None` == default retry is set to 3 attempts, under the hood. 4984 job_max_retry=1 if self._emit_connector_builder_messages else None, 4985 failed_retry_wait_time_in_seconds=failed_retry_wait_time_in_seconds, 4986 ), 4987 stream_slicer=stream_slicer, 4988 config=config, 4989 parameters=model.parameters or {}, 4990 ) 4991 4992 return AsyncRetriever( 4993 record_selector=record_selector, 4994 stream_slicer=async_job_partition_router, 4995 config=config, 4996 parameters=model.parameters or {}, 4997 ) 4998 4999 def create_spec(self, model: SpecModel, config: Config, **kwargs: Any) -> Spec: 5000 config_migrations = [ 5001 self._create_component_from_model(migration, config) 5002 for migration in ( 5003 model.config_normalization_rules.config_migrations 5004 if ( 5005 model.config_normalization_rules 5006 and model.config_normalization_rules.config_migrations 5007 ) 5008 else [] 5009 ) 5010 ] 5011 config_transformations = [ 5012 self._create_component_from_model(transformation, config) 5013 for transformation in ( 5014 model.config_normalization_rules.transformations 5015 if ( 5016 model.config_normalization_rules 5017 and model.config_normalization_rules.transformations 5018 ) 5019 else [] 5020 ) 5021 ] 5022 config_validations = [ 5023 self._create_component_from_model(validation, config) 5024 for validation in ( 5025 model.config_normalization_rules.validations 5026 if ( 5027 model.config_normalization_rules 5028 and model.config_normalization_rules.validations 5029 ) 5030 else [] 5031 ) 5032 ] 5033 5034 return Spec( 5035 connection_specification=model.connection_specification, 5036 documentation_url=model.documentation_url, 5037 advanced_auth=model.advanced_auth, 5038 parameters={}, 5039 config_migrations=config_migrations, 5040 config_transformations=config_transformations, 5041 config_validations=config_validations, 5042 ) 5043 5044 def create_substream_partition_router( 5045 self, 5046 model: SubstreamPartitionRouterModel, 5047 config: Config, 5048 *, 5049 stream_name: str, 5050 **kwargs: Any, 5051 ) -> SubstreamPartitionRouter: 5052 parent_stream_configs = [] 5053 if model.parent_stream_configs: 5054 parent_stream_configs.extend( 5055 [ 5056 self.create_parent_stream_config_with_substream_wrapper( 5057 model=parent_stream_config, config=config, stream_name=stream_name, **kwargs 5058 ) 5059 for parent_stream_config in model.parent_stream_configs 5060 ] 5061 ) 5062 5063 return SubstreamPartitionRouter( 5064 parent_stream_configs=parent_stream_configs, 5065 parameters=model.parameters or {}, 5066 config=config, 5067 ) 5068 5069 def create_parent_stream_config_with_substream_wrapper( 5070 self, model: ParentStreamConfigModel, config: Config, *, stream_name: str, **kwargs: Any 5071 ) -> Any: 5072 child_state = self._connector_state_manager.get_stream_state(stream_name, None) 5073 if NO_CURSOR_STATE_KEY in child_state: 5074 # Full refresh streams checkpoint a `{NO_CURSOR_STATE_KEY: true}` sentinel. When such a 5075 # stream is later converted to incremental with an incremental_dependency parent, 5076 # `_instantiate_parent_stream_state_manager` would treat the sentinel's boolean as a legacy 5077 # cursor value and re-key it under the parent's cursor field, crashing cursor initialization. 5078 child_state = { 5079 key: value for key, value in child_state.items() if key != NO_CURSOR_STATE_KEY 5080 } 5081 5082 parent_state: Optional[Mapping[str, Any]] = ( 5083 child_state if model.incremental_dependency and child_state else None 5084 ) 5085 connector_state_manager = self._instantiate_parent_stream_state_manager( 5086 child_state, config, model, parent_state 5087 ) 5088 5089 substream_factory = ModelToComponentFactory( 5090 custom_components_trusted=self._custom_components_trusted, 5091 connector_state_manager=connector_state_manager, 5092 limit_pages_fetched_per_slice=self._limit_pages_fetched_per_slice, 5093 limit_slices_fetched=self._limit_slices_fetched, 5094 emit_connector_builder_messages=self._emit_connector_builder_messages, 5095 disable_retries=self._disable_retries, 5096 disable_cache=self._disable_cache, 5097 message_repository=StateFilteringMessageRepository( 5098 LogAppenderMessageRepositoryDecorator( 5099 { 5100 "airbyte_cdk": {"stream": {"is_substream": True}}, 5101 "http": {"is_auxiliary": True}, 5102 }, 5103 self._message_repository, 5104 self._evaluate_log_level(self._emit_connector_builder_messages), 5105 ), 5106 ), 5107 api_budget=self._api_budget, 5108 # Share the authenticator registry so parent and child streams draw from the 5109 # same token quota counters 5110 rate_limited_authenticators=self._rate_limited_authenticators, 5111 ) 5112 5113 return substream_factory.create_parent_stream_config( 5114 model=model, config=config, stream_name=stream_name, **kwargs 5115 ) 5116 5117 def _instantiate_parent_stream_state_manager( 5118 self, 5119 child_state: MutableMapping[str, Any], 5120 config: Config, 5121 model: ParentStreamConfigModel, 5122 parent_state: Optional[Mapping[str, Any]] = None, 5123 ) -> ConnectorStateManager: 5124 """ 5125 With DefaultStream, the state needs to be provided during __init__ of the cursor as opposed to the 5126 `set_initial_state` flow that existed for the declarative cursors. This state is taken from 5127 self._connector_state_manager.get_stream_state (`self` being a newly created ModelToComponentFactory to account 5128 for the MessageRepository being different). So we need to pass a ConnectorStateManager to the 5129 ModelToComponentFactory that has the parent states. This method populates this if there is a child state and if 5130 incremental_dependency is set. 5131 """ 5132 if model.incremental_dependency and child_state: 5133 parent_stream_name = model.stream.name or "" 5134 extracted_parent_state = ConcurrentPerPartitionCursor.get_parent_state( 5135 child_state, parent_stream_name 5136 ) 5137 5138 if not extracted_parent_state: 5139 extracted_parent_state = ConcurrentPerPartitionCursor.get_global_state( 5140 child_state, parent_stream_name 5141 ) 5142 5143 if not extracted_parent_state and not isinstance(extracted_parent_state, dict): 5144 cursor_values = list(child_state.values()) 5145 # Only a scalar legacy cursor value can seed the parent. Sentinels such as 5146 # `{"__ab_full_refresh_sync_complete": true}` or legacy per-partition `states` 5147 # would crash the parent cursor initialization. 5148 if ( 5149 len(cursor_values) == 1 5150 and isinstance(cursor_values[0], (str, int, float)) 5151 and not isinstance(cursor_values[0], bool) 5152 ): 5153 incremental_sync_model: Union[ 5154 DatetimeBasedCursorModel, 5155 IncrementingCountCursorModel, 5156 ] = ( 5157 model.stream.incremental_sync # type: ignore # if we are there, it is because there is incremental_dependency and therefore there is an incremental_sync on the parent stream 5158 if isinstance(model.stream, DeclarativeStreamModel) 5159 else self._get_state_delegating_stream_model( 5160 model.stream, parent_state=parent_state 5161 ).incremental_sync 5162 ) 5163 cursor_field = InterpolatedString.create( 5164 incremental_sync_model.cursor_field, 5165 parameters=incremental_sync_model.parameters or {}, 5166 ).eval(config) 5167 extracted_parent_state = AirbyteStateMessage( 5168 type=AirbyteStateType.STREAM, 5169 stream=AirbyteStreamState( 5170 stream_descriptor=StreamDescriptor( 5171 name=parent_stream_name, namespace=None 5172 ), 5173 stream_state=AirbyteStateBlob({cursor_field: cursor_values[0]}), 5174 ), 5175 ) 5176 return ConnectorStateManager([extracted_parent_state] if extracted_parent_state else []) 5177 5178 return ConnectorStateManager([]) 5179 5180 @staticmethod 5181 def create_wait_time_from_header( 5182 model: WaitTimeFromHeaderModel, config: Config, **kwargs: Any 5183 ) -> WaitTimeFromHeaderBackoffStrategy: 5184 return WaitTimeFromHeaderBackoffStrategy( 5185 header=model.header, 5186 parameters=model.parameters or {}, 5187 config=config, 5188 regex=model.regex, 5189 max_waiting_time_in_seconds=model.max_waiting_time_in_seconds, 5190 ) 5191 5192 @staticmethod 5193 def create_wait_until_time_from_header( 5194 model: WaitUntilTimeFromHeaderModel, config: Config, **kwargs: Any 5195 ) -> WaitUntilTimeFromHeaderBackoffStrategy: 5196 return WaitUntilTimeFromHeaderBackoffStrategy( 5197 header=model.header, 5198 parameters=model.parameters or {}, 5199 config=config, 5200 min_wait=model.min_wait, 5201 regex=model.regex, 5202 max_waiting_time_in_seconds=model.max_waiting_time_in_seconds, 5203 ) 5204 5205 def get_message_repository(self) -> MessageRepository: 5206 return self._message_repository 5207 5208 def _evaluate_log_level(self, emit_connector_builder_messages: bool) -> Level: 5209 return Level.DEBUG if emit_connector_builder_messages else Level.INFO 5210 5211 @staticmethod 5212 def create_components_mapping_definition( 5213 model: ComponentMappingDefinitionModel, config: Config, **kwargs: Any 5214 ) -> ComponentMappingDefinition: 5215 interpolated_value = InterpolatedString.create( 5216 model.value, parameters=model.parameters or {} 5217 ) 5218 field_path = [ 5219 InterpolatedString.create(path, parameters=model.parameters or {}) 5220 for path in model.field_path 5221 ] 5222 return ComponentMappingDefinition( 5223 field_path=field_path, # type: ignore[arg-type] # field_path can be str and InterpolatedString 5224 value=interpolated_value, 5225 value_type=ModelToComponentFactory._json_schema_type_name_to_type(model.value_type), 5226 create_or_update=model.create_or_update, 5227 condition=model.condition, 5228 parameters=model.parameters or {}, 5229 ) 5230 5231 def create_http_components_resolver( 5232 self, model: HttpComponentsResolverModel, config: Config, stream_name: Optional[str] = None 5233 ) -> Any: 5234 partition_router = self._build_stream_slicer_from_partition_router(model.retriever, config) 5235 retriever = self._create_component_from_model( 5236 model=model.retriever, 5237 config=config, 5238 name=f"{stream_name if stream_name else '__http_components_resolver'}", 5239 primary_key=None, 5240 stream_slicer=partition_router, 5241 partition_router=partition_router, 5242 transformations=[], 5243 ) 5244 5245 components_mapping = [] 5246 for component_mapping_definition_model in model.components_mapping: 5247 if component_mapping_definition_model.condition: 5248 raise ValueError("`condition` is only supported for `ConfigComponentsResolver`") 5249 components_mapping.append( 5250 self._create_component_from_model( 5251 model=component_mapping_definition_model, 5252 value_type=ModelToComponentFactory._json_schema_type_name_to_type( 5253 component_mapping_definition_model.value_type 5254 ), 5255 config=config, 5256 ) 5257 ) 5258 5259 return HttpComponentsResolver( 5260 retriever=retriever, 5261 # AsyncRetriever reads records only from the job slices its own slicer yields 5262 stream_slicer=retriever.stream_slicer 5263 if isinstance(retriever, AsyncRetriever) 5264 else partition_router, 5265 config=config, 5266 components_mapping=components_mapping, 5267 parameters=model.parameters or {}, 5268 ) 5269 5270 @staticmethod 5271 def create_stream_config( 5272 model: StreamConfigModel, config: Config, **kwargs: Any 5273 ) -> StreamConfig: 5274 model_configs_pointer: List[Union[InterpolatedString, str]] = ( 5275 [x for x in model.configs_pointer] if model.configs_pointer else [] 5276 ) 5277 5278 return StreamConfig( 5279 configs_pointer=model_configs_pointer, 5280 default_values=model.default_values, 5281 parameters=model.parameters or {}, 5282 ) 5283 5284 def create_config_components_resolver( 5285 self, 5286 model: ConfigComponentsResolverModel, 5287 config: Config, 5288 ) -> Any: 5289 model_stream_configs = ( 5290 model.stream_config if isinstance(model.stream_config, list) else [model.stream_config] 5291 ) 5292 5293 stream_configs = [ 5294 self._create_component_from_model( 5295 stream_config, config=config, parameters=model.parameters or {} 5296 ) 5297 for stream_config in model_stream_configs 5298 ] 5299 5300 components_mapping = [ 5301 self._create_component_from_model( 5302 model=components_mapping_definition_model, 5303 value_type=ModelToComponentFactory._json_schema_type_name_to_type( 5304 components_mapping_definition_model.value_type 5305 ), 5306 config=config, 5307 parameters=model.parameters, 5308 ) 5309 for components_mapping_definition_model in model.components_mapping 5310 ] 5311 5312 return ConfigComponentsResolver( 5313 stream_configs=stream_configs, 5314 config=config, 5315 components_mapping=components_mapping, 5316 parameters=model.parameters or {}, 5317 ) 5318 5319 def create_parametrized_components_resolver( 5320 self, 5321 model: ParametrizedComponentsResolverModel, 5322 config: Config, 5323 ) -> ParametrizedComponentsResolver: 5324 stream_parameters = StreamParametersDefinition( 5325 list_of_parameters_for_stream=model.stream_parameters.list_of_parameters_for_stream 5326 ) 5327 5328 components_mapping = [] 5329 for components_mapping_definition_model in model.components_mapping: 5330 if components_mapping_definition_model.condition: 5331 raise ValueError("`condition` is only supported for `ConfigComponentsResolver`") 5332 components_mapping.append( 5333 self._create_component_from_model( 5334 model=components_mapping_definition_model, 5335 value_type=ModelToComponentFactory._json_schema_type_name_to_type( 5336 components_mapping_definition_model.value_type 5337 ), 5338 config=config, 5339 ) 5340 ) 5341 return ParametrizedComponentsResolver( 5342 stream_parameters=stream_parameters, 5343 config=config, 5344 components_mapping=components_mapping, 5345 parameters=model.parameters or {}, 5346 ) 5347 5348 _UNSUPPORTED_DECODER_ERROR = ( 5349 "Specified decoder of {decoder_type} is not supported for pagination." 5350 "Please set as `JsonDecoder`, `XmlDecoder`, or a `CompositeRawDecoder` with an inner_parser of `JsonParser` or `GzipParser` instead." 5351 "If using `GzipParser`, please ensure that the lowest level inner_parser is a `JsonParser`." 5352 ) 5353 5354 def _is_supported_decoder_for_pagination(self, decoder: Decoder) -> bool: 5355 if isinstance(decoder, (JsonDecoder, XmlDecoder)): 5356 return True 5357 elif isinstance(decoder, CompositeRawDecoder): 5358 return self._is_supported_parser_for_pagination(decoder.parser) 5359 else: 5360 return False 5361 5362 def _is_supported_parser_for_pagination(self, parser: Parser) -> bool: 5363 if isinstance(parser, JsonParser): 5364 return True 5365 elif isinstance(parser, GzipParser): 5366 return isinstance(parser.inner_parser, JsonParser) 5367 else: 5368 return False 5369 5370 def create_http_api_budget( 5371 self, model: HTTPAPIBudgetModel, config: Config, **kwargs: Any 5372 ) -> HttpAPIBudget: 5373 policies = [ 5374 self._create_component_from_model(model=policy, config=config) 5375 for policy in model.policies 5376 ] 5377 5378 return HttpAPIBudget( 5379 policies=policies, 5380 ratelimit_reset_header=model.ratelimit_reset_header or "ratelimit-reset", 5381 ratelimit_remaining_header=model.ratelimit_remaining_header or "ratelimit-remaining", 5382 status_codes_for_ratelimit_hit=model.status_codes_for_ratelimit_hit or [429], 5383 ) 5384 5385 def create_fixed_window_call_rate_policy( 5386 self, model: FixedWindowCallRatePolicyModel, config: Config, **kwargs: Any 5387 ) -> FixedWindowCallRatePolicy: 5388 matchers = [ 5389 self._create_component_from_model(model=matcher, config=config) 5390 for matcher in model.matchers 5391 ] 5392 5393 # Set the initial reset timestamp to 10 days from now. 5394 # This value will be updated by the first request. 5395 return FixedWindowCallRatePolicy( 5396 next_reset_ts=datetime.datetime.now() + datetime.timedelta(days=10), 5397 period=parse_duration(model.period), 5398 call_limit=model.call_limit, 5399 matchers=matchers, 5400 ) 5401 5402 def create_file_uploader( 5403 self, model: FileUploaderModel, config: Config, **kwargs: Any 5404 ) -> FileUploader: 5405 self._reject_reduce_page_size_action(model.requester, "requester of a `file_uploader`") 5406 self._reject_split_request_window_action(model.requester, "requester of a `file_uploader`") 5407 name = "File Uploader" 5408 requester = self._create_component_from_model( 5409 model=model.requester, 5410 config=config, 5411 name=name, 5412 **kwargs, 5413 ) 5414 download_target_extractor = self._create_component_from_model( 5415 model=model.download_target_extractor, 5416 config=config, 5417 name=name, 5418 **kwargs, 5419 ) 5420 emit_connector_builder_messages = self._emit_connector_builder_messages 5421 file_uploader = DefaultFileUploader( 5422 requester=requester, 5423 download_target_extractor=download_target_extractor, 5424 config=config, 5425 file_writer=NoopFileWriter() 5426 if emit_connector_builder_messages 5427 else LocalFileSystemFileWriter(), 5428 parameters=model.parameters or {}, 5429 filename_extractor=model.filename_extractor if model.filename_extractor else None, 5430 ) 5431 5432 return ( 5433 ConnectorBuilderFileUploader(file_uploader) 5434 if emit_connector_builder_messages 5435 else file_uploader 5436 ) 5437 5438 def create_moving_window_call_rate_policy( 5439 self, model: MovingWindowCallRatePolicyModel, config: Config, **kwargs: Any 5440 ) -> MovingWindowCallRatePolicy: 5441 rates = [ 5442 self._create_component_from_model(model=rate, config=config) for rate in model.rates 5443 ] 5444 matchers = [ 5445 self._create_component_from_model(model=matcher, config=config) 5446 for matcher in model.matchers 5447 ] 5448 return MovingWindowCallRatePolicy( 5449 rates=rates, 5450 matchers=matchers, 5451 ) 5452 5453 def create_unlimited_call_rate_policy( 5454 self, model: UnlimitedCallRatePolicyModel, config: Config, **kwargs: Any 5455 ) -> UnlimitedCallRatePolicy: 5456 matchers = [ 5457 self._create_component_from_model(model=matcher, config=config) 5458 for matcher in model.matchers 5459 ] 5460 5461 return UnlimitedCallRatePolicy( 5462 matchers=matchers, 5463 ) 5464 5465 def create_rate(self, model: RateModel, config: Config, **kwargs: Any) -> Rate: 5466 interpolated_limit = InterpolatedString.create(str(model.limit), parameters={}) 5467 return Rate( 5468 limit=int(interpolated_limit.eval(config=config)), 5469 interval=parse_duration(model.interval), 5470 ) 5471 5472 def create_http_request_matcher( 5473 self, model: HttpRequestRegexMatcherModel, config: Config, **kwargs: Any 5474 ) -> HttpRequestRegexMatcher: 5475 weight = model.weight 5476 if weight is not None: 5477 if isinstance(weight, str): 5478 weight = int(InterpolatedString.create(weight, parameters={}).eval(config)) 5479 else: 5480 weight = int(weight) 5481 if weight < 1: 5482 raise ValueError(f"weight must be >= 1, got {weight}") 5483 return HttpRequestRegexMatcher( 5484 method=model.method, 5485 url_base=model.url_base, 5486 url_path_pattern=model.url_path_pattern, 5487 params=model.params, 5488 headers=model.headers, 5489 weight=weight, 5490 ) 5491 5492 def create_rate_limited_multiple_token_authenticator( 5493 self, 5494 model: RateLimitedMultipleTokenAuthenticatorModel, 5495 config: Config, 5496 **kwargs: Any, 5497 ) -> RateLimitedMultipleTokenAuthenticator: 5498 if isinstance(model.tokens, str): 5499 tokens_value = InterpolatedString.create(model.tokens, parameters={}).eval(config) 5500 delimiter = model.token_delimiter or "," 5501 tokens = [ 5502 token.strip() for token in str(tokens_value).split(delimiter) if token.strip() 5503 ] 5504 else: 5505 tokens = [ 5506 token_value 5507 for token in model.tokens 5508 if ( 5509 token_value := str( 5510 InterpolatedString.create(token, parameters={}).eval(config) 5511 ).strip() 5512 ) 5513 ] 5514 5515 quota_specs = [ 5516 { 5517 "name": quota_model.name, 5518 "remaining_path": quota_model.remaining_path, 5519 "reset_path": quota_model.reset_path, 5520 "limit_path": quota_model.limit_path, 5521 "remaining_header": quota_model.remaining_header, 5522 "reset_header": quota_model.reset_header, 5523 "limit_header": quota_model.limit_header, 5524 # Normalize the same way as the runtime TokenQuota below, so an omitted field 5525 # and an explicit `[]` key identically and keep sharing one set of counters. 5526 "exhaustion_status_codes": quota_model.exhaustion_status_codes or [], 5527 "matchers": [ 5528 { 5529 "method": matcher_model.method, 5530 "url_base": matcher_model.url_base, 5531 "url_path_pattern": matcher_model.url_path_pattern, 5532 "params": matcher_model.params, 5533 "headers": matcher_model.headers, 5534 "weight": matcher_model.weight, 5535 } 5536 for matcher_model in quota_model.matchers or [] 5537 ], 5538 } 5539 for quota_model in model.quotas 5540 ] 5541 5542 quota_status_url = str( 5543 InterpolatedString.create(model.quota_status_source.url, parameters={}).eval(config) 5544 ) 5545 quota_status_http_method = ( 5546 model.quota_status_source.http_method.value 5547 if model.quota_status_source.http_method 5548 else "GET" 5549 ) 5550 quota_status_headers = { 5551 key: str(InterpolatedString.create(value, parameters={}).eval(config)) 5552 for key, value in (model.quota_status_source.request_headers or {}).items() 5553 } 5554 # Normalize the same way as the quota specs above, so an omitted field and an explicit 5555 # `[]` key identically and keep sharing one set of counters. Deduplicated as well as 5556 # sorted, because the runtime turns this into a set: without it `[404]` and `[404, 404]` 5557 # would key differently and stop sharing counters while behaving identically. 5558 quota_status_unavailable_status_codes = sorted( 5559 set(model.quota_status_source.unavailable_status_codes or []) 5560 ) 5561 auth_method = model.auth_method or "Bearer" 5562 header = model.header or "Authorization" 5563 max_wait_time_str = str( 5564 InterpolatedString.create(model.max_wait_time or "PT2H", parameters={}).eval(config) 5565 ) 5566 max_wait_time = parse_duration(max_wait_time_str) 5567 if not isinstance(max_wait_time, datetime.timedelta): 5568 raise ValueError( 5569 f"max_wait_time must be a fixed-length ISO 8601 duration (e.g. 'PT2H'); " 5570 f"calendar-unit durations like '{max_wait_time_str}' are not supported" 5571 ) 5572 budget_reserve_fraction = ( 5573 model.budget_reserve_fraction if model.budget_reserve_fraction is not None else 0.1 5574 ) 5575 budget_min_reserve = ( 5576 model.budget_min_reserve if model.budget_min_reserve is not None else 50 5577 ) 5578 5579 # Reuse the same instance for identical definitions so that all streams share the 5580 # same token quota counters (similar to how api_budget is shared). The key is built 5581 # from the resolved constructor arguments rather than the raw model so that 5582 # stream-specific `$parameters` propagated onto the model (and its nested components) 5583 # cannot break instance sharing. 5584 cache_key = json.dumps( 5585 { 5586 "tokens": tokens, 5587 "quotas": quota_specs, 5588 "quota_status_url": quota_status_url, 5589 "quota_status_http_method": quota_status_http_method, 5590 "quota_status_headers": quota_status_headers, 5591 "quota_status_unavailable_status_codes": quota_status_unavailable_status_codes, 5592 "auth_method": auth_method, 5593 "header": header, 5594 "max_wait_time": max_wait_time.total_seconds(), 5595 "budget_reserve_fraction": budget_reserve_fraction, 5596 "budget_min_reserve": budget_min_reserve, 5597 }, 5598 sort_keys=True, 5599 ) 5600 if cache_key in self._rate_limited_authenticators: 5601 return self._rate_limited_authenticators[cache_key] 5602 5603 quotas = [ 5604 TokenQuota( 5605 name=quota_model.name, 5606 remaining_path=quota_model.remaining_path, 5607 reset_path=quota_model.reset_path, 5608 limit_path=quota_model.limit_path, 5609 remaining_header=quota_model.remaining_header, 5610 reset_header=quota_model.reset_header, 5611 limit_header=quota_model.limit_header, 5612 exhaustion_status_codes=quota_model.exhaustion_status_codes or [], 5613 matchers=[ 5614 self.create_http_request_matcher(matcher_model, config) 5615 for matcher_model in quota_model.matchers or [] 5616 ], 5617 ) 5618 for quota_model in model.quotas 5619 ] 5620 5621 authenticator = RateLimitedMultipleTokenAuthenticator( 5622 tokens=tokens, 5623 quotas=quotas, 5624 quota_status_url=quota_status_url, 5625 quota_status_http_method=quota_status_http_method, 5626 quota_status_headers=quota_status_headers, 5627 quota_status_unavailable_status_codes=quota_status_unavailable_status_codes, 5628 auth_method=auth_method, 5629 header=header, 5630 max_wait_time=max_wait_time, 5631 budget_reserve_fraction=budget_reserve_fraction, 5632 budget_min_reserve=budget_min_reserve, 5633 ) 5634 self._rate_limited_authenticators[cache_key] = authenticator 5635 return authenticator 5636 5637 def set_api_budget(self, component_definition: ComponentDefinition, config: Config) -> None: 5638 self._api_budget = self.create_component( 5639 model_type=HTTPAPIBudgetModel, component_definition=component_definition, config=config 5640 ) 5641 5642 def create_grouping_partition_router( 5643 self, 5644 model: GroupingPartitionRouterModel, 5645 config: Config, 5646 *, 5647 stream_name: str, 5648 **kwargs: Any, 5649 ) -> GroupingPartitionRouter: 5650 underlying_router = self._create_component_from_model( 5651 model=model.underlying_partition_router, 5652 config=config, 5653 stream_name=stream_name, 5654 **kwargs, 5655 ) 5656 if model.group_size < 1: 5657 raise ValueError(f"Group size must be greater than 0, got {model.group_size}") 5658 5659 # Request options in underlying partition routers are not supported for GroupingPartitionRouter 5660 # because they are specific to individual partitions and cannot be aggregated or handled 5661 # when grouping, potentially leading to incorrect API calls. Any request customization 5662 # should be managed at the stream level through the requester's configuration. 5663 if isinstance(underlying_router, SubstreamPartitionRouter): 5664 if any( 5665 parent_config.request_option 5666 for parent_config in underlying_router.parent_stream_configs 5667 ): 5668 raise ValueError("Request options are not supported for GroupingPartitionRouter.") 5669 5670 if isinstance(underlying_router, ListPartitionRouter): 5671 if underlying_router.request_option: 5672 raise ValueError("Request options are not supported for GroupingPartitionRouter.") 5673 5674 return GroupingPartitionRouter( 5675 group_size=model.group_size, 5676 underlying_partition_router=underlying_router, 5677 deduplicate=model.deduplicate if model.deduplicate is not None else True, 5678 config=config, 5679 ) 5680 5681 def create_union_partition_router( 5682 self, 5683 model: UnionPartitionRouterModel, 5684 config: Config, 5685 *, 5686 stream_name: str, 5687 **kwargs: Any, 5688 ) -> UnionPartitionRouter: 5689 # The schema enforces minItems: 2 for manifests; this guard covers construction paths 5690 # that bypass JSON-schema validation (the generated model carries no min_items constraint). 5691 if len(model.partition_routers) < 2: 5692 raise ValueError( 5693 f"UnionPartitionRouter for stream {stream_name} needs at least 2 child partition routers" 5694 ) 5695 5696 partition_routers = [ 5697 self._create_component_from_model( 5698 model=child, 5699 config=config, 5700 stream_name=stream_name, 5701 **kwargs, 5702 ) 5703 for child in model.partition_routers 5704 ] 5705 5706 # partition_field depends only on config/parameters, so it is evaluated once at build 5707 # time; the runtime component always receives a plain string. 5708 partition_field = InterpolatedString.create( 5709 model.partition_field, parameters=model.parameters or {} 5710 ).eval(config) 5711 5712 # Fail fast at build time when a built-in child router is statically known to emit a 5713 # partition field different from the union's. CustomPartitionRouter children are opaque 5714 # and can only be validated at runtime. 5715 for child_model in model.partition_routers: 5716 child_partition_fields: List[str] = [] 5717 if isinstance(child_model, ListPartitionRouterModel): 5718 child_partition_fields.append( 5719 InterpolatedString.create( 5720 child_model.cursor_field, parameters=child_model.parameters or {} 5721 ).eval(config) 5722 ) 5723 elif isinstance(child_model, SubstreamPartitionRouterModel): 5724 for parent_stream_config in child_model.parent_stream_configs: 5725 child_partition_fields.append( 5726 InterpolatedString.create( 5727 parent_stream_config.partition_field, 5728 parameters=parent_stream_config.parameters 5729 or child_model.parameters 5730 or {}, 5731 ).eval(config) 5732 ) 5733 elif isinstance(child_model, UnionPartitionRouterModel): 5734 child_partition_fields.append( 5735 InterpolatedString.create( 5736 child_model.partition_field, parameters=child_model.parameters or {} 5737 ).eval(config) 5738 ) 5739 for child_partition_field in child_partition_fields: 5740 if child_partition_field != partition_field: 5741 raise ValueError( 5742 f"UnionPartitionRouter expects all child partition routers to emit the " 5743 f"partition field '{partition_field}', but a " 5744 f"{child_model.type} child emits '{child_partition_field}'." 5745 ) 5746 5747 # A union slice comes from exactly one child partition router, so request options 5748 # declared on children cannot be applied consistently to requests built from the 5749 # normalized union slices. Partition values should be consumed via interpolation 5750 # (e.g. stream_partition) instead. Note that this validation only covers built-in 5751 # router types; CustomPartitionRouter children are opaque, so any request options 5752 # they implement internally cannot be detected or rejected here. 5753 for router in partition_routers: 5754 if isinstance(router, SubstreamPartitionRouter): 5755 if any( 5756 parent_config.request_option for parent_config in router.parent_stream_configs 5757 ): 5758 raise ValueError("Request options are not supported for UnionPartitionRouter.") 5759 if isinstance(router, ListPartitionRouter) and router.request_option: 5760 raise ValueError("Request options are not supported for UnionPartitionRouter.") 5761 5762 return UnionPartitionRouter( 5763 partition_routers=partition_routers, 5764 partition_field=partition_field, 5765 parameters=model.parameters or {}, 5766 ) 5767 5768 def _ensure_query_properties_to_model( 5769 self, requester: Union[HttpRequesterModel, CustomRequesterModel] 5770 ) -> None: 5771 """ 5772 For some reason, it seems like CustomRequesterModel request_parameters stays as dictionaries which means that 5773 the other conditions relying on it being QueryPropertiesModel instead of a dict fail. Here, we migrate them to 5774 proper model. 5775 """ 5776 if not hasattr(requester, "request_parameters"): 5777 return 5778 5779 request_parameters = requester.request_parameters 5780 if request_parameters and isinstance(request_parameters, Dict): 5781 for request_parameter_key in request_parameters.keys(): 5782 request_parameter = request_parameters[request_parameter_key] 5783 if ( 5784 isinstance(request_parameter, Dict) 5785 and request_parameter.get("type") == "QueryProperties" 5786 ): 5787 request_parameters[request_parameter_key] = QueryPropertiesModel.parse_obj( 5788 request_parameter 5789 ) 5790 5791 def _get_catalog_defined_cursor_field( 5792 self, stream_name: str, allow_catalog_defined_cursor_field: bool 5793 ) -> Optional[CursorField]: 5794 if not allow_catalog_defined_cursor_field: 5795 return None 5796 5797 configured_stream = self._stream_name_to_configured_stream.get(stream_name) 5798 5799 # Depending on the operation is being performed, there may not be a configured stream yet. In this 5800 # case we return None which will then use the default cursor field defined on the cursor model. 5801 # We also treat cursor_field: [""] (list with empty string) as no cursor field, since this can 5802 # occur when the platform serializes "no cursor configured" streams incorrectly. 5803 if ( 5804 not configured_stream 5805 or not configured_stream.cursor_field 5806 or not configured_stream.cursor_field[0] 5807 ): 5808 return None 5809 elif len(configured_stream.cursor_field) > 1: 5810 raise ValueError( 5811 f"The `{stream_name}` stream does not support nested cursor_field. Please specify only a single cursor_field for the stream in the configured catalog." 5812 ) 5813 else: 5814 return CursorField( 5815 cursor_field_key=configured_stream.cursor_field[0], 5816 supports_catalog_defined_cursor_field=allow_catalog_defined_cursor_field, 5817 )
ModelToComponentFactory( limit_pages_fetched_per_slice: Optional[int] = None, limit_slices_fetched: Optional[int] = None, emit_connector_builder_messages: bool = False, disable_retries: bool = False, disable_cache: bool = False, message_repository: Optional[airbyte_cdk.MessageRepository] = None, connector_state_manager: Optional[airbyte_cdk.ConnectorStateManager] = None, max_concurrent_async_job_count: Optional[int] = None, configured_catalog: Optional[airbyte_protocol_dataclasses.models.airbyte_protocol.ConfiguredAirbyteCatalog] = None, api_budget: Optional[airbyte_cdk.sources.streams.call_rate.APIBudget] = None, rate_limited_authenticators: Optional[Dict[str, airbyte_cdk.sources.declarative.auth.RateLimitedMultipleTokenAuthenticator]] = None, custom_components_trusted: bool = True)
755 def __init__( 756 self, 757 limit_pages_fetched_per_slice: Optional[int] = None, 758 limit_slices_fetched: Optional[int] = None, 759 emit_connector_builder_messages: bool = False, 760 disable_retries: bool = False, 761 disable_cache: bool = False, 762 message_repository: Optional[MessageRepository] = None, 763 connector_state_manager: Optional[ConnectorStateManager] = None, 764 max_concurrent_async_job_count: Optional[int] = None, 765 configured_catalog: Optional[ConfiguredAirbyteCatalog] = None, 766 api_budget: Optional[APIBudget] = None, 767 rate_limited_authenticators: Optional[ 768 Dict[str, RateLimitedMultipleTokenAuthenticator] 769 ] = None, 770 custom_components_trusted: bool = True, 771 ): 772 self._init_mappings() 773 self._custom_components_trusted = custom_components_trusted 774 self._limit_pages_fetched_per_slice = limit_pages_fetched_per_slice 775 self._limit_slices_fetched = limit_slices_fetched 776 self._emit_connector_builder_messages = emit_connector_builder_messages 777 self._disable_retries = disable_retries 778 self._disable_cache = disable_cache 779 self._message_repository = message_repository or InMemoryMessageRepository( 780 self._evaluate_log_level(emit_connector_builder_messages) 781 ) 782 self._stream_name_to_configured_stream = self._create_stream_name_to_configured_stream( 783 configured_catalog 784 ) 785 self._connector_state_manager = connector_state_manager or ConnectorStateManager() 786 self._api_budget: Optional[Union[APIBudget]] = api_budget 787 # Shared instances so all streams see the same token quota counters (like api_budget) 788 self._rate_limited_authenticators: Dict[str, RateLimitedMultipleTokenAuthenticator] = ( 789 rate_limited_authenticators if rate_limited_authenticators is not None else {} 790 ) 791 self._job_tracker: JobTracker = JobTracker(max_concurrent_async_job_count or 1) 792 # placeholder for deprecation warnings 793 self._collected_deprecation_logs: List[ConnectorBuilderLogMessage] = []
def
create_component( self, model_type: Type[pydantic.v1.main.BaseModel], component_definition: Mapping[str, Any], config: Mapping[str, Any], **kwargs: Any) -> Any:
922 def create_component( 923 self, 924 model_type: Type[BaseModel], 925 component_definition: ComponentDefinition, 926 config: Config, 927 **kwargs: Any, 928 ) -> Any: 929 """ 930 Takes a given Pydantic model type and Mapping representing a component definition and creates a declarative component and 931 subcomponents which will be used at runtime. This is done by first parsing the mapping into a Pydantic model and then creating 932 creating declarative components from that model. 933 934 :param model_type: The type of declarative component that is being initialized 935 :param component_definition: The mapping that represents a declarative component 936 :param config: The connector config that is provided by the customer 937 :return: The declarative component to be used at runtime 938 """ 939 940 component_type = component_definition.get("type") 941 if component_definition.get("type") != model_type.__name__: 942 raise ValueError( 943 f"Expected manifest component of type {model_type.__name__}, but received {component_type} instead" 944 ) 945 946 declarative_component_model = model_type.parse_obj(component_definition) 947 948 if not isinstance(declarative_component_model, model_type): 949 raise ValueError( 950 f"Expected {model_type.__name__} component, but received {declarative_component_model.__class__.__name__}" 951 ) 952 953 return self._create_component_from_model( 954 model=declarative_component_model, config=config, **kwargs 955 )
Takes a given Pydantic model type and Mapping representing a component definition and creates a declarative component and subcomponents which will be used at runtime. This is done by first parsing the mapping into a Pydantic model and then creating creating declarative components from that model.
Parameters
- model_type: The type of declarative component that is being initialized
- component_definition: The mapping that represents a declarative component
- config: The connector config that is provided by the customer
Returns
The declarative component to be used at runtime
972 def get_model_deprecations(self) -> List[ConnectorBuilderLogMessage]: 973 """ 974 Returns the deprecation warnings that were collected during the creation of components. 975 """ 976 return self._collected_deprecation_logs
Returns the deprecation warnings that were collected during the creation of components.
def
create_config_migration( self, model: airbyte_cdk.sources.declarative.models.declarative_component_schema.ConfigMigration, config: Mapping[str, Any]) -> airbyte_cdk.sources.declarative.spec.ConfigMigration:
993 def create_config_migration( 994 self, model: ConfigMigrationModel, config: Config 995 ) -> ConfigMigration: 996 transformations: List[ConfigTransformation] = [ 997 self._create_component_from_model(transformation, config) 998 for transformation in model.transformations 999 ] 1000 1001 return ConfigMigration( 1002 description=model.description, 1003 transformations=transformations, 1004 )
def
create_config_add_fields( self, model: airbyte_cdk.sources.declarative.models.declarative_component_schema.ConfigAddFields, config: Mapping[str, Any], **kwargs: Any) -> airbyte_cdk.sources.declarative.transformations.config_transformations.ConfigAddFields:
1006 def create_config_add_fields( 1007 self, model: ConfigAddFieldsModel, config: Config, **kwargs: Any 1008 ) -> ConfigAddFields: 1009 fields = [self._create_component_from_model(field, config) for field in model.fields] 1010 return ConfigAddFields( 1011 fields=fields, 1012 condition=model.condition or "", 1013 )
@staticmethod
def
create_config_remove_fields( model: airbyte_cdk.sources.declarative.models.declarative_component_schema.ConfigRemoveFields, config: Mapping[str, Any], **kwargs: Any) -> airbyte_cdk.sources.declarative.transformations.config_transformations.ConfigRemoveFields:
@staticmethod
def
create_config_remap_field( model: airbyte_cdk.sources.declarative.models.declarative_component_schema.ConfigRemapField, config: Mapping[str, Any], **kwargs: Any) -> airbyte_cdk.sources.declarative.transformations.config_transformations.ConfigRemapField:
1024 @staticmethod 1025 def create_config_remap_field( 1026 model: ConfigRemapFieldModel, config: Config, **kwargs: Any 1027 ) -> ConfigRemapField: 1028 mapping = cast(Mapping[str, Any], model.map) 1029 return ConfigRemapField( 1030 map=mapping, 1031 field_path=model.field_path, 1032 config=config, 1033 )
def
create_dpath_validator( self, model: airbyte_cdk.sources.declarative.models.declarative_component_schema.DpathValidator, config: Mapping[str, Any]) -> airbyte_cdk.sources.declarative.validators.DpathValidator:
def
create_predicate_validator( self, model: airbyte_cdk.sources.declarative.models.declarative_component_schema.PredicateValidator, config: Mapping[str, Any]) -> airbyte_cdk.sources.declarative.validators.PredicateValidator:
1043 def create_predicate_validator( 1044 self, model: PredicateValidatorModel, config: Config 1045 ) -> PredicateValidator: 1046 strategy = self._create_component_from_model(model.validation_strategy, config) 1047 1048 return PredicateValidator( 1049 value=model.value, 1050 strategy=strategy, 1051 )
@staticmethod
def
create_validate_adheres_to_schema( model: airbyte_cdk.sources.declarative.models.declarative_component_schema.ValidateAdheresToSchema, config: Mapping[str, Any], **kwargs: Any) -> airbyte_cdk.sources.declarative.validators.ValidateAdheresToSchema:
1053 @staticmethod 1054 def create_validate_adheres_to_schema( 1055 model: ValidateAdheresToSchemaModel, config: Config, **kwargs: Any 1056 ) -> ValidateAdheresToSchema: 1057 base_schema = cast(Mapping[str, Any], model.base_schema) 1058 return ValidateAdheresToSchema( 1059 schema=base_schema, 1060 )
@staticmethod
def
create_added_field_definition( model: airbyte_cdk.sources.declarative.models.declarative_component_schema.AddedFieldDefinition, config: Mapping[str, Any], **kwargs: Any) -> airbyte_cdk.AddedFieldDefinition:
1062 @staticmethod 1063 def create_added_field_definition( 1064 model: AddedFieldDefinitionModel, config: Config, **kwargs: Any 1065 ) -> AddedFieldDefinition: 1066 interpolated_value = InterpolatedString.create( 1067 model.value, parameters=model.parameters or {} 1068 ) 1069 return AddedFieldDefinition( 1070 path=model.path, 1071 value=interpolated_value, 1072 value_type=ModelToComponentFactory._json_schema_type_name_to_type(model.value_type), 1073 parameters=model.parameters or {}, 1074 )
def
create_add_fields( self, model: airbyte_cdk.sources.declarative.models.declarative_component_schema.AddFields, config: Mapping[str, Any], **kwargs: Any) -> airbyte_cdk.AddFields:
1076 def create_add_fields(self, model: AddFieldsModel, config: Config, **kwargs: Any) -> AddFields: 1077 added_field_definitions = [ 1078 self._create_component_from_model( 1079 model=added_field_definition_model, 1080 value_type=ModelToComponentFactory._json_schema_type_name_to_type( 1081 added_field_definition_model.value_type 1082 ), 1083 config=config, 1084 ) 1085 for added_field_definition_model in model.fields 1086 ] 1087 return AddFields( 1088 fields=added_field_definitions, 1089 condition=model.condition or "", 1090 parameters=model.parameters or {}, 1091 )
def
create_keys_to_lower_transformation( self, model: airbyte_cdk.sources.declarative.models.declarative_component_schema.KeysToLower, config: Mapping[str, Any], **kwargs: Any) -> airbyte_cdk.sources.declarative.transformations.keys_to_lower_transformation.KeysToLowerTransformation:
def
create_keys_to_snake_transformation( self, model: airbyte_cdk.sources.declarative.models.declarative_component_schema.KeysToSnakeCase, config: Mapping[str, Any], **kwargs: Any) -> airbyte_cdk.sources.declarative.transformations.keys_to_snake_transformation.KeysToSnakeCaseTransformation:
def
create_keys_replace_transformation( self, model: airbyte_cdk.sources.declarative.models.declarative_component_schema.KeysReplace, config: Mapping[str, Any], **kwargs: Any) -> airbyte_cdk.sources.declarative.transformations.keys_replace_transformation.KeysReplaceTransformation:
def
create_flatten_fields( self, model: airbyte_cdk.sources.declarative.models.declarative_component_schema.FlattenFields, config: Mapping[str, Any], **kwargs: Any) -> airbyte_cdk.sources.declarative.transformations.flatten_fields.FlattenFields:
def
create_dpath_flatten_fields( self, model: airbyte_cdk.sources.declarative.models.declarative_component_schema.DpathFlattenFields, config: Mapping[str, Any], **kwargs: Any) -> airbyte_cdk.sources.declarative.transformations.dpath_flatten_fields.DpathFlattenFields:
1117 def create_dpath_flatten_fields( 1118 self, model: DpathFlattenFieldsModel, config: Config, **kwargs: Any 1119 ) -> DpathFlattenFields: 1120 model_field_path: List[Union[InterpolatedString, str]] = [x for x in model.field_path] 1121 key_transformation = ( 1122 KeyTransformation( 1123 config=config, 1124 prefix=model.key_transformation.prefix, 1125 suffix=model.key_transformation.suffix, 1126 parameters=model.parameters or {}, 1127 ) 1128 if model.key_transformation is not None 1129 else None 1130 ) 1131 return DpathFlattenFields( 1132 config=config, 1133 field_path=model_field_path, 1134 delete_origin_value=model.delete_origin_value 1135 if model.delete_origin_value is not None 1136 else False, 1137 replace_record=model.replace_record if model.replace_record is not None else False, 1138 key_transformation=key_transformation, 1139 parameters=model.parameters or {}, 1140 )
def
create_api_key_authenticator( self, model: airbyte_cdk.sources.declarative.models.declarative_component_schema.ApiKeyAuthenticator, config: Mapping[str, Any], token_provider: Optional[airbyte_cdk.sources.declarative.auth.token_provider.TokenProvider] = None, **kwargs: Any) -> airbyte_cdk.ApiKeyAuthenticator:
1154 def create_api_key_authenticator( 1155 self, 1156 model: ApiKeyAuthenticatorModel, 1157 config: Config, 1158 token_provider: Optional[TokenProvider] = None, 1159 **kwargs: Any, 1160 ) -> ApiKeyAuthenticator: 1161 if model.inject_into is None and model.header is None: 1162 raise ValueError( 1163 "Expected either inject_into or header to be set for ApiKeyAuthenticator" 1164 ) 1165 1166 if model.inject_into is not None and model.header is not None: 1167 raise ValueError( 1168 "inject_into and header cannot be set both for ApiKeyAuthenticator - remove the deprecated header option" 1169 ) 1170 1171 if token_provider is not None and model.api_token != "": 1172 raise ValueError( 1173 "If token_provider is set, api_token is ignored and has to be set to empty string." 1174 ) 1175 1176 request_option = ( 1177 self._create_component_from_model( 1178 model.inject_into, config, parameters=model.parameters or {} 1179 ) 1180 if model.inject_into 1181 else RequestOption( 1182 inject_into=RequestOptionType.header, 1183 field_name=model.header or "", 1184 parameters=model.parameters or {}, 1185 ) 1186 ) 1187 1188 return ApiKeyAuthenticator( 1189 token_provider=( 1190 token_provider 1191 if token_provider is not None 1192 else InterpolatedStringTokenProvider( 1193 api_token=model.api_token or "", 1194 config=config, 1195 parameters=model.parameters or {}, 1196 ) 1197 ), 1198 request_option=request_option, 1199 config=config, 1200 parameters=model.parameters or {}, 1201 )
def
create_legacy_to_per_partition_state_migration( self, model: airbyte_cdk.sources.declarative.models.declarative_component_schema.LegacyToPerPartitionStateMigration, config: Mapping[str, Any], declarative_stream: airbyte_cdk.sources.declarative.models.declarative_component_schema.DeclarativeStream) -> airbyte_cdk.LegacyToPerPartitionStateMigration:
1203 def create_legacy_to_per_partition_state_migration( 1204 self, 1205 model: LegacyToPerPartitionStateMigrationModel, 1206 config: Mapping[str, Any], 1207 declarative_stream: DeclarativeStreamModel, 1208 ) -> LegacyToPerPartitionStateMigration: 1209 retriever = declarative_stream.retriever 1210 if not isinstance(retriever, (SimpleRetrieverModel, AsyncRetrieverModel)): 1211 raise ValueError( 1212 f"LegacyToPerPartitionStateMigrations can only be applied on a DeclarativeStream with a SimpleRetriever or AsyncRetriever. Got {type(retriever)}" 1213 ) 1214 partition_router = retriever.partition_router 1215 if not isinstance( 1216 partition_router, 1217 ( 1218 SubstreamPartitionRouterModel, 1219 CustomPartitionRouterModel, 1220 UnionPartitionRouterModel, 1221 ), 1222 ): 1223 raise ValueError( 1224 f"LegacyToPerPartitionStateMigrations can only be applied on a SimpleRetriever with a SubstreamPartitionRouter, UnionPartitionRouter or CustomPartitionRouter. Got {type(partition_router)}" 1225 ) 1226 if not isinstance(partition_router, UnionPartitionRouterModel) and not hasattr( 1227 partition_router, "parent_stream_configs" 1228 ): 1229 raise ValueError( 1230 "LegacyToPerPartitionStateMigrations can only be applied with a parent stream configuration." 1231 ) 1232 1233 if not hasattr(declarative_stream, "incremental_sync"): 1234 raise ValueError( 1235 "LegacyToPerPartitionStateMigrations can only be applied with an incremental_sync configuration." 1236 ) 1237 1238 return LegacyToPerPartitionStateMigration( 1239 partition_router, # type: ignore # was already checked above 1240 declarative_stream.incremental_sync, # type: ignore # was already checked. Migration can be applied only to incremental streams. 1241 config, 1242 declarative_stream.parameters, # type: ignore # different type is expected here Mapping[str, Any], got Dict[str, Any] 1243 )
def
create_session_token_authenticator( self, model: airbyte_cdk.sources.declarative.models.declarative_component_schema.SessionTokenAuthenticator, config: Mapping[str, Any], *, name: str, **kwargs: Any) -> Union[airbyte_cdk.ApiKeyAuthenticator, airbyte_cdk.BearerAuthenticator]:
1245 def create_session_token_authenticator( 1246 self, model: SessionTokenAuthenticatorModel, config: Config, *, name: str, **kwargs: Any 1247 ) -> Union[ApiKeyAuthenticator, BearerAuthenticator]: 1248 self._reject_reduce_page_size_action( 1249 model.login_requester, f"`login_requester` of the SessionTokenAuthenticator of {name}" 1250 ) 1251 self._reject_split_request_window_action( 1252 model.login_requester, f"`login_requester` of the SessionTokenAuthenticator of {name}" 1253 ) 1254 decoder = ( 1255 self._create_component_from_model(model=model.decoder, config=config) 1256 if model.decoder 1257 else JsonDecoder(parameters={}) 1258 ) 1259 login_requester = self._create_component_from_model( 1260 model=model.login_requester, 1261 config=config, 1262 name=f"{name}_login_requester", 1263 decoder=decoder, 1264 ) 1265 token_provider = SessionTokenProvider( 1266 login_requester=login_requester, 1267 session_token_path=model.session_token_path, 1268 expiration_duration=parse_duration(model.expiration_duration) 1269 if model.expiration_duration 1270 else None, 1271 parameters=model.parameters or {}, 1272 message_repository=self._message_repository, 1273 decoder=decoder, 1274 ) 1275 if model.request_authentication.type == "Bearer": 1276 return ModelToComponentFactory.create_bearer_authenticator( 1277 BearerAuthenticatorModel(type="BearerAuthenticator", api_token=""), # type: ignore # $parameters has a default value 1278 config, 1279 token_provider=token_provider, 1280 ) 1281 else: 1282 # Get the api_token template if specified, default to just the session token 1283 api_token_template = ( 1284 getattr(model.request_authentication, "api_token", None) or "{{ session_token }}" 1285 ) 1286 final_token_provider: TokenProvider = InterpolatedSessionTokenProvider( 1287 config=config, 1288 api_token=api_token_template, 1289 session_token_provider=token_provider, 1290 parameters=model.parameters or {}, 1291 ) 1292 return self.create_api_key_authenticator( 1293 ApiKeyAuthenticatorModel( 1294 type="ApiKeyAuthenticator", 1295 api_token="", 1296 inject_into=model.request_authentication.inject_into, 1297 ), # type: ignore # $parameters and headers default to None 1298 config=config, 1299 token_provider=final_token_provider, 1300 )
@staticmethod
def
create_basic_http_authenticator( model: airbyte_cdk.sources.declarative.models.declarative_component_schema.BasicHttpAuthenticator, config: Mapping[str, Any], **kwargs: Any) -> airbyte_cdk.BasicHttpAuthenticator:
1302 @staticmethod 1303 def create_basic_http_authenticator( 1304 model: BasicHttpAuthenticatorModel, config: Config, **kwargs: Any 1305 ) -> BasicHttpAuthenticator: 1306 return BasicHttpAuthenticator( 1307 password=model.password or "", 1308 username=model.username, 1309 config=config, 1310 parameters=model.parameters or {}, 1311 )
@staticmethod
def
create_bearer_authenticator( model: airbyte_cdk.sources.declarative.models.declarative_component_schema.BearerAuthenticator, config: Mapping[str, Any], token_provider: Optional[airbyte_cdk.sources.declarative.auth.token_provider.TokenProvider] = None, **kwargs: Any) -> airbyte_cdk.BearerAuthenticator:
1313 @staticmethod 1314 def create_bearer_authenticator( 1315 model: BearerAuthenticatorModel, 1316 config: Config, 1317 token_provider: Optional[TokenProvider] = None, 1318 **kwargs: Any, 1319 ) -> BearerAuthenticator: 1320 if token_provider is not None and model.api_token != "": 1321 raise ValueError( 1322 "If token_provider is set, api_token is ignored and has to be set to empty string." 1323 ) 1324 return BearerAuthenticator( 1325 token_provider=( 1326 token_provider 1327 if token_provider is not None 1328 else InterpolatedStringTokenProvider( 1329 api_token=model.api_token or "", 1330 config=config, 1331 parameters=model.parameters or {}, 1332 ) 1333 ), 1334 config=config, 1335 parameters=model.parameters or {}, 1336 )
@staticmethod
def
create_dynamic_stream_check_config( model: airbyte_cdk.sources.declarative.models.declarative_component_schema.DynamicStreamCheckConfig, config: Mapping[str, Any], **kwargs: Any) -> airbyte_cdk.sources.declarative.checks.DynamicStreamCheckConfig:
1338 @staticmethod 1339 def create_dynamic_stream_check_config( 1340 model: DynamicStreamCheckConfigModel, config: Config, **kwargs: Any 1341 ) -> DynamicStreamCheckConfig: 1342 return DynamicStreamCheckConfig( 1343 dynamic_stream_name=model.dynamic_stream_name, 1344 stream_count=model.stream_count, 1345 )
def
create_check_stream( self, model: airbyte_cdk.sources.declarative.models.declarative_component_schema.CheckStream, config: Mapping[str, Any], **kwargs: Any) -> airbyte_cdk.sources.declarative.checks.CheckStream:
1347 def create_check_stream( 1348 self, model: CheckStreamModel, config: Config, **kwargs: Any 1349 ) -> CheckStream: 1350 if model.dynamic_streams_check_configs is None and model.stream_names is None: 1351 raise ValueError( 1352 "Expected either stream_names or dynamic_streams_check_configs to be set for CheckStream" 1353 ) 1354 1355 dynamic_streams_check_configs = ( 1356 [ 1357 self._create_component_from_model(model=dynamic_stream_check_config, config=config) 1358 for dynamic_stream_check_config in model.dynamic_streams_check_configs 1359 ] 1360 if model.dynamic_streams_check_configs 1361 else [] 1362 ) 1363 1364 # `model.config_overrides` is deliberately not read here. The source applies it around the whole 1365 # check operation (`ConcurrentDeclarativeSource._config_overridden_for_check`), which is what makes 1366 # it work for every checker type rather than only this one. Do not wire it in a second time. 1367 return CheckStream( 1368 stream_names=model.stream_names or [], 1369 dynamic_streams_check_configs=dynamic_streams_check_configs, 1370 parameters={}, 1371 )
@staticmethod
def
create_check_dynamic_stream( model: airbyte_cdk.sources.declarative.models.declarative_component_schema.CheckDynamicStream, config: Mapping[str, Any], **kwargs: Any) -> airbyte_cdk.sources.declarative.checks.CheckDynamicStream:
1373 @staticmethod 1374 def create_check_dynamic_stream( 1375 model: CheckDynamicStreamModel, config: Config, **kwargs: Any 1376 ) -> CheckDynamicStream: 1377 assert model.use_check_availability is not None # for mypy 1378 1379 use_check_availability = model.use_check_availability 1380 1381 # See `create_check_stream`: `model.config_overrides` is applied by the source, not here. 1382 return CheckDynamicStream( 1383 stream_count=model.stream_count, 1384 use_check_availability=use_check_availability, 1385 parameters={}, 1386 )
def
create_composite_error_handler( self, model: airbyte_cdk.sources.declarative.models.declarative_component_schema.CompositeErrorHandler, config: Mapping[str, Any], **kwargs: Any) -> airbyte_cdk.sources.declarative.requesters.error_handlers.CompositeErrorHandler:
1388 def create_composite_error_handler( 1389 self, model: CompositeErrorHandlerModel, config: Config, **kwargs: Any 1390 ) -> CompositeErrorHandler: 1391 error_handlers = [ 1392 self._create_component_from_model(model=error_handler_model, config=config) 1393 for error_handler_model in model.error_handlers 1394 ] 1395 return CompositeErrorHandler( 1396 error_handlers=error_handlers, parameters=model.parameters or {} 1397 )
@staticmethod
def
create_concurrency_level( model: airbyte_cdk.sources.declarative.models.declarative_component_schema.ConcurrencyLevel, config: Mapping[str, Any], **kwargs: Any) -> airbyte_cdk.sources.declarative.concurrency_level.ConcurrencyLevel:
1399 @staticmethod 1400 def create_concurrency_level( 1401 model: ConcurrencyLevelModel, config: Config, **kwargs: Any 1402 ) -> ConcurrencyLevel: 1403 return ConcurrencyLevel( 1404 default_concurrency=model.default_concurrency, 1405 max_concurrency=model.max_concurrency, 1406 config=config, 1407 parameters={}, 1408 )
@staticmethod
def
apply_stream_state_migrations( stream_state_migrations: Optional[List[Any]], stream_state: MutableMapping[str, Any]) -> MutableMapping[str, Any]:
1410 @staticmethod 1411 def apply_stream_state_migrations( 1412 stream_state_migrations: List[Any] | None, stream_state: MutableMapping[str, Any] 1413 ) -> MutableMapping[str, Any]: 1414 if stream_state_migrations: 1415 for state_migration in stream_state_migrations: 1416 if state_migration.should_migrate(stream_state): 1417 # The state variable is expected to be mutable but the migrate method returns an immutable mapping. 1418 stream_state = dict(state_migration.migrate(stream_state)) 1419 return stream_state
def
create_concurrent_cursor_from_datetime_based_cursor( self, model_type: Type[pydantic.v1.main.BaseModel], component_definition: Mapping[str, Any], stream_name: str, stream_namespace: Optional[str], stream_state: MutableMapping[str, Any], config: Mapping[str, Any], message_repository: Optional[airbyte_cdk.MessageRepository] = None, runtime_lookback_window: Optional[datetime.timedelta] = None, **kwargs: Any) -> airbyte_cdk.ConcurrentCursor:
1421 def create_concurrent_cursor_from_datetime_based_cursor( 1422 self, 1423 model_type: Type[BaseModel], 1424 component_definition: ComponentDefinition, 1425 stream_name: str, 1426 stream_namespace: Optional[str], 1427 stream_state: MutableMapping[str, Any], 1428 config: Config, 1429 message_repository: Optional[MessageRepository] = None, 1430 runtime_lookback_window: Optional[datetime.timedelta] = None, 1431 **kwargs: Any, 1432 ) -> ConcurrentCursor: 1433 component_type = component_definition.get("type") 1434 if component_definition.get("type") != model_type.__name__: 1435 raise ValueError( 1436 f"Expected manifest component of type {model_type.__name__}, but received {component_type} instead" 1437 ) 1438 1439 # FIXME the interfaces of the concurrent cursor are kind of annoying as they take a `ComponentDefinition` instead of the actual model. This was done because the ConcurrentDeclarativeSource didn't have access to the models [here for example](https://github.com/airbytehq/airbyte-python-cdk/blob/f525803b3fec9329e4cc8478996a92bf884bfde9/airbyte_cdk/sources/declarative/concurrent_declarative_source.py#L354C54-L354C91). So now we have two cases: 1440 # * The ComponentDefinition comes from model.__dict__ in which case we have `parameters` 1441 # * The ComponentDefinition comes from the manifest as a dict in which case we have `$parameters` 1442 # We should change those interfaces to use the model once we clean up the code in CDS at which point the parameter propagation should happen as part of the ModelToComponentFactory. 1443 if "$parameters" not in component_definition and "parameters" in component_definition: 1444 component_definition["$parameters"] = component_definition.get("parameters") # type: ignore # This is a dict 1445 datetime_based_cursor_model = model_type.parse_obj(component_definition) 1446 1447 if not isinstance(datetime_based_cursor_model, DatetimeBasedCursorModel): 1448 raise ValueError( 1449 f"Expected {model_type.__name__} component, but received {datetime_based_cursor_model.__class__.__name__}" 1450 ) 1451 1452 model_parameters = datetime_based_cursor_model.parameters or {} 1453 1454 cursor_field = self._get_catalog_defined_cursor_field( 1455 stream_name=stream_name, 1456 allow_catalog_defined_cursor_field=datetime_based_cursor_model.allow_catalog_defined_cursor_field 1457 or False, 1458 ) 1459 1460 if not cursor_field: 1461 interpolated_cursor_field = InterpolatedString.create( 1462 datetime_based_cursor_model.cursor_field, 1463 parameters=model_parameters, 1464 ) 1465 cursor_field = CursorField( 1466 cursor_field_key=interpolated_cursor_field.eval(config=config), 1467 supports_catalog_defined_cursor_field=datetime_based_cursor_model.allow_catalog_defined_cursor_field 1468 or False, 1469 ) 1470 1471 interpolated_partition_field_start = InterpolatedString.create( 1472 datetime_based_cursor_model.partition_field_start or "start_time", 1473 parameters=model_parameters, 1474 ) 1475 interpolated_partition_field_end = InterpolatedString.create( 1476 datetime_based_cursor_model.partition_field_end or "end_time", 1477 parameters=model_parameters, 1478 ) 1479 1480 slice_boundary_fields = ( 1481 interpolated_partition_field_start.eval(config=config), 1482 interpolated_partition_field_end.eval(config=config), 1483 ) 1484 1485 datetime_format = datetime_based_cursor_model.datetime_format 1486 1487 cursor_granularity = ( 1488 parse_duration(datetime_based_cursor_model.cursor_granularity) 1489 if datetime_based_cursor_model.cursor_granularity 1490 else None 1491 ) 1492 1493 lookback_window = None 1494 interpolated_lookback_window = ( 1495 InterpolatedString.create( 1496 datetime_based_cursor_model.lookback_window, 1497 parameters=model_parameters, 1498 ) 1499 if datetime_based_cursor_model.lookback_window 1500 else None 1501 ) 1502 if interpolated_lookback_window: 1503 evaluated_lookback_window = interpolated_lookback_window.eval(config=config) 1504 if evaluated_lookback_window: 1505 lookback_window = parse_duration(evaluated_lookback_window) 1506 1507 connector_state_converter: DateTimeStreamStateConverter 1508 connector_state_converter = CustomFormatConcurrentStreamStateConverter( 1509 datetime_format=datetime_format, 1510 input_datetime_formats=datetime_based_cursor_model.cursor_datetime_formats, 1511 is_sequential_state=True, # ConcurrentPerPartitionCursor only works with sequential state 1512 cursor_granularity=cursor_granularity, 1513 ) 1514 1515 # Adjusts the stream state by applying the runtime lookback window. 1516 # This is used to ensure correct state handling in case of failed partitions. 1517 stream_state_value = stream_state.get(cursor_field.cursor_field_key) 1518 if runtime_lookback_window and stream_state_value: 1519 new_stream_state = ( 1520 connector_state_converter.parse_timestamp(stream_state_value) 1521 - runtime_lookback_window 1522 ) 1523 stream_state[cursor_field.cursor_field_key] = connector_state_converter.output_format( 1524 new_stream_state 1525 ) 1526 1527 start_date_runtime_value: Union[InterpolatedString, str, MinMaxDatetime] 1528 if isinstance(datetime_based_cursor_model.start_datetime, MinMaxDatetimeModel): 1529 start_date_runtime_value = self.create_min_max_datetime( 1530 model=datetime_based_cursor_model.start_datetime, config=config 1531 ) 1532 else: 1533 start_date_runtime_value = datetime_based_cursor_model.start_datetime 1534 1535 end_date_runtime_value: Optional[Union[InterpolatedString, str, MinMaxDatetime]] 1536 if isinstance(datetime_based_cursor_model.end_datetime, MinMaxDatetimeModel): 1537 end_date_runtime_value = self.create_min_max_datetime( 1538 model=datetime_based_cursor_model.end_datetime, config=config 1539 ) 1540 else: 1541 end_date_runtime_value = datetime_based_cursor_model.end_datetime 1542 1543 interpolated_start_date = MinMaxDatetime.create( 1544 interpolated_string_or_min_max_datetime=start_date_runtime_value, 1545 parameters=datetime_based_cursor_model.parameters, 1546 ) 1547 interpolated_end_date = ( 1548 None 1549 if not end_date_runtime_value 1550 else MinMaxDatetime.create( 1551 end_date_runtime_value, datetime_based_cursor_model.parameters 1552 ) 1553 ) 1554 1555 # If datetime format is not specified then start/end datetime should inherit it from the stream slicer 1556 if not interpolated_start_date.datetime_format: 1557 interpolated_start_date.datetime_format = datetime_format 1558 if interpolated_end_date and not interpolated_end_date.datetime_format: 1559 interpolated_end_date.datetime_format = datetime_format 1560 1561 start_date = interpolated_start_date.get_datetime(config=config) 1562 end_date_provider = ( 1563 partial(interpolated_end_date.get_datetime, config) 1564 if interpolated_end_date 1565 else connector_state_converter.get_end_provider() 1566 ) 1567 1568 if ( 1569 datetime_based_cursor_model.step and not datetime_based_cursor_model.cursor_granularity 1570 ) or ( 1571 not datetime_based_cursor_model.step and datetime_based_cursor_model.cursor_granularity 1572 ): 1573 raise ValueError( 1574 f"If step is defined, cursor_granularity should be as well and vice-versa. " 1575 f"Right now, step is `{datetime_based_cursor_model.step}` and cursor_granularity is `{datetime_based_cursor_model.cursor_granularity}`" 1576 ) 1577 1578 # When step is not defined, default to a step size from the starting date to the present moment 1579 step_length = datetime.timedelta.max 1580 interpolated_step = ( 1581 InterpolatedString.create( 1582 datetime_based_cursor_model.step, 1583 parameters=model_parameters, 1584 ) 1585 if datetime_based_cursor_model.step 1586 else None 1587 ) 1588 if interpolated_step: 1589 evaluated_step = interpolated_step.eval(config) 1590 if evaluated_step: 1591 step_length = parse_duration(evaluated_step) 1592 1593 clamping_strategy: ClampingStrategy = NoClamping() 1594 if datetime_based_cursor_model.clamping: 1595 # While it is undesirable to interpolate within the model factory (as opposed to at runtime), 1596 # it is still better than shifting interpolation low-code concept into the ConcurrentCursor runtime 1597 # object which we want to keep agnostic of being low-code 1598 target = InterpolatedString( 1599 string=datetime_based_cursor_model.clamping.target, 1600 parameters=model_parameters, 1601 ) 1602 evaluated_target = target.eval(config=config) 1603 match evaluated_target: 1604 case "DAY": 1605 clamping_strategy = DayClampingStrategy() 1606 end_date_provider = ClampingEndProvider( 1607 DayClampingStrategy(is_ceiling=False), 1608 end_date_provider, # type: ignore # Having issues w/ inspection for GapType and CursorValueType as shown in existing tests. Confirmed functionality is working in practice 1609 granularity=cursor_granularity or datetime.timedelta(seconds=1), 1610 ) 1611 case "WEEK": 1612 if ( 1613 not datetime_based_cursor_model.clamping.target_details 1614 or "weekday" not in datetime_based_cursor_model.clamping.target_details 1615 ): 1616 raise ValueError( 1617 "Given WEEK clamping, weekday needs to be provided as target_details" 1618 ) 1619 weekday = self._assemble_weekday( 1620 datetime_based_cursor_model.clamping.target_details["weekday"] 1621 ) 1622 clamping_strategy = WeekClampingStrategy(weekday) 1623 end_date_provider = ClampingEndProvider( 1624 WeekClampingStrategy(weekday, is_ceiling=False), 1625 end_date_provider, # type: ignore # Having issues w/ inspection for GapType and CursorValueType as shown in existing tests. Confirmed functionality is working in practice 1626 granularity=cursor_granularity or datetime.timedelta(days=1), 1627 ) 1628 case "MONTH": 1629 clamping_strategy = MonthClampingStrategy() 1630 end_date_provider = ClampingEndProvider( 1631 MonthClampingStrategy(is_ceiling=False), 1632 end_date_provider, # type: ignore # Having issues w/ inspection for GapType and CursorValueType as shown in existing tests. Confirmed functionality is working in practice 1633 granularity=cursor_granularity or datetime.timedelta(days=1), 1634 ) 1635 case _: 1636 raise ValueError( 1637 f"Invalid clamping target {evaluated_target}, expected DAY, WEEK, MONTH" 1638 ) 1639 1640 return ConcurrentCursor( 1641 stream_name=stream_name, 1642 stream_namespace=stream_namespace, 1643 stream_state=stream_state, 1644 message_repository=message_repository or self._message_repository, 1645 connector_state_manager=self._connector_state_manager, 1646 connector_state_converter=connector_state_converter, 1647 cursor_field=cursor_field, 1648 slice_boundary_fields=slice_boundary_fields, 1649 start=start_date, # type: ignore # Having issues w/ inspection for GapType and CursorValueType as shown in existing tests. Confirmed functionality is working in practice 1650 end_provider=end_date_provider, # type: ignore # Having issues w/ inspection for GapType and CursorValueType as shown in existing tests. Confirmed functionality is working in practice 1651 lookback_window=lookback_window, 1652 slice_range=step_length, 1653 cursor_granularity=cursor_granularity, 1654 clamping_strategy=clamping_strategy, 1655 is_compare_strictly=datetime_based_cursor_model.is_compare_strictly or False, 1656 )
def
create_concurrent_cursor_from_incrementing_count_cursor( self, model_type: Type[pydantic.v1.main.BaseModel], component_definition: Mapping[str, Any], stream_name: str, stream_namespace: Optional[str], stream_state: MutableMapping[str, Any], config: Mapping[str, Any], message_repository: Optional[airbyte_cdk.MessageRepository] = None, **kwargs: Any) -> airbyte_cdk.ConcurrentCursor:
1658 def create_concurrent_cursor_from_incrementing_count_cursor( 1659 self, 1660 model_type: Type[BaseModel], 1661 component_definition: ComponentDefinition, 1662 stream_name: str, 1663 stream_namespace: Optional[str], 1664 stream_state: MutableMapping[str, Any], 1665 config: Config, 1666 message_repository: Optional[MessageRepository] = None, 1667 **kwargs: Any, 1668 ) -> ConcurrentCursor: 1669 component_type = component_definition.get("type") 1670 if component_definition.get("type") != model_type.__name__: 1671 raise ValueError( 1672 f"Expected manifest component of type {model_type.__name__}, but received {component_type} instead" 1673 ) 1674 1675 incrementing_count_cursor_model = model_type.parse_obj(component_definition) 1676 1677 if not isinstance(incrementing_count_cursor_model, IncrementingCountCursorModel): 1678 raise ValueError( 1679 f"Expected {model_type.__name__} component, but received {incrementing_count_cursor_model.__class__.__name__}" 1680 ) 1681 1682 start_value: Union[int, str, None] = incrementing_count_cursor_model.start_value 1683 # Pydantic Union type coercion can convert int 0 to string '0' depending on Union order. 1684 # We need to handle both int and str representations of numeric values. 1685 # Evaluate the InterpolatedString and convert to int for the ConcurrentCursor. 1686 if start_value is not None: 1687 interpolated_start_value = InterpolatedString.create( 1688 str(start_value), # Ensure we pass a string to InterpolatedString.create 1689 parameters=incrementing_count_cursor_model.parameters or {}, 1690 ) 1691 evaluated_start_value: int = int(interpolated_start_value.eval(config=config)) 1692 else: 1693 evaluated_start_value = 0 1694 1695 cursor_field = self._get_catalog_defined_cursor_field( 1696 stream_name=stream_name, 1697 allow_catalog_defined_cursor_field=incrementing_count_cursor_model.allow_catalog_defined_cursor_field 1698 or False, 1699 ) 1700 1701 if not cursor_field: 1702 interpolated_cursor_field = InterpolatedString.create( 1703 incrementing_count_cursor_model.cursor_field, 1704 parameters=incrementing_count_cursor_model.parameters or {}, 1705 ) 1706 cursor_field = CursorField( 1707 cursor_field_key=interpolated_cursor_field.eval(config=config), 1708 supports_catalog_defined_cursor_field=incrementing_count_cursor_model.allow_catalog_defined_cursor_field 1709 or False, 1710 ) 1711 1712 connector_state_converter = IncrementingCountStreamStateConverter( 1713 is_sequential_state=True, # ConcurrentPerPartitionCursor only works with sequential state 1714 ) 1715 1716 return ConcurrentCursor( 1717 stream_name=stream_name, 1718 stream_namespace=stream_namespace, 1719 stream_state=stream_state, 1720 message_repository=message_repository or self._message_repository, 1721 connector_state_manager=self._connector_state_manager, 1722 connector_state_converter=connector_state_converter, 1723 cursor_field=cursor_field, 1724 slice_boundary_fields=None, 1725 start=evaluated_start_value, # type: ignore # Having issues w/ inspection for GapType and CursorValueType as shown in existing tests. Confirmed functionality is working in practice 1726 end_provider=connector_state_converter.get_end_provider(), # type: ignore # Having issues w/ inspection for GapType and CursorValueType as shown in existing tests. Confirmed functionality is working in practice 1727 )
def
create_concurrent_cursor_from_perpartition_cursor( self, state_manager: airbyte_cdk.ConnectorStateManager, model_type: Type[pydantic.v1.main.BaseModel], component_definition: Mapping[str, Any], stream_name: str, stream_namespace: Optional[str], config: Mapping[str, Any], stream_state: MutableMapping[str, Any], partition_router: airbyte_cdk.sources.declarative.partition_routers.PartitionRouter, attempt_to_create_cursor_if_not_provided: bool = False, **kwargs: Any) -> airbyte_cdk.sources.declarative.incremental.ConcurrentPerPartitionCursor:
1748 def create_concurrent_cursor_from_perpartition_cursor( 1749 self, 1750 state_manager: ConnectorStateManager, 1751 model_type: Type[BaseModel], 1752 component_definition: ComponentDefinition, 1753 stream_name: str, 1754 stream_namespace: Optional[str], 1755 config: Config, 1756 stream_state: MutableMapping[str, Any], 1757 partition_router: PartitionRouter, 1758 attempt_to_create_cursor_if_not_provided: bool = False, 1759 **kwargs: Any, 1760 ) -> ConcurrentPerPartitionCursor: 1761 component_type = component_definition.get("type") 1762 if component_definition.get("type") != model_type.__name__: 1763 raise ValueError( 1764 f"Expected manifest component of type {model_type.__name__}, but received {component_type} instead" 1765 ) 1766 1767 # FIXME the interfaces of the concurrent cursor are kind of annoying as they take a `ComponentDefinition` instead of the actual model. This was done because the ConcurrentDeclarativeSource didn't have access to the models [here for example](https://github.com/airbytehq/airbyte-python-cdk/blob/f525803b3fec9329e4cc8478996a92bf884bfde9/airbyte_cdk/sources/declarative/concurrent_declarative_source.py#L354C54-L354C91). So now we have two cases: 1768 # * The ComponentDefinition comes from model.__dict__ in which case we have `parameters` 1769 # * The ComponentDefinition comes from the manifest as a dict in which case we have `$parameters` 1770 # We should change those interfaces to use the model once we clean up the code in CDS at which point the parameter propagation should happen as part of the ModelToComponentFactory. 1771 if "$parameters" not in component_definition and "parameters" in component_definition: 1772 component_definition["$parameters"] = component_definition.get("parameters") # type: ignore # This is a dict 1773 datetime_based_cursor_model = model_type.parse_obj(component_definition) 1774 1775 if not isinstance(datetime_based_cursor_model, DatetimeBasedCursorModel): 1776 raise ValueError( 1777 f"Expected {model_type.__name__} component, but received {datetime_based_cursor_model.__class__.__name__}" 1778 ) 1779 1780 cursor_field = self._get_catalog_defined_cursor_field( 1781 stream_name=stream_name, 1782 allow_catalog_defined_cursor_field=datetime_based_cursor_model.allow_catalog_defined_cursor_field 1783 or False, 1784 ) 1785 1786 if not cursor_field: 1787 interpolated_cursor_field = InterpolatedString.create( 1788 datetime_based_cursor_model.cursor_field, 1789 # FIXME the interfaces of the concurrent cursor are kind of annoying as they take a `ComponentDefinition` instead of the actual model. This was done because the ConcurrentDeclarativeSource didn't have access to the models [here for example](https://github.com/airbytehq/airbyte-python-cdk/blob/f525803b3fec9329e4cc8478996a92bf884bfde9/airbyte_cdk/sources/declarative/concurrent_declarative_source.py#L354C54-L354C91). So now we have two cases: 1790 # * The ComponentDefinition comes from model.__dict__ in which case we have `parameters` 1791 # * The ComponentDefinition comes from the manifest as a dict in which case we have `$parameters` 1792 # We should change those interfaces to use the model once we clean up the code in CDS at which point the parameter propagation should happen as part of the ModelToComponentFactory. 1793 parameters=datetime_based_cursor_model.parameters or {}, 1794 ) 1795 cursor_field = CursorField( 1796 cursor_field_key=interpolated_cursor_field.eval(config=config), 1797 supports_catalog_defined_cursor_field=datetime_based_cursor_model.allow_catalog_defined_cursor_field 1798 or False, 1799 ) 1800 1801 datetime_format = datetime_based_cursor_model.datetime_format 1802 1803 cursor_granularity = ( 1804 parse_duration(datetime_based_cursor_model.cursor_granularity) 1805 if datetime_based_cursor_model.cursor_granularity 1806 else None 1807 ) 1808 1809 connector_state_converter: DateTimeStreamStateConverter 1810 connector_state_converter = CustomFormatConcurrentStreamStateConverter( 1811 datetime_format=datetime_format, 1812 input_datetime_formats=datetime_based_cursor_model.cursor_datetime_formats, 1813 is_sequential_state=True, # ConcurrentPerPartitionCursor only works with sequential state 1814 cursor_granularity=cursor_granularity, 1815 ) 1816 1817 # Create the cursor factory 1818 cursor_factory = ConcurrentCursorFactory( 1819 partial( 1820 self.create_concurrent_cursor_from_datetime_based_cursor, 1821 state_manager=state_manager, 1822 model_type=model_type, 1823 component_definition=component_definition, 1824 stream_name=stream_name, 1825 stream_namespace=stream_namespace, 1826 config=config, 1827 message_repository=NoopMessageRepository(), 1828 ) 1829 ) 1830 1831 # Per-partition state doesn't make sense for GroupingPartitionRouter, so force the global state 1832 use_global_cursor = isinstance( 1833 partition_router, GroupingPartitionRouter 1834 ) or component_definition.get("global_substream_cursor", False) 1835 1836 # Return the concurrent cursor and state converter 1837 return ConcurrentPerPartitionCursor( 1838 cursor_factory=cursor_factory, 1839 partition_router=partition_router, 1840 stream_name=stream_name, 1841 stream_namespace=stream_namespace, 1842 stream_state=stream_state, 1843 message_repository=self._message_repository, # type: ignore 1844 connector_state_manager=state_manager, 1845 connector_state_converter=connector_state_converter, 1846 cursor_field=cursor_field, 1847 use_global_cursor=use_global_cursor, 1848 attempt_to_create_cursor_if_not_provided=attempt_to_create_cursor_if_not_provided, 1849 )
@staticmethod
def
create_constant_backoff_strategy( model: airbyte_cdk.sources.declarative.models.declarative_component_schema.ConstantBackoffStrategy, config: Mapping[str, Any], **kwargs: Any) -> airbyte_cdk.sources.declarative.requesters.error_handlers.backoff_strategies.ConstantBackoffStrategy:
1851 @staticmethod 1852 def create_constant_backoff_strategy( 1853 model: ConstantBackoffStrategyModel, config: Config, **kwargs: Any 1854 ) -> ConstantBackoffStrategy: 1855 ModelToComponentFactory._validate_jitter_range(model.jitter_range_in_seconds) 1856 return ConstantBackoffStrategy( 1857 backoff_time_in_seconds=model.backoff_time_in_seconds, 1858 jitter_range_in_seconds=model.jitter_range_in_seconds, 1859 config=config, 1860 parameters=model.parameters or {}, 1861 )
def
create_cursor_pagination( self, model: airbyte_cdk.sources.declarative.models.declarative_component_schema.CursorPagination, config: Mapping[str, Any], decoder: Optional[airbyte_cdk.Decoder] = None, **kwargs: Any) -> airbyte_cdk.CursorPaginationStrategy:
1868 def create_cursor_pagination( 1869 self, 1870 model: CursorPaginationModel, 1871 config: Config, 1872 decoder: Optional[Decoder] = None, 1873 **kwargs: Any, 1874 ) -> CursorPaginationStrategy: 1875 # None when nested under a custom component, which cannot pass a decoder 1876 if decoder is None: 1877 decoder = JsonDecoder(parameters={}) 1878 if isinstance(decoder, PaginationDecoderDecorator): 1879 inner_decoder = decoder.decoder 1880 else: 1881 inner_decoder = decoder 1882 decoder = PaginationDecoderDecorator(decoder=decoder) 1883 1884 if self._is_supported_decoder_for_pagination(inner_decoder): 1885 decoder_to_use = decoder 1886 else: 1887 raise ValueError( 1888 self._UNSUPPORTED_DECODER_ERROR.format(decoder_type=type(inner_decoder)) 1889 ) 1890 1891 # Pydantic v1 Union type coercion can convert int to string depending on Union order. 1892 # If page_size is a string that represents an integer (not an interpolation), convert it back. 1893 page_size = model.page_size 1894 if isinstance(page_size, str) and page_size.isdigit(): 1895 page_size = int(page_size) 1896 1897 return CursorPaginationStrategy( 1898 cursor_value=model.cursor_value, 1899 decoder=decoder_to_use, 1900 page_size=page_size, 1901 stop_condition=model.stop_condition, 1902 config=config, 1903 parameters=model.parameters or {}, 1904 )
def
create_custom_component(self, model: Any, config: Mapping[str, Any], **kwargs: Any) -> Any:
1906 def create_custom_component(self, model: Any, config: Config, **kwargs: Any) -> Any: 1907 """ 1908 Generically creates a custom component based on the model type and a class_name reference to the custom Python class being 1909 instantiated. Only the model's additional properties that match the custom class definition are passed to the constructor 1910 :param model: The Pydantic model of the custom component being created 1911 :param config: The custom defined connector config 1912 :return: The declarative component built from the Pydantic model to be used at runtime 1913 """ 1914 # Instantiating a custom component means importing and executing arbitrary code referenced 1915 # by `class_name`. Manifests supplied by a caller, whether through the config or directly to 1916 # the manifest server, are untrusted input and could point `class_name` at any importable 1917 # callable, so they honor the same `AIRBYTE_ENABLE_UNSAFE_CODE` gate as injected 1918 # `components.py` code. Manifests bundled in a published connector image are trusted and may 1919 # always use their bundled custom components. 1920 manifest_is_untrusted = not self._custom_components_trusted or bool( 1921 config.get(INJECTED_MANIFEST) 1922 ) 1923 if manifest_is_untrusted and not custom_code_execution_permitted(): 1924 raise AirbyteCustomCodeNotPermittedError 1925 1926 custom_component_class = self._get_class_from_fully_qualified_class_name(model.class_name) 1927 component_fields = get_type_hints(custom_component_class) 1928 model_args = model.dict() 1929 model_args["config"] = config 1930 1931 # There are cases where a parent component will pass arguments to a child component via kwargs. When there are field collisions 1932 # we defer to these arguments over the component's definition 1933 for key, arg in kwargs.items(): 1934 model_args[key] = arg 1935 1936 # Pydantic is unable to parse a custom component's fields that are subcomponents into models because their fields and types are not 1937 # defined in the schema. The fields and types are defined within the Python class implementation. Pydantic can only parse down to 1938 # the custom component and this code performs a second parse to convert the sub-fields first into models, then declarative components 1939 for model_field, model_value in model_args.items(): 1940 # If a custom component field doesn't have a type set, we try to use the type hints to infer the type 1941 if ( 1942 isinstance(model_value, dict) 1943 and "type" not in model_value 1944 and model_field in component_fields 1945 ): 1946 derived_type = self._derive_component_type_from_type_hints( 1947 component_fields.get(model_field) 1948 ) 1949 if derived_type: 1950 model_value["type"] = derived_type 1951 1952 if self._is_component(model_value): 1953 model_args[model_field] = self._create_nested_component( 1954 model, 1955 model_field, 1956 model_value, 1957 config, 1958 **kwargs, 1959 ) 1960 elif isinstance(model_value, list): 1961 vals = [] 1962 for v in model_value: 1963 if isinstance(v, dict) and "type" not in v and model_field in component_fields: 1964 derived_type = self._derive_component_type_from_type_hints( 1965 component_fields.get(model_field) 1966 ) 1967 if derived_type: 1968 v["type"] = derived_type 1969 if self._is_component(v): 1970 vals.append( 1971 self._create_nested_component( 1972 model, 1973 model_field, 1974 v, 1975 config, 1976 **kwargs, 1977 ) 1978 ) 1979 else: 1980 vals.append(v) 1981 model_args[model_field] = vals 1982 1983 kwargs = { 1984 class_field: model_args[class_field] 1985 for class_field in component_fields.keys() 1986 if class_field in model_args 1987 } 1988 1989 if "api_budget" in component_fields and kwargs.get("api_budget") is None: 1990 kwargs["api_budget"] = self._api_budget 1991 1992 return custom_component_class(**kwargs)
Generically creates a custom component based on the model type and a class_name reference to the custom Python class being instantiated. Only the model's additional properties that match the custom class definition are passed to the constructor
Parameters
- model: The Pydantic model of the custom component being created
- config: The custom defined connector config
Returns
The declarative component built from the Pydantic model to be used at runtime
def
create_default_stream( self, model: airbyte_cdk.sources.declarative.models.declarative_component_schema.DeclarativeStream, config: Mapping[str, Any], is_parent: bool = False, **kwargs: Any) -> airbyte_cdk.sources.streams.concurrent.abstract_stream.AbstractStream:
2127 def create_default_stream( 2128 self, model: DeclarativeStreamModel, config: Config, is_parent: bool = False, **kwargs: Any 2129 ) -> AbstractStream: 2130 primary_key = model.primary_key.__root__ if model.primary_key else None 2131 self._migrate_state(model, config) 2132 self._warn_on_ineffective_incremental_dependency(model) 2133 2134 partition_router = self._build_stream_slicer_from_partition_router( 2135 model.retriever, 2136 config, 2137 stream_name=model.name, 2138 **kwargs, 2139 ) 2140 concurrent_cursor = self._build_concurrent_cursor(model, partition_router, config) 2141 if model.incremental_sync and isinstance(model.incremental_sync, DatetimeBasedCursorModel): 2142 cursor_model: DatetimeBasedCursorModel = model.incremental_sync 2143 2144 end_time_option = ( 2145 self._create_component_from_model( 2146 cursor_model.end_time_option, config, parameters=cursor_model.parameters or {} 2147 ) 2148 if cursor_model.end_time_option 2149 else None 2150 ) 2151 start_time_option = ( 2152 self._create_component_from_model( 2153 cursor_model.start_time_option, config, parameters=cursor_model.parameters or {} 2154 ) 2155 if cursor_model.start_time_option 2156 else None 2157 ) 2158 2159 datetime_request_options_provider = DatetimeBasedRequestOptionsProvider( 2160 start_time_option=start_time_option, 2161 end_time_option=end_time_option, 2162 partition_field_start=cursor_model.partition_field_start, 2163 partition_field_end=cursor_model.partition_field_end, 2164 config=config, 2165 parameters=model.parameters or {}, 2166 ) 2167 request_options_provider = ( 2168 datetime_request_options_provider 2169 if not isinstance(concurrent_cursor, ConcurrentPerPartitionCursor) 2170 else PerPartitionRequestOptionsProvider( 2171 partition_router, datetime_request_options_provider 2172 ) 2173 ) 2174 elif model.incremental_sync and isinstance( 2175 model.incremental_sync, IncrementingCountCursorModel 2176 ): 2177 if isinstance(concurrent_cursor, ConcurrentPerPartitionCursor): 2178 raise ValueError( 2179 "PerPartition does not support per partition states because switching to global state is time based" 2180 ) 2181 2182 cursor_model: IncrementingCountCursorModel = model.incremental_sync # type: ignore 2183 2184 start_time_option = ( 2185 self._create_component_from_model( 2186 cursor_model.start_value_option, # type: ignore # mypy still thinks cursor_model of type DatetimeBasedCursor 2187 config, 2188 parameters=cursor_model.parameters or {}, 2189 ) 2190 if cursor_model.start_value_option # type: ignore # mypy still thinks cursor_model of type DatetimeBasedCursor 2191 else None 2192 ) 2193 2194 # The concurrent engine defaults the start/end fields on the slice to "start" and "end", but 2195 # the default DatetimeBasedRequestOptionsProvider() sets them to start_time/end_time 2196 partition_field_start = "start" 2197 2198 request_options_provider = DatetimeBasedRequestOptionsProvider( 2199 start_time_option=start_time_option, 2200 partition_field_start=partition_field_start, 2201 config=config, 2202 parameters=model.parameters or {}, 2203 ) 2204 else: 2205 request_options_provider = None 2206 2207 transformations = [] 2208 if model.transformations: 2209 for transformation_model in model.transformations: 2210 transformations.append( 2211 self._create_component_from_model(model=transformation_model, config=config) 2212 ) 2213 file_uploader = None 2214 if model.file_uploader: 2215 file_uploader = self._create_component_from_model( 2216 model=model.file_uploader, config=config 2217 ) 2218 2219 stream_slicer: ConcurrentStreamSlicer = ( 2220 partition_router 2221 if isinstance(concurrent_cursor, FinalStateCursor) 2222 else concurrent_cursor 2223 ) 2224 2225 retriever = self._create_component_from_model( 2226 model=model.retriever, 2227 config=config, 2228 name=model.name, 2229 primary_key=primary_key, 2230 request_options_provider=request_options_provider, 2231 stream_slicer=stream_slicer, 2232 partition_router=partition_router, 2233 has_stop_condition_cursor=self._is_stop_condition_on_cursor(model), 2234 is_client_side_incremental_sync=self._is_client_side_filtering_enabled(model), 2235 cursor=concurrent_cursor, 2236 transformations=transformations, 2237 file_uploader=file_uploader, 2238 incremental_sync=model.incremental_sync, 2239 ) 2240 if isinstance(retriever, AsyncRetriever): 2241 stream_slicer = retriever.stream_slicer 2242 2243 schema_loader: SchemaLoader 2244 if model.schema_loader and isinstance(model.schema_loader, list): 2245 nested_schema_loaders = [ 2246 self._create_component_from_model(model=nested_schema_loader, config=config) 2247 for nested_schema_loader in model.schema_loader 2248 ] 2249 schema_loader = CompositeSchemaLoader( 2250 schema_loaders=nested_schema_loaders, parameters={} 2251 ) 2252 elif model.schema_loader: 2253 schema_loader = self._create_component_from_model( 2254 model=model.schema_loader, # type: ignore # If defined, schema_loader is guaranteed not to be a list and will be one of the existing base models 2255 config=config, 2256 ) 2257 else: 2258 options = model.parameters or {} 2259 if "name" not in options: 2260 options["name"] = model.name 2261 schema_loader = DefaultSchemaLoader(config=config, parameters=options) 2262 schema_loader = CachingSchemaLoaderDecorator(schema_loader) 2263 2264 stream_name = model.name or "" 2265 return DefaultStream( 2266 partition_generator=StreamSlicerPartitionGenerator( 2267 DeclarativePartitionFactory( 2268 stream_name, 2269 schema_loader, 2270 retriever, 2271 self._message_repository, 2272 ), 2273 stream_slicer, 2274 slice_limit=self._limit_slices_fetched, 2275 ), 2276 name=stream_name, 2277 json_schema=schema_loader.get_json_schema, 2278 primary_key=get_primary_key_from_stream(primary_key), 2279 cursor_field=( 2280 concurrent_cursor.cursor_field 2281 if hasattr(concurrent_cursor, "cursor_field") 2282 else None 2283 ), 2284 logger=logging.getLogger(f"airbyte.{stream_name}"), 2285 cursor=concurrent_cursor, 2286 supports_file_transfer=hasattr(model, "file_uploader") and bool(model.file_uploader), 2287 )
def
create_default_error_handler( self, model: airbyte_cdk.sources.declarative.models.declarative_component_schema.DefaultErrorHandler, config: Mapping[str, Any], **kwargs: Any) -> airbyte_cdk.sources.declarative.requesters.error_handlers.DefaultErrorHandler:
2459 def create_default_error_handler( 2460 self, model: DefaultErrorHandlerModel, config: Config, **kwargs: Any 2461 ) -> DefaultErrorHandler: 2462 backoff_strategies = [] 2463 if model.backoff_strategies: 2464 for backoff_strategy_model in model.backoff_strategies: 2465 backoff_strategies.append( 2466 self._create_component_from_model(model=backoff_strategy_model, config=config) 2467 ) 2468 2469 response_filters = [] 2470 if model.response_filters: 2471 for response_filter_model in model.response_filters: 2472 response_filters.append( 2473 self._create_component_from_model(model=response_filter_model, config=config) 2474 ) 2475 response_filters.append( 2476 HttpResponseFilter(config=config, parameters=model.parameters or {}) 2477 ) 2478 2479 return DefaultErrorHandler( 2480 backoff_strategies=backoff_strategies, 2481 max_retries=model.max_retries, 2482 response_filters=response_filters, 2483 config=config, 2484 parameters=model.parameters or {}, 2485 )
def
create_default_paginator( self, model: airbyte_cdk.sources.declarative.models.declarative_component_schema.DefaultPaginator, config: Mapping[str, Any], *, url_base: str, extractor_model: Union[airbyte_cdk.sources.declarative.models.declarative_component_schema.DpathExtractor, airbyte_cdk.sources.declarative.models.declarative_component_schema.CustomRecordExtractor, airbyte_cdk.sources.declarative.models.declarative_component_schema.CombinedExtractor, NoneType] = None, decoder: Optional[airbyte_cdk.Decoder] = None, cursor_used_for_stop_condition: Optional[airbyte_cdk.Cursor] = None) -> Union[airbyte_cdk.DefaultPaginator, airbyte_cdk.sources.declarative.requesters.paginators.PaginatorTestReadDecorator]:
2487 def create_default_paginator( 2488 self, 2489 model: DefaultPaginatorModel, 2490 config: Config, 2491 *, 2492 url_base: str, 2493 extractor_model: Optional[ 2494 Union[CustomRecordExtractorModel, DpathExtractorModel, CombinedExtractorModel] 2495 ] = None, 2496 decoder: Optional[Decoder] = None, 2497 cursor_used_for_stop_condition: Optional[Cursor] = None, 2498 ) -> Union[DefaultPaginator, PaginatorTestReadDecorator]: 2499 if decoder: 2500 if self._is_supported_decoder_for_pagination(decoder): 2501 decoder_to_use = PaginationDecoderDecorator(decoder=decoder) 2502 else: 2503 raise ValueError(self._UNSUPPORTED_DECODER_ERROR.format(decoder_type=type(decoder))) 2504 else: 2505 decoder_to_use = PaginationDecoderDecorator(decoder=JsonDecoder(parameters={})) 2506 page_size_option = ( 2507 self._create_component_from_model(model=model.page_size_option, config=config) 2508 if model.page_size_option 2509 else None 2510 ) 2511 page_token_option = ( 2512 self._create_component_from_model(model=model.page_token_option, config=config) 2513 if model.page_token_option 2514 else None 2515 ) 2516 pagination_strategy = self._create_component_from_model( 2517 model=model.pagination_strategy, 2518 config=config, 2519 decoder=decoder_to_use, 2520 extractor_model=extractor_model, 2521 ) 2522 if cursor_used_for_stop_condition: 2523 pagination_strategy = StopConditionPaginationStrategyDecorator( 2524 pagination_strategy, CursorStopCondition(cursor_used_for_stop_condition) 2525 ) 2526 paginator = DefaultPaginator( 2527 decoder=decoder_to_use, 2528 page_size_option=page_size_option, 2529 page_token_option=page_token_option, 2530 pagination_strategy=pagination_strategy, 2531 url_base=url_base, 2532 config=config, 2533 parameters=model.parameters or {}, 2534 ) 2535 if self._limit_pages_fetched_per_slice: 2536 return PaginatorTestReadDecorator(paginator, self._limit_pages_fetched_per_slice) 2537 return paginator
def
create_combined_extractor( self, model: airbyte_cdk.sources.declarative.models.declarative_component_schema.CombinedExtractor, config: Mapping[str, Any], decoder: Optional[airbyte_cdk.Decoder] = None, **kwargs: Any) -> airbyte_cdk.CombinedExtractor:
2595 def create_combined_extractor( 2596 self, 2597 model: CombinedExtractorModel, 2598 config: Config, 2599 decoder: Optional[Decoder] = None, 2600 **kwargs: Any, 2601 ) -> CombinedExtractor: 2602 self._reject_combined_extractor_over_streaming_decoder(decoder) 2603 extractors = [ 2604 self._create_component_from_model(model=sub_extractor, config=config, decoder=decoder) 2605 for sub_extractor in model.extractors 2606 ] 2607 return CombinedExtractor( 2608 extractors=extractors, 2609 mode=CombineMode(model.mode.value) if model.mode else CombineMode.union, 2610 skip_empty_records=bool(model.skip_empty_records), 2611 parameters=model.parameters or {}, 2612 )
def
create_dpath_extractor( self, model: airbyte_cdk.sources.declarative.models.declarative_component_schema.DpathExtractor, config: Mapping[str, Any], decoder: Optional[airbyte_cdk.Decoder] = None, **kwargs: Any) -> airbyte_cdk.DpathExtractor:
2614 def create_dpath_extractor( 2615 self, 2616 model: DpathExtractorModel, 2617 config: Config, 2618 decoder: Optional[Decoder] = None, 2619 **kwargs: Any, 2620 ) -> DpathExtractor: 2621 if decoder: 2622 decoder_to_use = decoder 2623 else: 2624 decoder_to_use = JsonDecoder(parameters={}) 2625 model_field_path: List[Union[InterpolatedString, str]] = [x for x in model.field_path] 2626 2627 record_expander = None 2628 if model.record_expander: 2629 record_expander = self._create_component_from_model( 2630 model=model.record_expander, 2631 config=config, 2632 ) 2633 2634 return DpathExtractor( 2635 decoder=decoder_to_use, 2636 field_path=model_field_path, 2637 config=config, 2638 parameters=model.parameters or {}, 2639 record_expander=record_expander, 2640 )
def
create_record_expander( self, model: airbyte_cdk.sources.declarative.models.declarative_component_schema.RecordExpander, config: Mapping[str, Any], **kwargs: Any) -> airbyte_cdk.sources.declarative.expanders.RecordExpander:
2642 def create_record_expander( 2643 self, 2644 model: RecordExpanderModel, 2645 config: Config, 2646 **kwargs: Any, 2647 ) -> RecordExpander: 2648 truncated_list_retriever = None 2649 suppress_incomplete_fetch_warning = False 2650 if model.truncated_list_retriever: 2651 retriever_model = model.truncated_list_retriever 2652 name = "record_expander_truncated_list" 2653 # `CustomRetriever` allows extra fields, so read from the dumped model to cover both types. 2654 retriever_fields = retriever_model.dict() 2655 for unsupported_option in ("partition_router", "pagination_reset"): 2656 if retriever_fields.get(unsupported_option): 2657 raise ValueError( 2658 f"`{unsupported_option}` is not supported on `truncated_list_retriever`." 2659 ) 2660 log_formatter = lambda response: format_http_message( 2661 response, 2662 f"Record expander '{name}' request", 2663 "Request performed in order to fetch the complete nested list of a truncated record.", 2664 name, 2665 is_auxiliary=True, 2666 ) 2667 # `name`/`primary_key`/`transformations` are also what a `CustomRetriever` forwards to its 2668 # nested `requester`/`record_selector`, so they are passed for both retriever types. 2669 truncated_list_retriever = self._create_component_from_model( 2670 model=retriever_model, 2671 config=config, 2672 name=name, 2673 primary_key=None, 2674 transformations=[], 2675 log_formatter=log_formatter, 2676 ) 2677 # Only a capped paginator makes a shortfall expected; `NoPagination` and retrievers 2678 # without a paginator are never capped. 2679 suppress_incomplete_fetch_warning = isinstance( 2680 truncated_list_retriever, SimpleRetriever 2681 ) and isinstance(truncated_list_retriever.paginator, PaginatorTestReadDecorator) 2682 return RecordExpander( 2683 expand_records_from_field=model.expand_records_from_field, 2684 config=config, 2685 parameters=model.parameters or {}, 2686 remain_original_record=model.remain_original_record or False, 2687 parent_fields=[ 2688 self._create_component_from_model(model=parent_field, config=config) 2689 for parent_field in model.parent_fields 2690 ] 2691 if model.parent_fields 2692 else None, 2693 merge_parent=model.merge_parent or False, 2694 on_no_records=OnNoRecords(model.on_no_records.value) 2695 if model.on_no_records 2696 else OnNoRecords.skip, 2697 truncation_indicator_path=model.truncation_indicator_path, 2698 truncated_list_retriever=truncated_list_retriever, 2699 message_repository=self._message_repository, 2700 suppress_incomplete_fetch_warning=suppress_incomplete_fetch_warning, 2701 )
@staticmethod
def
create_parent_field_path( model: airbyte_cdk.sources.declarative.models.declarative_component_schema.ParentFieldPath, config: Mapping[str, Any], **kwargs: Any) -> airbyte_cdk.sources.declarative.expanders.ParentFieldPath:
2703 @staticmethod 2704 def create_parent_field_path( 2705 model: ParentFieldPathModel, 2706 config: Config, 2707 **kwargs: Any, 2708 ) -> ParentFieldPath: 2709 return ParentFieldPath( 2710 parent_path=model.parent_path, 2711 record_path=model.record_path, 2712 config=config, 2713 parameters=model.parameters or {}, 2714 )
@staticmethod
def
create_response_to_file_extractor( model: airbyte_cdk.sources.declarative.models.declarative_component_schema.ResponseToFileExtractor, **kwargs: Any) -> airbyte_cdk.sources.declarative.extractors.ResponseToFileExtractor:
2716 @staticmethod 2717 def create_response_to_file_extractor( 2718 model: ResponseToFileExtractorModel, 2719 **kwargs: Any, 2720 ) -> ResponseToFileExtractor: 2721 return ResponseToFileExtractor( 2722 parameters=model.parameters or {}, 2723 preserve_na_values=model.preserve_na_values or False, 2724 )
@staticmethod
def
create_exponential_backoff_strategy( model: airbyte_cdk.sources.declarative.models.declarative_component_schema.ExponentialBackoffStrategy, config: Mapping[str, Any]) -> airbyte_cdk.sources.declarative.requesters.error_handlers.backoff_strategies.ExponentialBackoffStrategy:
2726 @staticmethod 2727 def create_exponential_backoff_strategy( 2728 model: ExponentialBackoffStrategyModel, config: Config 2729 ) -> ExponentialBackoffStrategy: 2730 ModelToComponentFactory._validate_jitter_range(model.jitter_range_in_seconds) 2731 return ExponentialBackoffStrategy( 2732 factor=model.factor or 5, 2733 jitter_range_in_seconds=model.jitter_range_in_seconds, 2734 parameters=model.parameters or {}, 2735 config=config, 2736 )
@staticmethod
def
create_group_by_key( model: airbyte_cdk.sources.declarative.models.declarative_component_schema.GroupByKeyMergeStrategy, config: Mapping[str, Any]) -> airbyte_cdk.sources.declarative.requesters.query_properties.strategies.GroupByKey:
def
create_http_requester( self, model: airbyte_cdk.sources.declarative.models.declarative_component_schema.HttpRequester, config: Mapping[str, Any], decoder: airbyte_cdk.Decoder = JsonDecoder(), query_properties_key: Optional[str] = None, use_cache: Optional[bool] = None, *, name: str) -> airbyte_cdk.HttpRequester:
2742 def create_http_requester( 2743 self, 2744 model: HttpRequesterModel, 2745 config: Config, 2746 decoder: Decoder = JsonDecoder(parameters={}), 2747 query_properties_key: Optional[str] = None, 2748 use_cache: Optional[bool] = None, 2749 *, 2750 name: str, 2751 ) -> HttpRequester: 2752 authenticator = ( 2753 self._create_component_from_model( 2754 model=model.authenticator, 2755 config=config, 2756 url_base=model.url or model.url_base, 2757 name=name, 2758 decoder=decoder, 2759 ) 2760 if model.authenticator 2761 else None 2762 ) 2763 error_handler = ( 2764 self._create_component_from_model(model=model.error_handler, config=config) 2765 if model.error_handler 2766 else DefaultErrorHandler( 2767 backoff_strategies=[], 2768 response_filters=[], 2769 config=config, 2770 parameters=model.parameters or {}, 2771 ) 2772 ) 2773 2774 api_budget = self._api_budget 2775 2776 request_options_provider = InterpolatedRequestOptionsProvider( 2777 request_body=model.request_body, 2778 request_body_data=model.request_body_data, 2779 request_body_json=model.request_body_json, 2780 request_headers=model.request_headers, 2781 request_parameters=model.request_parameters, # type: ignore # QueryProperties have been removed in `create_simple_retriever` 2782 query_properties_key=query_properties_key, 2783 config=config, 2784 parameters=model.parameters or {}, 2785 ) 2786 2787 assert model.use_cache is not None # for mypy 2788 assert model.http_method is not None # for mypy 2789 2790 should_use_cache = (model.use_cache or bool(use_cache)) and not self._disable_cache 2791 2792 return HttpRequester( 2793 name=name, 2794 url=model.url, 2795 url_base=model.url_base, 2796 path=model.path, 2797 authenticator=authenticator, 2798 error_handler=error_handler, 2799 api_budget=api_budget, 2800 http_method=HttpMethod[model.http_method.value], 2801 request_options_provider=request_options_provider, 2802 config=config, 2803 disable_retries=self._disable_retries, 2804 parameters=model.parameters or {}, 2805 message_repository=self._message_repository, 2806 use_cache=should_use_cache, 2807 connect_timeout_in_seconds=model.connect_timeout_in_seconds, 2808 read_timeout_in_seconds=model.read_timeout_in_seconds, 2809 use_tcp_keepalive=model.use_tcp_keepalive or False, 2810 decoder=decoder, 2811 stream_response=decoder.is_stream_response() if decoder else False, 2812 )
@staticmethod
def
create_http_response_filter( model: airbyte_cdk.sources.declarative.models.declarative_component_schema.HttpResponseFilter, config: Mapping[str, Any], **kwargs: Any) -> airbyte_cdk.sources.declarative.requesters.error_handlers.HttpResponseFilter:
2814 @staticmethod 2815 def create_http_response_filter( 2816 model: HttpResponseFilterModel, config: Config, **kwargs: Any 2817 ) -> HttpResponseFilter: 2818 if model.action: 2819 action = ResponseAction(model.action.value) 2820 else: 2821 action = None 2822 2823 failure_type = FailureType(model.failure_type.value) if model.failure_type else None 2824 2825 http_codes = ( 2826 set(model.http_codes) if model.http_codes else set() 2827 ) # JSON schema notation has no set data type. The schema enforces an array of unique elements 2828 2829 return HttpResponseFilter( 2830 action=action, 2831 failure_type=failure_type, 2832 error_message=model.error_message or "", 2833 error_message_contains=model.error_message_contains or "", 2834 http_codes=http_codes, 2835 predicate=model.predicate or "", 2836 config=config, 2837 parameters=model.parameters or {}, 2838 )
@staticmethod
def
create_inline_schema_loader( model: airbyte_cdk.sources.declarative.models.declarative_component_schema.InlineSchemaLoader, config: Mapping[str, Any], **kwargs: Any) -> airbyte_cdk.sources.declarative.schema.InlineSchemaLoader:
def
create_complex_field_type( self, model: airbyte_cdk.sources.declarative.models.declarative_component_schema.ComplexFieldType, config: Mapping[str, Any], **kwargs: Any) -> airbyte_cdk.sources.declarative.schema.ComplexFieldType:
2846 def create_complex_field_type( 2847 self, model: ComplexFieldTypeModel, config: Config, **kwargs: Any 2848 ) -> ComplexFieldType: 2849 items = ( 2850 self._create_component_from_model(model=model.items, config=config) 2851 if isinstance(model.items, ComplexFieldTypeModel) 2852 else model.items 2853 ) 2854 2855 return ComplexFieldType(field_type=model.field_type, items=items)
def
create_types_map( self, model: airbyte_cdk.sources.declarative.models.declarative_component_schema.TypesMap, config: Mapping[str, Any], **kwargs: Any) -> airbyte_cdk.sources.declarative.schema.TypesMap:
2857 def create_types_map(self, model: TypesMapModel, config: Config, **kwargs: Any) -> TypesMap: 2858 target_type = ( 2859 self._create_component_from_model(model=model.target_type, config=config) 2860 if isinstance(model.target_type, ComplexFieldTypeModel) 2861 else model.target_type 2862 ) 2863 2864 return TypesMap( 2865 target_type=target_type, 2866 current_type=model.current_type, 2867 condition=model.condition if model.condition is not None else "True", 2868 )
def
create_schema_type_identifier( self, model: airbyte_cdk.sources.declarative.models.declarative_component_schema.SchemaTypeIdentifier, config: Mapping[str, Any], **kwargs: Any) -> airbyte_cdk.sources.declarative.schema.SchemaTypeIdentifier:
2870 def create_schema_type_identifier( 2871 self, model: SchemaTypeIdentifierModel, config: Config, **kwargs: Any 2872 ) -> SchemaTypeIdentifier: 2873 types_mapping = [] 2874 if model.types_mapping: 2875 types_mapping.extend( 2876 [ 2877 self._create_component_from_model(types_map, config=config) 2878 for types_map in model.types_mapping 2879 ] 2880 ) 2881 model_schema_pointer: List[Union[InterpolatedString, str]] = ( 2882 [x for x in model.schema_pointer] if model.schema_pointer else [] 2883 ) 2884 model_key_pointer: List[Union[InterpolatedString, str]] = [x for x in model.key_pointer] 2885 model_type_pointer: Optional[List[Union[InterpolatedString, str]]] = ( 2886 [x for x in model.type_pointer] if model.type_pointer else None 2887 ) 2888 2889 return SchemaTypeIdentifier( 2890 schema_pointer=model_schema_pointer, 2891 key_pointer=model_key_pointer, 2892 type_pointer=model_type_pointer, 2893 types_mapping=types_mapping, 2894 parameters=model.parameters or {}, 2895 )
def
create_dynamic_schema_loader( self, model: airbyte_cdk.sources.declarative.models.declarative_component_schema.DynamicSchemaLoader, config: Mapping[str, Any], **kwargs: Any) -> airbyte_cdk.sources.declarative.schema.DynamicSchemaLoader:
2897 def create_dynamic_schema_loader( 2898 self, model: DynamicSchemaLoaderModel, config: Config, **kwargs: Any 2899 ) -> DynamicSchemaLoader: 2900 schema_transformations = [] 2901 if model.schema_transformations: 2902 for transformation_model in model.schema_transformations: 2903 schema_transformations.append( 2904 self._create_component_from_model(model=transformation_model, config=config) 2905 ) 2906 name = "dynamic_properties" 2907 partition_router = self._build_stream_slicer_from_partition_router(model.retriever, config) 2908 retriever_kwargs: Dict[str, Any] = {} 2909 if isinstance(model.retriever, AsyncRetrieverModel): 2910 if model.retriever.partition_router: 2911 raise ValueError( 2912 "DynamicSchemaLoader does not support an AsyncRetriever with a partition_router: only the first record is read, so jobs for other partitions would be orphaned. Remove the partition_router from the schema loader's AsyncRetriever." 2913 ) 2914 # create_async_retriever requires stream_slicer; passing it to a CustomRetriever would override its own 2915 retriever_kwargs["stream_slicer"] = partition_router 2916 retriever = self._create_component_from_model( 2917 model=model.retriever, 2918 config=config, 2919 name=name, 2920 primary_key=None, 2921 partition_router=partition_router, 2922 **retriever_kwargs, 2923 transformations=[], 2924 use_cache=True, 2925 log_formatter=( 2926 lambda response: format_http_message( 2927 response, 2928 f"Schema loader '{name}' request", 2929 f"Request performed in order to extract schema.", 2930 name, 2931 is_auxiliary=True, 2932 ) 2933 ), 2934 ) 2935 schema_type_identifier = self._create_component_from_model( 2936 model.schema_type_identifier, config=config, parameters=model.parameters or {} 2937 ) 2938 schema_filter = ( 2939 self._create_component_from_model( 2940 model.schema_filter, config=config, parameters=model.parameters or {} 2941 ) 2942 if model.schema_filter is not None 2943 else None 2944 ) 2945 2946 return DynamicSchemaLoader( 2947 retriever=retriever, 2948 config=config, 2949 schema_transformations=schema_transformations, 2950 schema_filter=schema_filter, 2951 schema_type_identifier=schema_type_identifier, 2952 parameters=model.parameters or {}, 2953 )
@staticmethod
def
create_json_decoder( model: airbyte_cdk.sources.declarative.models.declarative_component_schema.JsonDecoder, config: Mapping[str, Any], **kwargs: Any) -> airbyte_cdk.Decoder:
def
create_csv_decoder( self, model: airbyte_cdk.sources.declarative.models.declarative_component_schema.CsvDecoder, config: Mapping[str, Any], **kwargs: Any) -> airbyte_cdk.Decoder:
def
create_jsonl_decoder( self, model: airbyte_cdk.sources.declarative.models.declarative_component_schema.JsonlDecoder, config: Mapping[str, Any], **kwargs: Any) -> airbyte_cdk.Decoder:
def
create_json_items_decoder( self, model: airbyte_cdk.sources.declarative.models.declarative_component_schema.JsonItemsDecoder, config: Mapping[str, Any], **kwargs: Any) -> airbyte_cdk.Decoder:
2973 def create_json_items_decoder( 2974 self, model: JsonItemsDecoderModel, config: Config, **kwargs: Any 2975 ) -> Decoder: 2976 return CompositeRawDecoder( 2977 parser=ModelToComponentFactory._get_parser(model, config), 2978 stream_response=False if self._emit_connector_builder_messages else True, 2979 )
def
create_gzip_decoder( self, model: airbyte_cdk.sources.declarative.models.declarative_component_schema.GzipDecoder, config: Mapping[str, Any], **kwargs: Any) -> airbyte_cdk.Decoder:
2981 def create_gzip_decoder( 2982 self, model: GzipDecoderModel, config: Config, **kwargs: Any 2983 ) -> Decoder: 2984 _compressed_response_types = { 2985 "gzip", 2986 "x-gzip", 2987 "gzip, deflate", 2988 "x-gzip, deflate", 2989 "application/zip", 2990 "application/gzip", 2991 "application/x-gzip", 2992 "application/x-zip-compressed", 2993 } 2994 2995 gzip_parser: GzipParser = ModelToComponentFactory._get_parser(model, config) # type: ignore # based on the model, we know this will be a GzipParser 2996 2997 if self._emit_connector_builder_messages: 2998 return CompositeRawDecoder(gzip_parser, False) 2999 3000 transport_gzip_parser = GzipParser(inner_parser=gzip_parser) 3001 return CompositeRawDecoder.by_headers( 3002 [ 3003 ({"Content-Encoding"}, {"gzip"}, transport_gzip_parser), 3004 ({"Content-Type"}, _compressed_response_types, gzip_parser), 3005 ], 3006 stream_response=True, 3007 fallback_parser=gzip_parser, 3008 )
@staticmethod
def
create_iterable_decoder( model: airbyte_cdk.sources.declarative.models.declarative_component_schema.IterableDecoder, config: Mapping[str, Any], **kwargs: Any) -> airbyte_cdk.sources.declarative.decoders.IterableDecoder:
@staticmethod
def
create_xml_decoder( model: airbyte_cdk.sources.declarative.models.declarative_component_schema.XmlDecoder, config: Mapping[str, Any], **kwargs: Any) -> airbyte_cdk.sources.declarative.decoders.XmlDecoder:
def
create_zipfile_decoder( self, model: airbyte_cdk.sources.declarative.models.declarative_component_schema.ZipfileDecoder, config: Mapping[str, Any], **kwargs: Any) -> airbyte_cdk.sources.declarative.decoders.ZipfileDecoder:
@staticmethod
def
create_json_file_schema_loader( model: airbyte_cdk.sources.declarative.models.declarative_component_schema.JsonFileSchemaLoader, config: Mapping[str, Any], **kwargs: Any) -> airbyte_cdk.JsonFileSchemaLoader:
def
create_jwt_authenticator( self, model: airbyte_cdk.sources.declarative.models.declarative_component_schema.JwtAuthenticator, config: Mapping[str, Any], **kwargs: Any) -> airbyte_cdk.sources.declarative.auth.JwtAuthenticator:
3062 def create_jwt_authenticator( 3063 self, model: JwtAuthenticatorModel, config: Config, **kwargs: Any 3064 ) -> JwtAuthenticator: 3065 jwt_headers = model.jwt_headers or JwtHeadersModel(kid=None, typ="JWT", cty=None) 3066 jwt_payload = model.jwt_payload or JwtPayloadModel(iss=None, sub=None, aud=None) 3067 request_option = ( 3068 self._create_component_from_model(model.request_option, config) 3069 if model.request_option 3070 else None 3071 ) 3072 return JwtAuthenticator( 3073 config=config, 3074 parameters=model.parameters or {}, 3075 algorithm=JwtAlgorithm(model.algorithm.value), 3076 secret_key=model.secret_key, 3077 base64_encode_secret_key=model.base64_encode_secret_key, 3078 token_duration=model.token_duration, 3079 header_prefix=model.header_prefix, 3080 kid=jwt_headers.kid, 3081 typ=jwt_headers.typ, 3082 cty=jwt_headers.cty, 3083 iss=jwt_payload.iss, 3084 sub=jwt_payload.sub, 3085 aud=jwt_payload.aud, 3086 additional_jwt_headers=model.additional_jwt_headers, 3087 additional_jwt_payload=model.additional_jwt_payload, 3088 passphrase=model.passphrase, 3089 request_option=request_option, 3090 )
def
create_list_partition_router( self, model: airbyte_cdk.sources.declarative.models.declarative_component_schema.ListPartitionRouter, config: Mapping[str, Any], **kwargs: Any) -> airbyte_cdk.sources.declarative.partition_routers.ListPartitionRouter:
3092 def create_list_partition_router( 3093 self, model: ListPartitionRouterModel, config: Config, **kwargs: Any 3094 ) -> ListPartitionRouter: 3095 request_option = ( 3096 self._create_component_from_model(model.request_option, config) 3097 if model.request_option 3098 else None 3099 ) 3100 return ListPartitionRouter( 3101 cursor_field=model.cursor_field, 3102 request_option=request_option, 3103 values=model.values, 3104 config=config, 3105 parameters=model.parameters or {}, 3106 )
@staticmethod
def
create_min_max_datetime( model: airbyte_cdk.sources.declarative.models.declarative_component_schema.MinMaxDatetime, config: Mapping[str, Any], **kwargs: Any) -> airbyte_cdk.MinMaxDatetime:
3108 @staticmethod 3109 def create_min_max_datetime( 3110 model: MinMaxDatetimeModel, config: Config, **kwargs: Any 3111 ) -> MinMaxDatetime: 3112 return MinMaxDatetime( 3113 datetime=model.datetime, 3114 datetime_format=model.datetime_format or "", 3115 max_datetime=model.max_datetime or "", 3116 min_datetime=model.min_datetime or "", 3117 parameters=model.parameters or {}, 3118 )
@staticmethod
def
create_no_auth( model: airbyte_cdk.sources.declarative.models.declarative_component_schema.NoAuth, config: Mapping[str, Any], **kwargs: Any) -> airbyte_cdk.NoAuth:
@staticmethod
def
create_no_pagination( model: airbyte_cdk.sources.declarative.models.declarative_component_schema.NoPagination, config: Mapping[str, Any], **kwargs: Any) -> airbyte_cdk.sources.declarative.requesters.paginators.NoPagination:
def
create_oauth_authenticator( self, model: airbyte_cdk.sources.declarative.models.declarative_component_schema.OAuthAuthenticator, config: Mapping[str, Any], **kwargs: Any) -> airbyte_cdk.DeclarativeOauth2Authenticator:
3130 def create_oauth_authenticator( 3131 self, model: OAuthAuthenticatorModel, config: Config, **kwargs: Any 3132 ) -> DeclarativeOauth2Authenticator: 3133 profile_assertion = ( 3134 self._create_component_from_model(model.profile_assertion, config=config) 3135 if model.profile_assertion 3136 else None 3137 ) 3138 3139 refresh_token_error_status_codes, refresh_token_error_key, refresh_token_error_values = ( 3140 self._get_refresh_token_error_information(model) 3141 ) 3142 if model.refresh_token_updater: 3143 # ignore type error because fixing it would have a lot of dependencies, revisit later 3144 return DeclarativeSingleUseRefreshTokenOauth2Authenticator( # type: ignore 3145 config, 3146 InterpolatedString.create( 3147 model.token_refresh_endpoint, # type: ignore 3148 parameters=model.parameters or {}, 3149 ).eval(config), 3150 access_token_name=InterpolatedString.create( 3151 model.access_token_name or "access_token", parameters=model.parameters or {} 3152 ).eval(config), 3153 refresh_token_name=model.refresh_token_updater.refresh_token_name, 3154 expires_in_name=InterpolatedString.create( 3155 model.expires_in_name or "expires_in", parameters=model.parameters or {} 3156 ).eval(config), 3157 client_id_name=InterpolatedString.create( 3158 model.client_id_name or "client_id", parameters=model.parameters or {} 3159 ).eval(config), 3160 client_id=InterpolatedString.create( 3161 model.client_id, parameters=model.parameters or {} 3162 ).eval(config) 3163 if model.client_id 3164 else model.client_id, 3165 client_secret_name=InterpolatedString.create( 3166 model.client_secret_name or "client_secret", parameters=model.parameters or {} 3167 ).eval(config), 3168 client_secret=InterpolatedString.create( 3169 model.client_secret, parameters=model.parameters or {} 3170 ).eval(config) 3171 if model.client_secret 3172 else model.client_secret, 3173 access_token_config_path=model.refresh_token_updater.access_token_config_path, 3174 refresh_token_config_path=model.refresh_token_updater.refresh_token_config_path, 3175 token_expiry_date_config_path=model.refresh_token_updater.token_expiry_date_config_path, 3176 grant_type_name=InterpolatedString.create( 3177 model.grant_type_name or "grant_type", parameters=model.parameters or {} 3178 ).eval(config), 3179 grant_type=InterpolatedString.create( 3180 model.grant_type or "refresh_token", parameters=model.parameters or {} 3181 ).eval(config), 3182 refresh_request_body=InterpolatedMapping( 3183 model.refresh_request_body or {}, parameters=model.parameters or {} 3184 ).eval(config), 3185 refresh_request_headers=InterpolatedMapping( 3186 model.refresh_request_headers or {}, parameters=model.parameters or {} 3187 ).eval(config), 3188 send_refresh_request_as_query_params=bool( 3189 model.send_refresh_request_as_query_params 3190 ), 3191 scopes=model.scopes, 3192 token_expiry_date_format=model.token_expiry_date_format, 3193 token_expiry_is_time_of_expiration=bool(model.token_expiry_date_format), 3194 message_repository=self._message_repository, 3195 refresh_token_error_status_codes=refresh_token_error_status_codes, 3196 refresh_token_error_key=refresh_token_error_key, 3197 refresh_token_error_values=refresh_token_error_values, 3198 ) 3199 # ignore type error because fixing it would have a lot of dependencies, revisit later 3200 return DeclarativeOauth2Authenticator( # type: ignore 3201 access_token_name=model.access_token_name or "access_token", 3202 access_token_value=model.access_token_value, 3203 client_id_name=model.client_id_name or "client_id", 3204 client_id=model.client_id, 3205 client_secret_name=model.client_secret_name or "client_secret", 3206 client_secret=model.client_secret, 3207 expires_in_name=model.expires_in_name or "expires_in", 3208 grant_type_name=model.grant_type_name or "grant_type", 3209 grant_type=model.grant_type or "refresh_token", 3210 refresh_request_body=model.refresh_request_body, 3211 refresh_request_headers=model.refresh_request_headers, 3212 send_refresh_request_as_query_params=bool(model.send_refresh_request_as_query_params), 3213 refresh_token_name=model.refresh_token_name or "refresh_token", 3214 refresh_token=model.refresh_token, 3215 scopes=model.scopes, 3216 token_expiry_date=model.token_expiry_date, 3217 token_expiry_date_format=model.token_expiry_date_format, 3218 token_expiry_is_time_of_expiration=bool(model.token_expiry_date_format), 3219 token_refresh_endpoint=model.token_refresh_endpoint, 3220 config=config, 3221 parameters=model.parameters or {}, 3222 message_repository=self._message_repository, 3223 profile_assertion=profile_assertion, 3224 use_profile_assertion=model.use_profile_assertion, 3225 refresh_token_error_status_codes=refresh_token_error_status_codes, 3226 refresh_token_error_key=refresh_token_error_key, 3227 refresh_token_error_values=refresh_token_error_values, 3228 )
def
create_offset_increment( self, model: airbyte_cdk.sources.declarative.models.declarative_component_schema.OffsetIncrement, config: Mapping[str, Any], decoder: Optional[airbyte_cdk.Decoder] = None, extractor_model: Union[airbyte_cdk.sources.declarative.models.declarative_component_schema.DpathExtractor, airbyte_cdk.sources.declarative.models.declarative_component_schema.CustomRecordExtractor, airbyte_cdk.sources.declarative.models.declarative_component_schema.CombinedExtractor, NoneType] = None, **kwargs: Any) -> airbyte_cdk.OffsetIncrement:
3337 def create_offset_increment( 3338 self, 3339 model: OffsetIncrementModel, 3340 config: Config, 3341 decoder: Optional[Decoder] = None, 3342 extractor_model: Optional[ 3343 Union[CustomRecordExtractorModel, DpathExtractorModel, CombinedExtractorModel] 3344 ] = None, 3345 **kwargs: Any, 3346 ) -> OffsetIncrement: 3347 # None when nested under a custom component, which cannot pass a decoder 3348 if decoder is None: 3349 decoder = JsonDecoder(parameters={}) 3350 if isinstance(decoder, PaginationDecoderDecorator): 3351 inner_decoder = decoder.decoder 3352 else: 3353 inner_decoder = decoder 3354 decoder = PaginationDecoderDecorator(decoder=decoder) 3355 3356 if self._is_supported_decoder_for_pagination(inner_decoder): 3357 decoder_to_use = decoder 3358 else: 3359 raise ValueError( 3360 self._UNSUPPORTED_DECODER_ERROR.format(decoder_type=type(inner_decoder)) 3361 ) 3362 3363 self._reject_union_combined_extractor_for_offset_increment(extractor_model) 3364 3365 # Pydantic v1 Union type coercion can convert int to string depending on Union order. 3366 # If page_size is a string that represents an integer (not an interpolation), convert it back. 3367 page_size = model.page_size 3368 if isinstance(page_size, str) and page_size.isdigit(): 3369 page_size = int(page_size) 3370 3371 return OffsetIncrement( 3372 page_size=page_size, 3373 config=config, 3374 decoder=decoder_to_use, 3375 inject_on_first_request=model.inject_on_first_request or False, 3376 parameters=model.parameters or {}, 3377 )
@staticmethod
def
create_page_increment( model: airbyte_cdk.sources.declarative.models.declarative_component_schema.PageIncrement, config: Mapping[str, Any], **kwargs: Any) -> airbyte_cdk.PageIncrement:
3379 @staticmethod 3380 def create_page_increment( 3381 model: PageIncrementModel, config: Config, **kwargs: Any 3382 ) -> PageIncrement: 3383 # Pydantic v1 Union type coercion can convert int to string depending on Union order. 3384 # If page_size is a string that represents an integer (not an interpolation), convert it back. 3385 page_size = model.page_size 3386 if isinstance(page_size, str) and page_size.isdigit(): 3387 page_size = int(page_size) 3388 3389 return PageIncrement( 3390 page_size=page_size, 3391 config=config, 3392 start_from_page=model.start_from_page or 0, 3393 inject_on_first_request=model.inject_on_first_request or False, 3394 parameters=model.parameters or {}, 3395 )
def
create_parent_stream_config( self, model: airbyte_cdk.sources.declarative.models.declarative_component_schema.ParentStreamConfig, config: Mapping[str, Any], *, stream_name: str, **kwargs: Any) -> airbyte_cdk.ParentStreamConfig:
3397 def create_parent_stream_config( 3398 self, model: ParentStreamConfigModel, config: Config, *, stream_name: str, **kwargs: Any 3399 ) -> ParentStreamConfig: 3400 declarative_stream = self._create_component_from_model( 3401 model.stream, 3402 config=config, 3403 is_parent=True, 3404 **kwargs, 3405 ) 3406 request_option = ( 3407 self._create_component_from_model(model.request_option, config=config) 3408 if model.request_option 3409 else None 3410 ) 3411 3412 if model.lazy_read_pointer and any("*" in pointer for pointer in model.lazy_read_pointer): 3413 raise ValueError( 3414 "The '*' wildcard in 'lazy_read_pointer' is not supported — only direct paths are allowed." 3415 ) 3416 3417 model_lazy_read_pointer: List[Union[InterpolatedString, str]] = ( 3418 [x for x in model.lazy_read_pointer] if model.lazy_read_pointer else [] 3419 ) 3420 3421 return ParentStreamConfig( 3422 parent_key=model.parent_key, 3423 request_option=request_option, 3424 stream=declarative_stream, 3425 partition_field=model.partition_field, 3426 config=config, 3427 incremental_dependency=model.incremental_dependency or False, 3428 parameters=model.parameters or {}, 3429 extra_fields=model.extra_fields, 3430 lazy_read_pointer=model_lazy_read_pointer, 3431 )
def
create_properties_from_endpoint( self, model: airbyte_cdk.sources.declarative.models.declarative_component_schema.PropertiesFromEndpoint, config: Mapping[str, Any], **kwargs: Any) -> airbyte_cdk.sources.declarative.requesters.query_properties.PropertiesFromEndpoint:
3433 def create_properties_from_endpoint( 3434 self, model: PropertiesFromEndpointModel, config: Config, **kwargs: Any 3435 ) -> PropertiesFromEndpoint: 3436 retriever = self._create_component_from_model( 3437 model=model.retriever, 3438 config=config, 3439 name="dynamic_properties", 3440 primary_key=None, 3441 stream_slicer=None, 3442 transformations=[], 3443 use_cache=True, # Enable caching on the HttpRequester/HttpClient because the properties endpoint will be called for every slice being processed, and it is highly unlikely for the response to different 3444 ) 3445 return PropertiesFromEndpoint( 3446 property_field_path=model.property_field_path, 3447 retriever=retriever, 3448 config=config, 3449 parameters=model.parameters or {}, 3450 )
def
create_property_chunking( self, model: airbyte_cdk.sources.declarative.models.declarative_component_schema.PropertyChunking, config: Mapping[str, Any], **kwargs: Any) -> airbyte_cdk.sources.declarative.requesters.query_properties.PropertyChunking:
3452 def create_property_chunking( 3453 self, model: PropertyChunkingModel, config: Config, **kwargs: Any 3454 ) -> PropertyChunking: 3455 record_merge_strategy = ( 3456 self._create_component_from_model( 3457 model=model.record_merge_strategy, config=config, **kwargs 3458 ) 3459 if model.record_merge_strategy 3460 else None 3461 ) 3462 3463 property_limit_type: PropertyLimitType 3464 match model.property_limit_type: 3465 case PropertyLimitTypeModel.property_count: 3466 property_limit_type = PropertyLimitType.property_count 3467 case PropertyLimitTypeModel.characters: 3468 property_limit_type = PropertyLimitType.characters 3469 case _: 3470 raise ValueError(f"Invalid PropertyLimitType {property_limit_type}") 3471 3472 return PropertyChunking( 3473 property_limit_type=property_limit_type, 3474 property_limit=model.property_limit, 3475 record_merge_strategy=record_merge_strategy, 3476 config=config, 3477 parameters=model.parameters or {}, 3478 )
def
create_query_properties( self, model: airbyte_cdk.sources.declarative.models.declarative_component_schema.QueryProperties, config: Mapping[str, Any], *, stream_name: str, **kwargs: Any) -> airbyte_cdk.sources.declarative.requesters.query_properties.QueryProperties:
3480 def create_query_properties( 3481 self, model: QueryPropertiesModel, config: Config, *, stream_name: str, **kwargs: Any 3482 ) -> QueryProperties: 3483 if isinstance(model.property_list, list): 3484 property_list = model.property_list 3485 else: 3486 property_list = self._create_component_from_model( 3487 model=model.property_list, config=config, **kwargs 3488 ) 3489 3490 property_chunking = ( 3491 self._create_component_from_model( 3492 model=model.property_chunking, config=config, **kwargs 3493 ) 3494 if model.property_chunking 3495 else None 3496 ) 3497 3498 property_selector = ( 3499 self._create_component_from_model( 3500 model=model.property_selector, config=config, stream_name=stream_name, **kwargs 3501 ) 3502 if model.property_selector 3503 else None 3504 ) 3505 3506 return QueryProperties( 3507 property_list=property_list, 3508 always_include_properties=model.always_include_properties, 3509 property_chunking=property_chunking, 3510 property_selector=property_selector, 3511 config=config, 3512 parameters=model.parameters or {}, 3513 )
def
create_json_schema_property_selector( self, model: airbyte_cdk.sources.declarative.models.declarative_component_schema.JsonSchemaPropertySelector, config: Mapping[str, Any], *, stream_name: str, **kwargs: Any) -> airbyte_cdk.sources.declarative.requesters.query_properties.property_selector.JsonSchemaPropertySelector:
3515 def create_json_schema_property_selector( 3516 self, 3517 model: JsonSchemaPropertySelectorModel, 3518 config: Config, 3519 *, 3520 stream_name: str, 3521 **kwargs: Any, 3522 ) -> JsonSchemaPropertySelector: 3523 configured_stream = self._stream_name_to_configured_stream.get(stream_name) 3524 3525 transformations = [] 3526 if model.transformations: 3527 for transformation_model in model.transformations: 3528 transformations.append( 3529 self._create_component_from_model(model=transformation_model, config=config) 3530 ) 3531 3532 return JsonSchemaPropertySelector( 3533 configured_stream=configured_stream, 3534 properties_transformations=transformations, 3535 config=config, 3536 parameters=model.parameters or {}, 3537 )
@staticmethod
def
create_record_filter( model: airbyte_cdk.sources.declarative.models.declarative_component_schema.RecordFilter, config: Mapping[str, Any], **kwargs: Any) -> airbyte_cdk.RecordFilter:
@staticmethod
def
create_request_path( model: airbyte_cdk.sources.declarative.models.declarative_component_schema.RequestPath, config: Mapping[str, Any], **kwargs: Any) -> airbyte_cdk.sources.declarative.requesters.request_path.RequestPath:
@staticmethod
def
create_request_option( model: airbyte_cdk.sources.declarative.models.declarative_component_schema.RequestOption, config: Mapping[str, Any], **kwargs: Any) -> airbyte_cdk.RequestOption:
3551 @staticmethod 3552 def create_request_option( 3553 model: RequestOptionModel, config: Config, **kwargs: Any 3554 ) -> RequestOption: 3555 inject_into = RequestOptionType(model.inject_into.value) 3556 field_path: Optional[List[Union[InterpolatedString, str]]] = ( 3557 [ 3558 InterpolatedString.create(segment, parameters=kwargs.get("parameters", {})) 3559 for segment in model.field_path 3560 ] 3561 if model.field_path 3562 else None 3563 ) 3564 field_name = ( 3565 InterpolatedString.create(model.field_name, parameters=kwargs.get("parameters", {})) 3566 if model.field_name 3567 else None 3568 ) 3569 return RequestOption( 3570 field_name=field_name, 3571 field_path=field_path, 3572 inject_into=inject_into, 3573 parameters=kwargs.get("parameters", {}), 3574 )
def
create_record_selector( self, model: airbyte_cdk.sources.declarative.models.declarative_component_schema.RecordSelector, config: Mapping[str, Any], *, name: str, transformations: Optional[List[airbyte_cdk.RecordTransformation]] = None, decoder: airbyte_cdk.Decoder | None = None, client_side_incremental_sync_cursor: Optional[airbyte_cdk.Cursor] = None, is_client_side_incremental_sync: bool = False, file_uploader: Optional[airbyte_cdk.sources.declarative.retrievers.file_uploader.DefaultFileUploader] = None, **kwargs: Any) -> airbyte_cdk.RecordSelector:
3576 def create_record_selector( 3577 self, 3578 model: RecordSelectorModel, 3579 config: Config, 3580 *, 3581 name: str, 3582 transformations: List[RecordTransformation] | None = None, 3583 decoder: Decoder | None = None, 3584 client_side_incremental_sync_cursor: Optional[Cursor] = None, 3585 is_client_side_incremental_sync: bool = False, 3586 file_uploader: Optional[DefaultFileUploader] = None, 3587 **kwargs: Any, 3588 ) -> RecordSelector: 3589 extractor = self._create_component_from_model( 3590 model=model.extractor, decoder=decoder, config=config 3591 ) 3592 self._count_dropped_empty_records(extractor) 3593 record_filter = ( 3594 self._create_component_from_model(model.record_filter, config=config) 3595 if model.record_filter 3596 else None 3597 ) 3598 3599 # A client-side incremental stream transforms before filtering by default. That default belongs to the flag, 3600 # not to the component that ends up doing the cursor comparison: a data feed does it in the retriever and 3601 # receives no cursor here, but its `record_filter` condition must keep running after the transformations. 3602 default_transform_before_filtering = bool( 3603 client_side_incremental_sync_cursor or is_client_side_incremental_sync 3604 ) 3605 transform_before_filtering = ( 3606 default_transform_before_filtering 3607 if model.transform_before_filtering is None 3608 else model.transform_before_filtering 3609 ) 3610 if client_side_incremental_sync_cursor: 3611 record_filter = ClientSideIncrementalRecordFilterDecorator( 3612 config=config, 3613 parameters=model.parameters, 3614 condition=model.record_filter.condition 3615 if (model.record_filter and hasattr(model.record_filter, "condition")) 3616 else None, 3617 cursor=client_side_incremental_sync_cursor, 3618 ) 3619 3620 if model.schema_normalization is None: 3621 # default to no schema normalization if not set 3622 model.schema_normalization = SchemaNormalizationModel.None_ 3623 3624 schema_normalization = ( 3625 TypeTransformer(SCHEMA_TRANSFORMER_TYPE_MAPPING[model.schema_normalization]) 3626 if isinstance(model.schema_normalization, SchemaNormalizationModel) 3627 else self._create_component_from_model(model.schema_normalization, config=config) # type: ignore[arg-type] # custom normalization model expected here 3628 ) 3629 3630 return RecordSelector( 3631 extractor=extractor, 3632 name=name, 3633 config=config, 3634 record_filter=record_filter, 3635 transformations=transformations or [], 3636 file_uploader=file_uploader, 3637 schema_normalization=schema_normalization, 3638 parameters=model.parameters or {}, 3639 transform_before_filtering=transform_before_filtering, 3640 )
@staticmethod
def
create_remove_fields( model: airbyte_cdk.sources.declarative.models.declarative_component_schema.RemoveFields, config: Mapping[str, Any], **kwargs: Any) -> airbyte_cdk.sources.declarative.transformations.RemoveFields:
def
create_selective_authenticator( self, model: airbyte_cdk.sources.declarative.models.declarative_component_schema.SelectiveAuthenticator, config: Mapping[str, Any], *, name: Optional[str] = None, url_base: Optional[str] = None, **kwargs: Any) -> airbyte_cdk.DeclarativeAuthenticator:
3650 def create_selective_authenticator( 3651 self, 3652 model: SelectiveAuthenticatorModel, 3653 config: Config, 3654 *, 3655 name: Optional[str] = None, 3656 url_base: Optional[str] = None, 3657 **kwargs: Any, 3658 ) -> DeclarativeAuthenticator: 3659 # Keyword-only so that _create_nested_component also fills them under a custom component, 3660 # from the parent's kwargs or $parameters. None is not forwarded, so a missing required one 3661 # keeps its hint. 3662 nested_kwargs = { 3663 key: value 3664 for key, value in {"name": name, "url_base": url_base}.items() 3665 if value is not None 3666 } 3667 authenticators = { 3668 key: self._create_component_from_model( 3669 model=auth, config=config, **nested_kwargs, **kwargs 3670 ) 3671 for key, auth in model.authenticators.items() 3672 } 3673 # SelectiveAuthenticator will return instance of DeclarativeAuthenticator or raise ValueError error 3674 return SelectiveAuthenticator( # type: ignore[abstract] 3675 config=config, 3676 authenticators=authenticators, 3677 authenticator_selection_path=model.authenticator_selection_path, 3678 **kwargs, 3679 )
@staticmethod
def
create_legacy_session_token_authenticator( model: airbyte_cdk.sources.declarative.models.declarative_component_schema.LegacySessionTokenAuthenticator, config: Mapping[str, Any], *, url_base: str, **kwargs: Any) -> airbyte_cdk.sources.declarative.auth.token.LegacySessionTokenAuthenticator:
3681 @staticmethod 3682 def create_legacy_session_token_authenticator( 3683 model: LegacySessionTokenAuthenticatorModel, config: Config, *, url_base: str, **kwargs: Any 3684 ) -> LegacySessionTokenAuthenticator: 3685 return LegacySessionTokenAuthenticator( 3686 api_url=url_base, 3687 header=model.header, 3688 login_url=model.login_url, 3689 password=model.password or "", 3690 session_token=model.session_token or "", 3691 session_token_response_key=model.session_token_response_key or "", 3692 username=model.username or "", 3693 validate_session_url=model.validate_session_url, 3694 config=config, 3695 parameters=model.parameters or {}, 3696 )
def
create_simple_retriever( self, model: airbyte_cdk.sources.declarative.models.declarative_component_schema.SimpleRetriever, config: Mapping[str, Any], *, name: str, primary_key: Union[str, List[str], List[List[str]], NoneType], request_options_provider: Optional[airbyte_cdk.sources.declarative.requesters.request_options.RequestOptionsProvider] = None, cursor: Optional[airbyte_cdk.Cursor] = None, has_stop_condition_cursor: bool = False, is_client_side_incremental_sync: bool = False, transformations: List[airbyte_cdk.RecordTransformation], file_uploader: Optional[airbyte_cdk.sources.declarative.retrievers.file_uploader.DefaultFileUploader] = None, incremental_sync: Union[airbyte_cdk.sources.declarative.models.declarative_component_schema.IncrementingCountCursor, airbyte_cdk.sources.declarative.models.declarative_component_schema.DatetimeBasedCursor, NoneType] = None, use_cache: Optional[bool] = None, log_formatter: Optional[Callable[[requests.models.Response], Any]] = None, partition_router: Optional[airbyte_cdk.sources.declarative.partition_routers.PartitionRouter] = None, **kwargs: Any) -> airbyte_cdk.SimpleRetriever:
3698 def create_simple_retriever( 3699 self, 3700 model: SimpleRetrieverModel, 3701 config: Config, 3702 *, 3703 name: str, 3704 primary_key: Optional[Union[str, List[str], List[List[str]]]], 3705 request_options_provider: Optional[RequestOptionsProvider] = None, 3706 cursor: Optional[Cursor] = None, 3707 has_stop_condition_cursor: bool = False, 3708 is_client_side_incremental_sync: bool = False, 3709 transformations: List[RecordTransformation], 3710 file_uploader: Optional[DefaultFileUploader] = None, 3711 incremental_sync: Optional[ 3712 Union[IncrementingCountCursorModel, DatetimeBasedCursorModel] 3713 ] = None, 3714 use_cache: Optional[bool] = None, 3715 log_formatter: Optional[Callable[[Response], Any]] = None, 3716 partition_router: Optional[PartitionRouter] = None, 3717 **kwargs: Any, 3718 ) -> SimpleRetriever: 3719 def _get_url(req: Requester) -> str: 3720 """ 3721 Closure to get the URL from the requester. This is used to get the URL in the case of a lazy retriever. 3722 This is needed because the URL is not set until the requester is created. 3723 """ 3724 3725 _url: str = ( 3726 model.requester.url 3727 if hasattr(model.requester, "url") and model.requester.url is not None 3728 else req.get_url(stream_state=None, stream_slice=None, next_page_token=None) 3729 ) 3730 _url_base: str = ( 3731 model.requester.url_base 3732 if hasattr(model.requester, "url_base") and model.requester.url_base is not None 3733 else req.get_url_base(stream_state=None, stream_slice=None, next_page_token=None) 3734 ) 3735 3736 return _url or _url_base 3737 3738 if cursor is None: 3739 cursor = FinalStateCursor(name, None, self._message_repository) 3740 3741 # A data feed drops the records the cursor considers already synced in the retriever, which sits downstream of 3742 # the paginator. Letting the record selector drop them as well would be redundant and would hide them from the 3743 # pagination stop condition, so a data feed never delegates that filtering to the record selector, whether 3744 # `is_client_side_incremental` is set or not. The `condition` from `record_filter` is intentionally left out of 3745 # the post-pagination filter and stays in the record selector, which preserves the existing behaviour: the 3746 # selector runs inside the page loop, so the records the condition rejects never reach the paginator's 3747 # accounting. Moving it downstream would start counting them. 3748 post_pagination_filter = ( 3749 ClientSideIncrementalRecordFilterDecorator( 3750 config=config, 3751 parameters=model.parameters or {}, 3752 condition=None, 3753 cursor=cursor, 3754 ) 3755 if has_stop_condition_cursor 3756 else None 3757 ) 3758 client_side_incremental_cursor = ( 3759 cursor if is_client_side_incremental_sync and not post_pagination_filter else None 3760 ) 3761 if post_pagination_filter and is_client_side_incremental_sync: 3762 LOGGER.warning( 3763 f"Stream {name}: `is_client_side_incremental` adds no record filtering when `is_data_feed` is set, " 3764 "as a data feed already filters on the cursor value. It still makes the record selector apply the " 3765 "transformations before the `record_filter` condition." 3766 ) 3767 3768 decoder = ( 3769 self._create_component_from_model(model=model.decoder, config=config) 3770 if model.decoder 3771 else JsonDecoder(parameters={}) 3772 ) 3773 record_selector = self._create_component_from_model( 3774 model=model.record_selector, 3775 name=name, 3776 config=config, 3777 decoder=decoder, 3778 transformations=transformations, 3779 client_side_incremental_sync_cursor=client_side_incremental_cursor, 3780 is_client_side_incremental_sync=is_client_side_incremental_sync, 3781 file_uploader=file_uploader, 3782 ) 3783 3784 query_properties: Optional[QueryProperties] = None 3785 query_properties_key: Optional[str] = None 3786 self._ensure_query_properties_to_model(model.requester) 3787 if self._has_query_properties_in_request_parameters(model.requester): 3788 # It is better to be explicit about an error if PropertiesFromEndpoint is defined in multiple 3789 # places instead of default to request_parameters which isn't clearly documented 3790 if ( 3791 hasattr(model.requester, "fetch_properties_from_endpoint") 3792 and model.requester.fetch_properties_from_endpoint 3793 ): 3794 raise ValueError( 3795 f"PropertiesFromEndpoint should only be specified once per stream, but found in {model.requester.type}.fetch_properties_from_endpoint and {model.requester.type}.request_parameters" 3796 ) 3797 3798 query_properties_definitions = [] 3799 for key, request_parameter in model.requester.request_parameters.items(): # type: ignore # request_parameters is already validated to be a Mapping using _has_query_properties_in_request_parameters() 3800 if isinstance(request_parameter, QueryPropertiesModel): 3801 query_properties_key = key 3802 query_properties_definitions.append(request_parameter) 3803 3804 if len(query_properties_definitions) > 1: 3805 raise ValueError( 3806 f"request_parameters only supports defining one QueryProperties field, but found {len(query_properties_definitions)} usages" 3807 ) 3808 3809 if len(query_properties_definitions) == 1: 3810 query_properties = self._create_component_from_model( 3811 model=query_properties_definitions[0], stream_name=name, config=config 3812 ) 3813 3814 # Removes QueryProperties components from the interpolated mappings because it has been designed 3815 # to be used by the SimpleRetriever and will be resolved from the provider from the slice directly 3816 # instead of through jinja interpolation 3817 if hasattr(model.requester, "request_parameters") and isinstance( 3818 model.requester.request_parameters, Mapping 3819 ): 3820 model.requester.request_parameters = self._remove_query_properties( 3821 model.requester.request_parameters 3822 ) 3823 elif ( 3824 hasattr(model.requester, "fetch_properties_from_endpoint") 3825 and model.requester.fetch_properties_from_endpoint 3826 ): 3827 # todo: Deprecate this condition once dependent connectors migrate to query_properties 3828 query_properties_definition = QueryPropertiesModel( 3829 type="QueryProperties", 3830 property_list=model.requester.fetch_properties_from_endpoint, 3831 always_include_properties=None, 3832 property_chunking=None, 3833 ) # type: ignore # $parameters has a default value 3834 3835 query_properties = self.create_query_properties( 3836 model=query_properties_definition, 3837 stream_name=name, 3838 config=config, 3839 ) 3840 elif hasattr(model.requester, "query_properties") and model.requester.query_properties: 3841 query_properties = self.create_query_properties( 3842 model=model.requester.query_properties, 3843 stream_name=name, 3844 config=config, 3845 ) 3846 3847 requester = self._create_component_from_model( 3848 model=model.requester, 3849 decoder=decoder, 3850 name=name, 3851 query_properties_key=query_properties_key, 3852 use_cache=use_cache, 3853 config=config, 3854 ) 3855 3856 if not request_options_provider: 3857 request_options_provider = DefaultRequestOptionsProvider(parameters={}) 3858 if isinstance(request_options_provider, DefaultRequestOptionsProvider) and isinstance( 3859 partition_router, PartitionRouter 3860 ): 3861 request_options_provider = partition_router 3862 3863 paginator = ( 3864 self._create_component_from_model( 3865 model=model.paginator, 3866 config=config, 3867 url_base=_get_url(requester), 3868 extractor_model=model.record_selector.extractor, 3869 decoder=decoder, 3870 cursor_used_for_stop_condition=cursor if has_stop_condition_cursor else None, 3871 ) 3872 if model.paginator 3873 else NoPagination(parameters={}) 3874 ) 3875 3876 ignore_stream_slicer_parameters_on_paginated_requests = ( 3877 model.ignore_stream_slicer_parameters_on_paginated_requests or False 3878 ) 3879 3880 reads_parent_stream_lazily = bool( 3881 model.partition_router 3882 and isinstance(model.partition_router, SubstreamPartitionRouterModel) 3883 and any( 3884 parent_stream_config.lazy_read_pointer 3885 for parent_stream_config in model.partition_router.parent_stream_configs 3886 ) 3887 ) 3888 if reads_parent_stream_lazily and ( 3889 model.page_size_reduction 3890 or self._uses_reduce_page_size_action(getattr(model.requester, "error_handler", None)) 3891 ): 3892 # Checked outside of the LazySimpleRetriever branch below, which only applies on the first 3893 # sync of a stream: gating it on the absence of state would accept the same manifest from the 3894 # second sync onwards. LazySimpleRetriever paginates the parent's embedded pages, so there is 3895 # no page of its own to re-issue with a smaller page size. 3896 raise ValueError( 3897 f"`page_size_reduction` and the REDUCE_PAGE_SIZE response action are not supported when " 3898 f"reading a parent stream lazily. Remove either the page size reduction or the parent " 3899 f"stream's `lazy_read_pointer` for stream {name}." 3900 ) 3901 3902 if reads_parent_stream_lazily and ( 3903 model.request_window_splitting 3904 or self._uses_split_request_window_action( 3905 getattr(model.requester, "error_handler", None) 3906 ) 3907 ): 3908 # Same reasoning as the page_size_reduction check above: LazySimpleRetriever reads records 3909 # embedded in the parent's own pages rather than requesting a window of its own, so there is 3910 # nothing here for a split child window to be read from. 3911 raise ValueError( 3912 f"`request_window_splitting` and the SPLIT_REQUEST_WINDOW response action are not " 3913 f"supported when reading a parent stream lazily. Remove either the request window " 3914 f"splitting or the parent stream's `lazy_read_pointer` for stream {name}." 3915 ) 3916 3917 if reads_parent_stream_lazily and not bool( 3918 self._connector_state_manager.get_stream_state(name, None) 3919 ): 3920 if incremental_sync: 3921 if incremental_sync.type != "DatetimeBasedCursor": 3922 raise ValueError( 3923 f"LazySimpleRetriever only supports DatetimeBasedCursor. Found: {incremental_sync.type}." 3924 ) 3925 3926 elif incremental_sync.step or incremental_sync.cursor_granularity: 3927 raise ValueError( 3928 f"Found more that one slice per parent. LazySimpleRetriever only supports single slice read for stream - {name}." 3929 ) 3930 3931 if model.decoder and model.decoder.type != "JsonDecoder": 3932 raise ValueError( 3933 f"LazySimpleRetriever only supports JsonDecoder. Found: {model.decoder.type}." 3934 ) 3935 3936 return LazySimpleRetriever( 3937 name=name, 3938 paginator=paginator, 3939 primary_key=primary_key, 3940 requester=requester, 3941 record_selector=record_selector, 3942 stream_slicer=_NO_STREAM_SLICING, 3943 request_option_provider=request_options_provider, 3944 config=config, 3945 ignore_stream_slicer_parameters_on_paginated_requests=ignore_stream_slicer_parameters_on_paginated_requests, 3946 post_pagination_filter=post_pagination_filter, 3947 parameters=model.parameters or {}, 3948 ) 3949 3950 if ( 3951 model.record_selector.record_filter 3952 and model.pagination_reset 3953 and model.pagination_reset.limits 3954 ): 3955 raise ValueError("PaginationResetLimits are not supported while having record filter.") 3956 3957 request_window_splitting = self._create_request_window_splitting( 3958 model, name, cursor, incremental_sync, query_properties, file_uploader 3959 ) 3960 return SimpleRetriever( 3961 name=name, 3962 paginator=paginator, 3963 primary_key=primary_key, 3964 requester=requester, 3965 record_selector=record_selector, 3966 stream_slicer=_NO_STREAM_SLICING, 3967 request_option_provider=request_options_provider, 3968 config=config, 3969 ignore_stream_slicer_parameters_on_paginated_requests=ignore_stream_slicer_parameters_on_paginated_requests, 3970 additional_query_properties=query_properties, 3971 log_formatter=self._get_log_formatter(log_formatter, name), 3972 pagination_tracker_factory=self._create_pagination_tracker_factory( 3973 model.pagination_reset, cursor 3974 ), 3975 page_size_reduction=self._create_page_size_reduction( 3976 model, name, query_properties, file_uploader 3977 ), 3978 request_window_splitting=request_window_splitting, 3979 request_window_splitter=( 3980 cursor.split_request_window 3981 if request_window_splitting and hasattr(cursor, "split_request_window") 3982 else None 3983 ), 3984 post_pagination_filter=post_pagination_filter, 3985 parameters=model.parameters or {}, 3986 )
def
create_state_delegating_stream( self, model: airbyte_cdk.sources.declarative.models.declarative_component_schema.StateDelegatingStream, config: Mapping[str, Any], **kwargs: Any) -> airbyte_cdk.sources.streams.concurrent.default_stream.DefaultStream:
4567 def create_state_delegating_stream( 4568 self, 4569 model: StateDelegatingStreamModel, 4570 config: Config, 4571 **kwargs: Any, 4572 ) -> DefaultStream: 4573 if ( 4574 model.full_refresh_stream.name != model.name 4575 or model.name != model.incremental_stream.name 4576 ): 4577 raise ValueError( 4578 f"state_delegating_stream, full_refresh_stream name and incremental_stream must have equal names. Instead has {model.name}, {model.full_refresh_stream.name} and {model.incremental_stream.name}." 4579 ) 4580 4581 # Resolve api_retention_period with config context (supports Jinja2 interpolation) 4582 resolved_retention_period: Optional[str] = None 4583 if model.api_retention_period: 4584 interpolated_retention = InterpolatedString.create( 4585 model.api_retention_period, parameters=model.parameters or {} 4586 ) 4587 resolved_value = interpolated_retention.eval(config=config) 4588 if resolved_value: 4589 resolved_retention_period = str(resolved_value) 4590 4591 if resolved_retention_period: 4592 for stream_model in (model.full_refresh_stream, model.incremental_stream): 4593 if isinstance(stream_model.incremental_sync, IncrementingCountCursorModel): 4594 raise ValueError( 4595 f"Stream '{model.name}' uses IncrementingCountCursor which is not supported " 4596 f"with api_retention_period. IncrementingCountCursor does not use datetime-based " 4597 f"cursors, so cursor age validation cannot be performed." 4598 ) 4599 4600 stream_state = self._connector_state_manager.get_stream_state(model.name, None) 4601 4602 if not stream_state: 4603 return self._create_component_from_model( # type: ignore[no-any-return] 4604 model.full_refresh_stream, config=config, **kwargs 4605 ) 4606 4607 incremental_stream: DefaultStream = self._create_component_from_model( 4608 model.incremental_stream, config=config, **kwargs 4609 ) # type: ignore[assignment] 4610 4611 # Only run cursor age validation for streams that are in the configured 4612 # catalog (or when no catalog was provided, e.g. during discover / connector 4613 # builder). Streams not selected by the user but instantiated as parent-stream 4614 # dependencies must not go through this path because it emits state messages 4615 # that the destination does not know about, causing "Stream not found" crashes. 4616 stream_is_in_catalog = ( 4617 not self._stream_name_to_configured_stream # no catalog → validate by default 4618 or model.name in self._stream_name_to_configured_stream 4619 ) 4620 if resolved_retention_period and stream_is_in_catalog: 4621 full_refresh_stream: DefaultStream = self._create_component_from_model( 4622 model.full_refresh_stream, config=config, **kwargs 4623 ) # type: ignore[assignment] 4624 if self._is_cursor_older_than_retention_period( 4625 stream_state, 4626 full_refresh_stream.cursor, 4627 incremental_stream.cursor, 4628 resolved_retention_period, 4629 model.name, 4630 ): 4631 # Clear state BEFORE constructing the full_refresh_stream so that 4632 # its cursor starts from start_date instead of the stale cursor. 4633 self._connector_state_manager.update_state_for_stream(model.name, None, {}) 4634 state_message = self._connector_state_manager.create_state_message(model.name, None) 4635 self._message_repository.emit_message(state_message) 4636 return self._create_component_from_model( # type: ignore[no-any-return] 4637 model.full_refresh_stream, config=config, **kwargs 4638 ) 4639 4640 return incremental_stream
def
create_async_retriever( self, model: airbyte_cdk.sources.declarative.models.declarative_component_schema.AsyncRetriever, config: Mapping[str, Any], *, name: str, primary_key: Union[str, List[str], List[List[str]], NoneType], stream_slicer: Optional[airbyte_cdk.sources.declarative.stream_slicers.StreamSlicer], client_side_incremental_sync: Optional[Dict[str, Any]] = None, transformations: List[airbyte_cdk.RecordTransformation], **kwargs: Any) -> airbyte_cdk.sources.declarative.retrievers.AsyncRetriever:
4739 def create_async_retriever( 4740 self, 4741 model: AsyncRetrieverModel, 4742 config: Config, 4743 *, 4744 name: str, 4745 primary_key: Optional[ 4746 Union[str, List[str], List[List[str]]] 4747 ], # this seems to be needed to match create_simple_retriever 4748 stream_slicer: Optional[StreamSlicer], 4749 client_side_incremental_sync: Optional[Dict[str, Any]] = None, 4750 transformations: List[RecordTransformation], 4751 **kwargs: Any, 4752 ) -> AsyncRetriever: 4753 if model.download_target_requester and not model.download_target_extractor: 4754 raise ValueError( 4755 f"`download_target_extractor` required if using a `download_target_requester`" 4756 ) 4757 4758 for requester_field in ( 4759 "creation_requester", 4760 "polling_requester", 4761 "download_requester", 4762 "download_target_requester", 4763 "abort_requester", 4764 "delete_requester", 4765 ): 4766 self._reject_reduce_page_size_action( 4767 getattr(model, requester_field, None), 4768 f"`{requester_field}` of the AsyncRetriever of stream {name}", 4769 ) 4770 self._reject_split_request_window_action( 4771 getattr(model, requester_field, None), 4772 f"`{requester_field}` of the AsyncRetriever of stream {name}", 4773 ) 4774 4775 def _get_download_retriever( 4776 requester: Requester, extractor: RecordExtractor, _decoder: Decoder 4777 ) -> SimpleRetriever: 4778 self._count_dropped_empty_records(extractor) 4779 # We create a record selector for the download retriever 4780 # with no schema normalization and no transformations, neither record filter 4781 # as all this occurs in the record_selector of the AsyncRetriever 4782 record_selector = RecordSelector( 4783 extractor=extractor, 4784 name=name, 4785 record_filter=None, 4786 transformations=[], 4787 schema_normalization=TypeTransformer(TransformConfig.NoTransform), 4788 config=config, 4789 parameters={}, 4790 ) 4791 paginator = ( 4792 self._create_component_from_model( 4793 model=model.download_paginator, 4794 decoder=_decoder, 4795 config=config, 4796 url_base="", 4797 # Only read by the `union` rejection of `OffsetIncrement`. 4798 extractor_model=( 4799 model.download_extractor 4800 if isinstance(model.download_extractor, CombinedExtractorModel) 4801 else None 4802 ), 4803 ) 4804 if model.download_paginator 4805 else NoPagination(parameters={}) 4806 ) 4807 4808 return SimpleRetriever( 4809 requester=requester, 4810 record_selector=record_selector, 4811 primary_key=None, 4812 name=name, 4813 paginator=paginator, 4814 config=config, 4815 parameters={}, 4816 log_formatter=self._get_log_formatter(None, name), 4817 ) 4818 4819 def _get_job_timeout() -> datetime.timedelta: 4820 user_defined_timeout: Optional[int] = ( 4821 int( 4822 InterpolatedString.create( 4823 str(model.polling_job_timeout), 4824 parameters={}, 4825 ).eval(config) 4826 ) 4827 if model.polling_job_timeout 4828 else None 4829 ) 4830 4831 # check for user defined timeout during the test read or 15 minutes 4832 test_read_timeout = datetime.timedelta(minutes=user_defined_timeout or 15) 4833 # default value for non-connector builder is 60 minutes. 4834 default_sync_timeout = datetime.timedelta(minutes=user_defined_timeout or 60) 4835 4836 return ( 4837 test_read_timeout if self._emit_connector_builder_messages else default_sync_timeout 4838 ) 4839 4840 decoder = ( 4841 self._create_component_from_model(model=model.decoder, config=config) 4842 if model.decoder 4843 else JsonDecoder(parameters={}) 4844 ) 4845 record_selector = self._create_component_from_model( 4846 model=model.record_selector, 4847 config=config, 4848 decoder=decoder, 4849 name=name, 4850 transformations=transformations, 4851 client_side_incremental_sync=client_side_incremental_sync, 4852 ) 4853 4854 stream_slicer = stream_slicer or SinglePartitionRouter(parameters={}) 4855 if self._should_limit_slices_fetched(): 4856 stream_slicer = cast( 4857 StreamSlicer, 4858 StreamSlicerTestReadDecorator( 4859 wrapped_slicer=stream_slicer, 4860 maximum_number_of_slices=self._limit_slices_fetched or 5, 4861 ), 4862 ) 4863 4864 creation_requester = self._create_component_from_model( 4865 model=model.creation_requester, 4866 decoder=decoder, 4867 config=config, 4868 name=f"job creation - {name}", 4869 ) 4870 polling_requester = self._create_component_from_model( 4871 model=model.polling_requester, 4872 decoder=decoder, 4873 config=config, 4874 name=f"job polling - {name}", 4875 ) 4876 job_download_components_name = f"job download - {name}" 4877 download_decoder = ( 4878 self._create_component_from_model(model=model.download_decoder, config=config) 4879 if model.download_decoder 4880 else JsonDecoder(parameters={}) 4881 ) 4882 download_extractor = ( 4883 self._create_component_from_model( 4884 model=model.download_extractor, 4885 config=config, 4886 decoder=download_decoder, 4887 parameters=model.parameters, 4888 ) 4889 if model.download_extractor 4890 else DpathExtractor( 4891 [], 4892 config=config, 4893 decoder=download_decoder, 4894 parameters=model.parameters or {}, 4895 ) 4896 ) 4897 download_requester = self._create_component_from_model( 4898 model=model.download_requester, 4899 decoder=download_decoder, 4900 config=config, 4901 name=job_download_components_name, 4902 ) 4903 download_retriever = _get_download_retriever( 4904 download_requester, download_extractor, download_decoder 4905 ) 4906 abort_requester = ( 4907 self._create_component_from_model( 4908 model=model.abort_requester, 4909 decoder=decoder, 4910 config=config, 4911 name=f"job abort - {name}", 4912 ) 4913 if model.abort_requester 4914 else None 4915 ) 4916 delete_requester = ( 4917 self._create_component_from_model( 4918 model=model.delete_requester, 4919 decoder=decoder, 4920 config=config, 4921 name=f"job delete - {name}", 4922 ) 4923 if model.delete_requester 4924 else None 4925 ) 4926 download_target_requester = ( 4927 self._create_component_from_model( 4928 model=model.download_target_requester, 4929 decoder=decoder, 4930 config=config, 4931 name=f"job extract_url - {name}", 4932 ) 4933 if model.download_target_requester 4934 else None 4935 ) 4936 status_extractor = self._create_component_from_model( 4937 model=model.status_extractor, decoder=decoder, config=config, name=name 4938 ) 4939 download_target_extractor = ( 4940 self._create_component_from_model( 4941 model=model.download_target_extractor, 4942 decoder=decoder, 4943 config=config, 4944 name=name, 4945 ) 4946 if model.download_target_extractor 4947 else None 4948 ) 4949 4950 job_repository: AsyncJobRepository = AsyncHttpJobRepository( 4951 creation_requester=creation_requester, 4952 polling_requester=polling_requester, 4953 download_retriever=download_retriever, 4954 download_target_requester=download_target_requester, 4955 abort_requester=abort_requester, 4956 delete_requester=delete_requester, 4957 status_extractor=status_extractor, 4958 status_mapping=self._create_async_job_status_mapping(model.status_mapping, config), 4959 download_target_extractor=download_target_extractor, 4960 job_timeout=_get_job_timeout(), 4961 ) 4962 4963 failed_retry_wait_time_in_seconds: Optional[int] = ( 4964 int( 4965 InterpolatedString.create( 4966 str(model.failed_retry_wait_time_in_seconds), 4967 parameters={}, 4968 ).eval(config) 4969 ) 4970 if model.failed_retry_wait_time_in_seconds 4971 else None 4972 ) 4973 4974 async_job_partition_router = AsyncJobPartitionRouter( 4975 job_orchestrator_factory=lambda stream_slices: AsyncJobOrchestrator( 4976 job_repository, 4977 stream_slices, 4978 self._job_tracker, 4979 self._message_repository, 4980 # FIXME work would need to be done here in order to detect if a stream as a parent stream that is bulk 4981 has_bulk_parent=False, 4982 # set the `job_max_retry` to 1 for the `Connector Builder`` use-case. 4983 # `None` == default retry is set to 3 attempts, under the hood. 4984 job_max_retry=1 if self._emit_connector_builder_messages else None, 4985 failed_retry_wait_time_in_seconds=failed_retry_wait_time_in_seconds, 4986 ), 4987 stream_slicer=stream_slicer, 4988 config=config, 4989 parameters=model.parameters or {}, 4990 ) 4991 4992 return AsyncRetriever( 4993 record_selector=record_selector, 4994 stream_slicer=async_job_partition_router, 4995 config=config, 4996 parameters=model.parameters or {}, 4997 )
def
create_spec( self, model: airbyte_cdk.sources.declarative.models.declarative_component_schema.Spec, config: Mapping[str, Any], **kwargs: Any) -> airbyte_cdk.sources.declarative.spec.Spec:
4999 def create_spec(self, model: SpecModel, config: Config, **kwargs: Any) -> Spec: 5000 config_migrations = [ 5001 self._create_component_from_model(migration, config) 5002 for migration in ( 5003 model.config_normalization_rules.config_migrations 5004 if ( 5005 model.config_normalization_rules 5006 and model.config_normalization_rules.config_migrations 5007 ) 5008 else [] 5009 ) 5010 ] 5011 config_transformations = [ 5012 self._create_component_from_model(transformation, config) 5013 for transformation in ( 5014 model.config_normalization_rules.transformations 5015 if ( 5016 model.config_normalization_rules 5017 and model.config_normalization_rules.transformations 5018 ) 5019 else [] 5020 ) 5021 ] 5022 config_validations = [ 5023 self._create_component_from_model(validation, config) 5024 for validation in ( 5025 model.config_normalization_rules.validations 5026 if ( 5027 model.config_normalization_rules 5028 and model.config_normalization_rules.validations 5029 ) 5030 else [] 5031 ) 5032 ] 5033 5034 return Spec( 5035 connection_specification=model.connection_specification, 5036 documentation_url=model.documentation_url, 5037 advanced_auth=model.advanced_auth, 5038 parameters={}, 5039 config_migrations=config_migrations, 5040 config_transformations=config_transformations, 5041 config_validations=config_validations, 5042 )
def
create_substream_partition_router( self, model: airbyte_cdk.sources.declarative.models.declarative_component_schema.SubstreamPartitionRouter, config: Mapping[str, Any], *, stream_name: str, **kwargs: Any) -> airbyte_cdk.SubstreamPartitionRouter:
5044 def create_substream_partition_router( 5045 self, 5046 model: SubstreamPartitionRouterModel, 5047 config: Config, 5048 *, 5049 stream_name: str, 5050 **kwargs: Any, 5051 ) -> SubstreamPartitionRouter: 5052 parent_stream_configs = [] 5053 if model.parent_stream_configs: 5054 parent_stream_configs.extend( 5055 [ 5056 self.create_parent_stream_config_with_substream_wrapper( 5057 model=parent_stream_config, config=config, stream_name=stream_name, **kwargs 5058 ) 5059 for parent_stream_config in model.parent_stream_configs 5060 ] 5061 ) 5062 5063 return SubstreamPartitionRouter( 5064 parent_stream_configs=parent_stream_configs, 5065 parameters=model.parameters or {}, 5066 config=config, 5067 )
def
create_parent_stream_config_with_substream_wrapper( self, model: airbyte_cdk.sources.declarative.models.declarative_component_schema.ParentStreamConfig, config: Mapping[str, Any], *, stream_name: str, **kwargs: Any) -> Any:
5069 def create_parent_stream_config_with_substream_wrapper( 5070 self, model: ParentStreamConfigModel, config: Config, *, stream_name: str, **kwargs: Any 5071 ) -> Any: 5072 child_state = self._connector_state_manager.get_stream_state(stream_name, None) 5073 if NO_CURSOR_STATE_KEY in child_state: 5074 # Full refresh streams checkpoint a `{NO_CURSOR_STATE_KEY: true}` sentinel. When such a 5075 # stream is later converted to incremental with an incremental_dependency parent, 5076 # `_instantiate_parent_stream_state_manager` would treat the sentinel's boolean as a legacy 5077 # cursor value and re-key it under the parent's cursor field, crashing cursor initialization. 5078 child_state = { 5079 key: value for key, value in child_state.items() if key != NO_CURSOR_STATE_KEY 5080 } 5081 5082 parent_state: Optional[Mapping[str, Any]] = ( 5083 child_state if model.incremental_dependency and child_state else None 5084 ) 5085 connector_state_manager = self._instantiate_parent_stream_state_manager( 5086 child_state, config, model, parent_state 5087 ) 5088 5089 substream_factory = ModelToComponentFactory( 5090 custom_components_trusted=self._custom_components_trusted, 5091 connector_state_manager=connector_state_manager, 5092 limit_pages_fetched_per_slice=self._limit_pages_fetched_per_slice, 5093 limit_slices_fetched=self._limit_slices_fetched, 5094 emit_connector_builder_messages=self._emit_connector_builder_messages, 5095 disable_retries=self._disable_retries, 5096 disable_cache=self._disable_cache, 5097 message_repository=StateFilteringMessageRepository( 5098 LogAppenderMessageRepositoryDecorator( 5099 { 5100 "airbyte_cdk": {"stream": {"is_substream": True}}, 5101 "http": {"is_auxiliary": True}, 5102 }, 5103 self._message_repository, 5104 self._evaluate_log_level(self._emit_connector_builder_messages), 5105 ), 5106 ), 5107 api_budget=self._api_budget, 5108 # Share the authenticator registry so parent and child streams draw from the 5109 # same token quota counters 5110 rate_limited_authenticators=self._rate_limited_authenticators, 5111 ) 5112 5113 return substream_factory.create_parent_stream_config( 5114 model=model, config=config, stream_name=stream_name, **kwargs 5115 )
@staticmethod
def
create_wait_time_from_header( model: airbyte_cdk.sources.declarative.models.declarative_component_schema.WaitTimeFromHeader, config: Mapping[str, Any], **kwargs: Any) -> airbyte_cdk.sources.declarative.requesters.error_handlers.backoff_strategies.WaitTimeFromHeaderBackoffStrategy:
5180 @staticmethod 5181 def create_wait_time_from_header( 5182 model: WaitTimeFromHeaderModel, config: Config, **kwargs: Any 5183 ) -> WaitTimeFromHeaderBackoffStrategy: 5184 return WaitTimeFromHeaderBackoffStrategy( 5185 header=model.header, 5186 parameters=model.parameters or {}, 5187 config=config, 5188 regex=model.regex, 5189 max_waiting_time_in_seconds=model.max_waiting_time_in_seconds, 5190 )
@staticmethod
def
create_wait_until_time_from_header( model: airbyte_cdk.sources.declarative.models.declarative_component_schema.WaitUntilTimeFromHeader, config: Mapping[str, Any], **kwargs: Any) -> airbyte_cdk.sources.declarative.requesters.error_handlers.backoff_strategies.WaitUntilTimeFromHeaderBackoffStrategy:
5192 @staticmethod 5193 def create_wait_until_time_from_header( 5194 model: WaitUntilTimeFromHeaderModel, config: Config, **kwargs: Any 5195 ) -> WaitUntilTimeFromHeaderBackoffStrategy: 5196 return WaitUntilTimeFromHeaderBackoffStrategy( 5197 header=model.header, 5198 parameters=model.parameters or {}, 5199 config=config, 5200 min_wait=model.min_wait, 5201 regex=model.regex, 5202 max_waiting_time_in_seconds=model.max_waiting_time_in_seconds, 5203 )
@staticmethod
def
create_components_mapping_definition( model: airbyte_cdk.sources.declarative.models.declarative_component_schema.ComponentMappingDefinition, config: Mapping[str, Any], **kwargs: Any) -> airbyte_cdk.sources.declarative.resolvers.ComponentMappingDefinition:
5211 @staticmethod 5212 def create_components_mapping_definition( 5213 model: ComponentMappingDefinitionModel, config: Config, **kwargs: Any 5214 ) -> ComponentMappingDefinition: 5215 interpolated_value = InterpolatedString.create( 5216 model.value, parameters=model.parameters or {} 5217 ) 5218 field_path = [ 5219 InterpolatedString.create(path, parameters=model.parameters or {}) 5220 for path in model.field_path 5221 ] 5222 return ComponentMappingDefinition( 5223 field_path=field_path, # type: ignore[arg-type] # field_path can be str and InterpolatedString 5224 value=interpolated_value, 5225 value_type=ModelToComponentFactory._json_schema_type_name_to_type(model.value_type), 5226 create_or_update=model.create_or_update, 5227 condition=model.condition, 5228 parameters=model.parameters or {}, 5229 )
def
create_http_components_resolver( self, model: airbyte_cdk.sources.declarative.models.declarative_component_schema.HttpComponentsResolver, config: Mapping[str, Any], stream_name: Optional[str] = None) -> Any:
5231 def create_http_components_resolver( 5232 self, model: HttpComponentsResolverModel, config: Config, stream_name: Optional[str] = None 5233 ) -> Any: 5234 partition_router = self._build_stream_slicer_from_partition_router(model.retriever, config) 5235 retriever = self._create_component_from_model( 5236 model=model.retriever, 5237 config=config, 5238 name=f"{stream_name if stream_name else '__http_components_resolver'}", 5239 primary_key=None, 5240 stream_slicer=partition_router, 5241 partition_router=partition_router, 5242 transformations=[], 5243 ) 5244 5245 components_mapping = [] 5246 for component_mapping_definition_model in model.components_mapping: 5247 if component_mapping_definition_model.condition: 5248 raise ValueError("`condition` is only supported for `ConfigComponentsResolver`") 5249 components_mapping.append( 5250 self._create_component_from_model( 5251 model=component_mapping_definition_model, 5252 value_type=ModelToComponentFactory._json_schema_type_name_to_type( 5253 component_mapping_definition_model.value_type 5254 ), 5255 config=config, 5256 ) 5257 ) 5258 5259 return HttpComponentsResolver( 5260 retriever=retriever, 5261 # AsyncRetriever reads records only from the job slices its own slicer yields 5262 stream_slicer=retriever.stream_slicer 5263 if isinstance(retriever, AsyncRetriever) 5264 else partition_router, 5265 config=config, 5266 components_mapping=components_mapping, 5267 parameters=model.parameters or {}, 5268 )
@staticmethod
def
create_stream_config( model: airbyte_cdk.sources.declarative.models.declarative_component_schema.StreamConfig, config: Mapping[str, Any], **kwargs: Any) -> airbyte_cdk.sources.declarative.resolvers.StreamConfig:
5270 @staticmethod 5271 def create_stream_config( 5272 model: StreamConfigModel, config: Config, **kwargs: Any 5273 ) -> StreamConfig: 5274 model_configs_pointer: List[Union[InterpolatedString, str]] = ( 5275 [x for x in model.configs_pointer] if model.configs_pointer else [] 5276 ) 5277 5278 return StreamConfig( 5279 configs_pointer=model_configs_pointer, 5280 default_values=model.default_values, 5281 parameters=model.parameters or {}, 5282 )
def
create_config_components_resolver( self, model: airbyte_cdk.sources.declarative.models.declarative_component_schema.ConfigComponentsResolver, config: Mapping[str, Any]) -> Any:
5284 def create_config_components_resolver( 5285 self, 5286 model: ConfigComponentsResolverModel, 5287 config: Config, 5288 ) -> Any: 5289 model_stream_configs = ( 5290 model.stream_config if isinstance(model.stream_config, list) else [model.stream_config] 5291 ) 5292 5293 stream_configs = [ 5294 self._create_component_from_model( 5295 stream_config, config=config, parameters=model.parameters or {} 5296 ) 5297 for stream_config in model_stream_configs 5298 ] 5299 5300 components_mapping = [ 5301 self._create_component_from_model( 5302 model=components_mapping_definition_model, 5303 value_type=ModelToComponentFactory._json_schema_type_name_to_type( 5304 components_mapping_definition_model.value_type 5305 ), 5306 config=config, 5307 parameters=model.parameters, 5308 ) 5309 for components_mapping_definition_model in model.components_mapping 5310 ] 5311 5312 return ConfigComponentsResolver( 5313 stream_configs=stream_configs, 5314 config=config, 5315 components_mapping=components_mapping, 5316 parameters=model.parameters or {}, 5317 )
def
create_parametrized_components_resolver( self, model: airbyte_cdk.sources.declarative.models.declarative_component_schema.ParametrizedComponentsResolver, config: Mapping[str, Any]) -> airbyte_cdk.sources.declarative.resolvers.ParametrizedComponentsResolver:
5319 def create_parametrized_components_resolver( 5320 self, 5321 model: ParametrizedComponentsResolverModel, 5322 config: Config, 5323 ) -> ParametrizedComponentsResolver: 5324 stream_parameters = StreamParametersDefinition( 5325 list_of_parameters_for_stream=model.stream_parameters.list_of_parameters_for_stream 5326 ) 5327 5328 components_mapping = [] 5329 for components_mapping_definition_model in model.components_mapping: 5330 if components_mapping_definition_model.condition: 5331 raise ValueError("`condition` is only supported for `ConfigComponentsResolver`") 5332 components_mapping.append( 5333 self._create_component_from_model( 5334 model=components_mapping_definition_model, 5335 value_type=ModelToComponentFactory._json_schema_type_name_to_type( 5336 components_mapping_definition_model.value_type 5337 ), 5338 config=config, 5339 ) 5340 ) 5341 return ParametrizedComponentsResolver( 5342 stream_parameters=stream_parameters, 5343 config=config, 5344 components_mapping=components_mapping, 5345 parameters=model.parameters or {}, 5346 )
def
create_http_api_budget( self, model: airbyte_cdk.sources.declarative.models.declarative_component_schema.HTTPAPIBudget, config: Mapping[str, Any], **kwargs: Any) -> airbyte_cdk.HttpAPIBudget:
5370 def create_http_api_budget( 5371 self, model: HTTPAPIBudgetModel, config: Config, **kwargs: Any 5372 ) -> HttpAPIBudget: 5373 policies = [ 5374 self._create_component_from_model(model=policy, config=config) 5375 for policy in model.policies 5376 ] 5377 5378 return HttpAPIBudget( 5379 policies=policies, 5380 ratelimit_reset_header=model.ratelimit_reset_header or "ratelimit-reset", 5381 ratelimit_remaining_header=model.ratelimit_remaining_header or "ratelimit-remaining", 5382 status_codes_for_ratelimit_hit=model.status_codes_for_ratelimit_hit or [429], 5383 )
def
create_fixed_window_call_rate_policy( self, model: airbyte_cdk.sources.declarative.models.declarative_component_schema.FixedWindowCallRatePolicy, config: Mapping[str, Any], **kwargs: Any) -> airbyte_cdk.sources.streams.call_rate.FixedWindowCallRatePolicy:
5385 def create_fixed_window_call_rate_policy( 5386 self, model: FixedWindowCallRatePolicyModel, config: Config, **kwargs: Any 5387 ) -> FixedWindowCallRatePolicy: 5388 matchers = [ 5389 self._create_component_from_model(model=matcher, config=config) 5390 for matcher in model.matchers 5391 ] 5392 5393 # Set the initial reset timestamp to 10 days from now. 5394 # This value will be updated by the first request. 5395 return FixedWindowCallRatePolicy( 5396 next_reset_ts=datetime.datetime.now() + datetime.timedelta(days=10), 5397 period=parse_duration(model.period), 5398 call_limit=model.call_limit, 5399 matchers=matchers, 5400 )
def
create_file_uploader( self, model: airbyte_cdk.sources.declarative.models.declarative_component_schema.FileUploader, config: Mapping[str, Any], **kwargs: Any) -> airbyte_cdk.sources.declarative.retrievers.file_uploader.FileUploader:
5402 def create_file_uploader( 5403 self, model: FileUploaderModel, config: Config, **kwargs: Any 5404 ) -> FileUploader: 5405 self._reject_reduce_page_size_action(model.requester, "requester of a `file_uploader`") 5406 self._reject_split_request_window_action(model.requester, "requester of a `file_uploader`") 5407 name = "File Uploader" 5408 requester = self._create_component_from_model( 5409 model=model.requester, 5410 config=config, 5411 name=name, 5412 **kwargs, 5413 ) 5414 download_target_extractor = self._create_component_from_model( 5415 model=model.download_target_extractor, 5416 config=config, 5417 name=name, 5418 **kwargs, 5419 ) 5420 emit_connector_builder_messages = self._emit_connector_builder_messages 5421 file_uploader = DefaultFileUploader( 5422 requester=requester, 5423 download_target_extractor=download_target_extractor, 5424 config=config, 5425 file_writer=NoopFileWriter() 5426 if emit_connector_builder_messages 5427 else LocalFileSystemFileWriter(), 5428 parameters=model.parameters or {}, 5429 filename_extractor=model.filename_extractor if model.filename_extractor else None, 5430 ) 5431 5432 return ( 5433 ConnectorBuilderFileUploader(file_uploader) 5434 if emit_connector_builder_messages 5435 else file_uploader 5436 )
def
create_moving_window_call_rate_policy( self, model: airbyte_cdk.sources.declarative.models.declarative_component_schema.MovingWindowCallRatePolicy, config: Mapping[str, Any], **kwargs: Any) -> airbyte_cdk.MovingWindowCallRatePolicy:
5438 def create_moving_window_call_rate_policy( 5439 self, model: MovingWindowCallRatePolicyModel, config: Config, **kwargs: Any 5440 ) -> MovingWindowCallRatePolicy: 5441 rates = [ 5442 self._create_component_from_model(model=rate, config=config) for rate in model.rates 5443 ] 5444 matchers = [ 5445 self._create_component_from_model(model=matcher, config=config) 5446 for matcher in model.matchers 5447 ] 5448 return MovingWindowCallRatePolicy( 5449 rates=rates, 5450 matchers=matchers, 5451 )
def
create_unlimited_call_rate_policy( self, model: airbyte_cdk.sources.declarative.models.declarative_component_schema.UnlimitedCallRatePolicy, config: Mapping[str, Any], **kwargs: Any) -> airbyte_cdk.sources.streams.call_rate.UnlimitedCallRatePolicy:
5453 def create_unlimited_call_rate_policy( 5454 self, model: UnlimitedCallRatePolicyModel, config: Config, **kwargs: Any 5455 ) -> UnlimitedCallRatePolicy: 5456 matchers = [ 5457 self._create_component_from_model(model=matcher, config=config) 5458 for matcher in model.matchers 5459 ] 5460 5461 return UnlimitedCallRatePolicy( 5462 matchers=matchers, 5463 )
def
create_rate( self, model: airbyte_cdk.sources.declarative.models.declarative_component_schema.Rate, config: Mapping[str, Any], **kwargs: Any) -> airbyte_cdk.Rate:
def
create_http_request_matcher( self, model: airbyte_cdk.sources.declarative.models.declarative_component_schema.HttpRequestRegexMatcher, config: Mapping[str, Any], **kwargs: Any) -> airbyte_cdk.sources.streams.call_rate.HttpRequestRegexMatcher:
5472 def create_http_request_matcher( 5473 self, model: HttpRequestRegexMatcherModel, config: Config, **kwargs: Any 5474 ) -> HttpRequestRegexMatcher: 5475 weight = model.weight 5476 if weight is not None: 5477 if isinstance(weight, str): 5478 weight = int(InterpolatedString.create(weight, parameters={}).eval(config)) 5479 else: 5480 weight = int(weight) 5481 if weight < 1: 5482 raise ValueError(f"weight must be >= 1, got {weight}") 5483 return HttpRequestRegexMatcher( 5484 method=model.method, 5485 url_base=model.url_base, 5486 url_path_pattern=model.url_path_pattern, 5487 params=model.params, 5488 headers=model.headers, 5489 weight=weight, 5490 )
def
create_rate_limited_multiple_token_authenticator( self, model: airbyte_cdk.sources.declarative.models.declarative_component_schema.RateLimitedMultipleTokenAuthenticator, config: Mapping[str, Any], **kwargs: Any) -> airbyte_cdk.sources.declarative.auth.RateLimitedMultipleTokenAuthenticator:
5492 def create_rate_limited_multiple_token_authenticator( 5493 self, 5494 model: RateLimitedMultipleTokenAuthenticatorModel, 5495 config: Config, 5496 **kwargs: Any, 5497 ) -> RateLimitedMultipleTokenAuthenticator: 5498 if isinstance(model.tokens, str): 5499 tokens_value = InterpolatedString.create(model.tokens, parameters={}).eval(config) 5500 delimiter = model.token_delimiter or "," 5501 tokens = [ 5502 token.strip() for token in str(tokens_value).split(delimiter) if token.strip() 5503 ] 5504 else: 5505 tokens = [ 5506 token_value 5507 for token in model.tokens 5508 if ( 5509 token_value := str( 5510 InterpolatedString.create(token, parameters={}).eval(config) 5511 ).strip() 5512 ) 5513 ] 5514 5515 quota_specs = [ 5516 { 5517 "name": quota_model.name, 5518 "remaining_path": quota_model.remaining_path, 5519 "reset_path": quota_model.reset_path, 5520 "limit_path": quota_model.limit_path, 5521 "remaining_header": quota_model.remaining_header, 5522 "reset_header": quota_model.reset_header, 5523 "limit_header": quota_model.limit_header, 5524 # Normalize the same way as the runtime TokenQuota below, so an omitted field 5525 # and an explicit `[]` key identically and keep sharing one set of counters. 5526 "exhaustion_status_codes": quota_model.exhaustion_status_codes or [], 5527 "matchers": [ 5528 { 5529 "method": matcher_model.method, 5530 "url_base": matcher_model.url_base, 5531 "url_path_pattern": matcher_model.url_path_pattern, 5532 "params": matcher_model.params, 5533 "headers": matcher_model.headers, 5534 "weight": matcher_model.weight, 5535 } 5536 for matcher_model in quota_model.matchers or [] 5537 ], 5538 } 5539 for quota_model in model.quotas 5540 ] 5541 5542 quota_status_url = str( 5543 InterpolatedString.create(model.quota_status_source.url, parameters={}).eval(config) 5544 ) 5545 quota_status_http_method = ( 5546 model.quota_status_source.http_method.value 5547 if model.quota_status_source.http_method 5548 else "GET" 5549 ) 5550 quota_status_headers = { 5551 key: str(InterpolatedString.create(value, parameters={}).eval(config)) 5552 for key, value in (model.quota_status_source.request_headers or {}).items() 5553 } 5554 # Normalize the same way as the quota specs above, so an omitted field and an explicit 5555 # `[]` key identically and keep sharing one set of counters. Deduplicated as well as 5556 # sorted, because the runtime turns this into a set: without it `[404]` and `[404, 404]` 5557 # would key differently and stop sharing counters while behaving identically. 5558 quota_status_unavailable_status_codes = sorted( 5559 set(model.quota_status_source.unavailable_status_codes or []) 5560 ) 5561 auth_method = model.auth_method or "Bearer" 5562 header = model.header or "Authorization" 5563 max_wait_time_str = str( 5564 InterpolatedString.create(model.max_wait_time or "PT2H", parameters={}).eval(config) 5565 ) 5566 max_wait_time = parse_duration(max_wait_time_str) 5567 if not isinstance(max_wait_time, datetime.timedelta): 5568 raise ValueError( 5569 f"max_wait_time must be a fixed-length ISO 8601 duration (e.g. 'PT2H'); " 5570 f"calendar-unit durations like '{max_wait_time_str}' are not supported" 5571 ) 5572 budget_reserve_fraction = ( 5573 model.budget_reserve_fraction if model.budget_reserve_fraction is not None else 0.1 5574 ) 5575 budget_min_reserve = ( 5576 model.budget_min_reserve if model.budget_min_reserve is not None else 50 5577 ) 5578 5579 # Reuse the same instance for identical definitions so that all streams share the 5580 # same token quota counters (similar to how api_budget is shared). The key is built 5581 # from the resolved constructor arguments rather than the raw model so that 5582 # stream-specific `$parameters` propagated onto the model (and its nested components) 5583 # cannot break instance sharing. 5584 cache_key = json.dumps( 5585 { 5586 "tokens": tokens, 5587 "quotas": quota_specs, 5588 "quota_status_url": quota_status_url, 5589 "quota_status_http_method": quota_status_http_method, 5590 "quota_status_headers": quota_status_headers, 5591 "quota_status_unavailable_status_codes": quota_status_unavailable_status_codes, 5592 "auth_method": auth_method, 5593 "header": header, 5594 "max_wait_time": max_wait_time.total_seconds(), 5595 "budget_reserve_fraction": budget_reserve_fraction, 5596 "budget_min_reserve": budget_min_reserve, 5597 }, 5598 sort_keys=True, 5599 ) 5600 if cache_key in self._rate_limited_authenticators: 5601 return self._rate_limited_authenticators[cache_key] 5602 5603 quotas = [ 5604 TokenQuota( 5605 name=quota_model.name, 5606 remaining_path=quota_model.remaining_path, 5607 reset_path=quota_model.reset_path, 5608 limit_path=quota_model.limit_path, 5609 remaining_header=quota_model.remaining_header, 5610 reset_header=quota_model.reset_header, 5611 limit_header=quota_model.limit_header, 5612 exhaustion_status_codes=quota_model.exhaustion_status_codes or [], 5613 matchers=[ 5614 self.create_http_request_matcher(matcher_model, config) 5615 for matcher_model in quota_model.matchers or [] 5616 ], 5617 ) 5618 for quota_model in model.quotas 5619 ] 5620 5621 authenticator = RateLimitedMultipleTokenAuthenticator( 5622 tokens=tokens, 5623 quotas=quotas, 5624 quota_status_url=quota_status_url, 5625 quota_status_http_method=quota_status_http_method, 5626 quota_status_headers=quota_status_headers, 5627 quota_status_unavailable_status_codes=quota_status_unavailable_status_codes, 5628 auth_method=auth_method, 5629 header=header, 5630 max_wait_time=max_wait_time, 5631 budget_reserve_fraction=budget_reserve_fraction, 5632 budget_min_reserve=budget_min_reserve, 5633 ) 5634 self._rate_limited_authenticators[cache_key] = authenticator 5635 return authenticator
def
set_api_budget( self, component_definition: Mapping[str, Any], config: Mapping[str, Any]) -> None:
def
create_grouping_partition_router( self, model: airbyte_cdk.sources.declarative.models.declarative_component_schema.GroupingPartitionRouter, config: Mapping[str, Any], *, stream_name: str, **kwargs: Any) -> airbyte_cdk.sources.declarative.partition_routers.GroupingPartitionRouter:
5642 def create_grouping_partition_router( 5643 self, 5644 model: GroupingPartitionRouterModel, 5645 config: Config, 5646 *, 5647 stream_name: str, 5648 **kwargs: Any, 5649 ) -> GroupingPartitionRouter: 5650 underlying_router = self._create_component_from_model( 5651 model=model.underlying_partition_router, 5652 config=config, 5653 stream_name=stream_name, 5654 **kwargs, 5655 ) 5656 if model.group_size < 1: 5657 raise ValueError(f"Group size must be greater than 0, got {model.group_size}") 5658 5659 # Request options in underlying partition routers are not supported for GroupingPartitionRouter 5660 # because they are specific to individual partitions and cannot be aggregated or handled 5661 # when grouping, potentially leading to incorrect API calls. Any request customization 5662 # should be managed at the stream level through the requester's configuration. 5663 if isinstance(underlying_router, SubstreamPartitionRouter): 5664 if any( 5665 parent_config.request_option 5666 for parent_config in underlying_router.parent_stream_configs 5667 ): 5668 raise ValueError("Request options are not supported for GroupingPartitionRouter.") 5669 5670 if isinstance(underlying_router, ListPartitionRouter): 5671 if underlying_router.request_option: 5672 raise ValueError("Request options are not supported for GroupingPartitionRouter.") 5673 5674 return GroupingPartitionRouter( 5675 group_size=model.group_size, 5676 underlying_partition_router=underlying_router, 5677 deduplicate=model.deduplicate if model.deduplicate is not None else True, 5678 config=config, 5679 )
def
create_union_partition_router( self, model: airbyte_cdk.sources.declarative.models.declarative_component_schema.UnionPartitionRouter, config: Mapping[str, Any], *, stream_name: str, **kwargs: Any) -> airbyte_cdk.UnionPartitionRouter:
5681 def create_union_partition_router( 5682 self, 5683 model: UnionPartitionRouterModel, 5684 config: Config, 5685 *, 5686 stream_name: str, 5687 **kwargs: Any, 5688 ) -> UnionPartitionRouter: 5689 # The schema enforces minItems: 2 for manifests; this guard covers construction paths 5690 # that bypass JSON-schema validation (the generated model carries no min_items constraint). 5691 if len(model.partition_routers) < 2: 5692 raise ValueError( 5693 f"UnionPartitionRouter for stream {stream_name} needs at least 2 child partition routers" 5694 ) 5695 5696 partition_routers = [ 5697 self._create_component_from_model( 5698 model=child, 5699 config=config, 5700 stream_name=stream_name, 5701 **kwargs, 5702 ) 5703 for child in model.partition_routers 5704 ] 5705 5706 # partition_field depends only on config/parameters, so it is evaluated once at build 5707 # time; the runtime component always receives a plain string. 5708 partition_field = InterpolatedString.create( 5709 model.partition_field, parameters=model.parameters or {} 5710 ).eval(config) 5711 5712 # Fail fast at build time when a built-in child router is statically known to emit a 5713 # partition field different from the union's. CustomPartitionRouter children are opaque 5714 # and can only be validated at runtime. 5715 for child_model in model.partition_routers: 5716 child_partition_fields: List[str] = [] 5717 if isinstance(child_model, ListPartitionRouterModel): 5718 child_partition_fields.append( 5719 InterpolatedString.create( 5720 child_model.cursor_field, parameters=child_model.parameters or {} 5721 ).eval(config) 5722 ) 5723 elif isinstance(child_model, SubstreamPartitionRouterModel): 5724 for parent_stream_config in child_model.parent_stream_configs: 5725 child_partition_fields.append( 5726 InterpolatedString.create( 5727 parent_stream_config.partition_field, 5728 parameters=parent_stream_config.parameters 5729 or child_model.parameters 5730 or {}, 5731 ).eval(config) 5732 ) 5733 elif isinstance(child_model, UnionPartitionRouterModel): 5734 child_partition_fields.append( 5735 InterpolatedString.create( 5736 child_model.partition_field, parameters=child_model.parameters or {} 5737 ).eval(config) 5738 ) 5739 for child_partition_field in child_partition_fields: 5740 if child_partition_field != partition_field: 5741 raise ValueError( 5742 f"UnionPartitionRouter expects all child partition routers to emit the " 5743 f"partition field '{partition_field}', but a " 5744 f"{child_model.type} child emits '{child_partition_field}'." 5745 ) 5746 5747 # A union slice comes from exactly one child partition router, so request options 5748 # declared on children cannot be applied consistently to requests built from the 5749 # normalized union slices. Partition values should be consumed via interpolation 5750 # (e.g. stream_partition) instead. Note that this validation only covers built-in 5751 # router types; CustomPartitionRouter children are opaque, so any request options 5752 # they implement internally cannot be detected or rejected here. 5753 for router in partition_routers: 5754 if isinstance(router, SubstreamPartitionRouter): 5755 if any( 5756 parent_config.request_option for parent_config in router.parent_stream_configs 5757 ): 5758 raise ValueError("Request options are not supported for UnionPartitionRouter.") 5759 if isinstance(router, ListPartitionRouter) and router.request_option: 5760 raise ValueError("Request options are not supported for UnionPartitionRouter.") 5761 5762 return UnionPartitionRouter( 5763 partition_routers=partition_routers, 5764 partition_field=partition_field, 5765 parameters=model.parameters or {}, 5766 )