airbyte_cdk.sources.declarative.parsers.model_to_component_factory
1# 2# Copyright (c) 2025 Airbyte, Inc., all rights reserved. 3# 4 5from __future__ import annotations 6 7import datetime 8import importlib 9import inspect 10import json 11import logging 12import math 13import re 14from functools import partial 15from typing import ( 16 TYPE_CHECKING, 17 Any, 18 Callable, 19 Dict, 20 List, 21 Mapping, 22 MutableMapping, 23 Optional, 24 Tuple, 25 Type, 26 Union, 27 cast, 28 get_args, 29 get_origin, 30 get_type_hints, 31) 32 33if TYPE_CHECKING: 34 from airbyte_cdk.legacy.sources.declarative.incremental.datetime_based_cursor import ( 35 DatetimeBasedCursor, 36 ) 37 38from airbyte_protocol_dataclasses.models import ConfiguredAirbyteStream 39from isodate import parse_duration 40from pydantic.v1 import BaseModel 41from requests import Response 42 43from airbyte_cdk.connector_builder.models import ( 44 LogMessage as ConnectorBuilderLogMessage, 45) 46from airbyte_cdk.models import ( 47 AirbyteStateBlob, 48 AirbyteStateMessage, 49 AirbyteStateType, 50 AirbyteStreamState, 51 ConfiguredAirbyteCatalog, 52 FailureType, 53 Level, 54 StreamDescriptor, 55) 56from airbyte_cdk.sources.connector_state_manager import ConnectorStateManager 57from airbyte_cdk.sources.declarative.async_job.job_orchestrator import AsyncJobOrchestrator 58from airbyte_cdk.sources.declarative.async_job.job_tracker import JobTracker 59from airbyte_cdk.sources.declarative.async_job.repository import AsyncJobRepository 60from airbyte_cdk.sources.declarative.async_job.status import AsyncJobStatus 61from airbyte_cdk.sources.declarative.auth import DeclarativeOauth2Authenticator, JwtAuthenticator 62from airbyte_cdk.sources.declarative.auth.declarative_authenticator import ( 63 DeclarativeAuthenticator, 64 NoAuth, 65) 66from airbyte_cdk.sources.declarative.auth.jwt import JwtAlgorithm 67from airbyte_cdk.sources.declarative.auth.oauth import ( 68 DeclarativeSingleUseRefreshTokenOauth2Authenticator, 69) 70from airbyte_cdk.sources.declarative.auth.rate_limited_multiple_token import ( 71 RateLimitedMultipleTokenAuthenticator, 72 TokenQuota, 73) 74from airbyte_cdk.sources.declarative.auth.selective_authenticator import SelectiveAuthenticator 75from airbyte_cdk.sources.declarative.auth.token import ( 76 ApiKeyAuthenticator, 77 BasicHttpAuthenticator, 78 BearerAuthenticator, 79 LegacySessionTokenAuthenticator, 80) 81from airbyte_cdk.sources.declarative.auth.token_provider import ( 82 InterpolatedSessionTokenProvider, 83 InterpolatedStringTokenProvider, 84 SessionTokenProvider, 85 TokenProvider, 86) 87from airbyte_cdk.sources.declarative.checks import ( 88 CheckDynamicStream, 89 CheckStream, 90 DynamicStreamCheckConfig, 91) 92from airbyte_cdk.sources.declarative.concurrency_level import ConcurrencyLevel 93from airbyte_cdk.sources.declarative.datetime.min_max_datetime import MinMaxDatetime 94from airbyte_cdk.sources.declarative.decoders import ( 95 Decoder, 96 IterableDecoder, 97 JsonDecoder, 98 PaginationDecoderDecorator, 99 XmlDecoder, 100 ZipfileDecoder, 101) 102from airbyte_cdk.sources.declarative.decoders.composite_raw_decoder import ( 103 CompositeRawDecoder, 104 CsvParser, 105 GzipParser, 106 JsonItemsParser, 107 JsonLineParser, 108 JsonParser, 109 Parser, 110) 111from airbyte_cdk.sources.declarative.expanders.record_expander import ( 112 OnNoRecords, 113 ParentFieldPath, 114 RecordExpander, 115) 116from airbyte_cdk.sources.declarative.extractors import ( 117 CombinedExtractor, 118 CombineMode, 119 DpathExtractor, 120 RecordFilter, 121 RecordSelector, 122 ResponseToFileExtractor, 123) 124from airbyte_cdk.sources.declarative.extractors.record_extractor import RecordExtractor 125from airbyte_cdk.sources.declarative.extractors.record_filter import ( 126 ClientSideIncrementalRecordFilterDecorator, 127) 128from airbyte_cdk.sources.declarative.incremental import ( 129 ConcurrentCursorFactory, 130 ConcurrentPerPartitionCursor, 131 SyncTokenCursor, 132) 133from airbyte_cdk.sources.declarative.interpolation import InterpolatedString 134from airbyte_cdk.sources.declarative.interpolation.interpolated_mapping import InterpolatedMapping 135from airbyte_cdk.sources.declarative.migrations.legacy_to_per_partition_state_migration import ( 136 LegacyToPerPartitionStateMigration, 137) 138from airbyte_cdk.sources.declarative.models import ( 139 CustomStateMigration, 140 PaginationResetLimits, 141) 142from airbyte_cdk.sources.declarative.models.base_model_with_deprecations import ( 143 DEPRECATION_LOGS_TAG, 144 BaseModelWithDeprecations, 145) 146from airbyte_cdk.sources.declarative.models.declarative_component_schema import ( 147 Action as HttpResponseFilterActionModel, 148) 149from airbyte_cdk.sources.declarative.models.declarative_component_schema import ( 150 Action1 as PaginationResetActionModel, 151) 152from airbyte_cdk.sources.declarative.models.declarative_component_schema import ( 153 AddedFieldDefinition as AddedFieldDefinitionModel, 154) 155from airbyte_cdk.sources.declarative.models.declarative_component_schema import ( 156 AddFields as AddFieldsModel, 157) 158from airbyte_cdk.sources.declarative.models.declarative_component_schema import ( 159 ApiKeyAuthenticator as ApiKeyAuthenticatorModel, 160) 161from airbyte_cdk.sources.declarative.models.declarative_component_schema import ( 162 AsyncJobStatusMap as AsyncJobStatusMapModel, 163) 164from airbyte_cdk.sources.declarative.models.declarative_component_schema import ( 165 AsyncRetriever as AsyncRetrieverModel, 166) 167from airbyte_cdk.sources.declarative.models.declarative_component_schema import ( 168 BasicHttpAuthenticator as BasicHttpAuthenticatorModel, 169) 170from airbyte_cdk.sources.declarative.models.declarative_component_schema import ( 171 BearerAuthenticator as BearerAuthenticatorModel, 172) 173from airbyte_cdk.sources.declarative.models.declarative_component_schema import ( 174 CheckDynamicStream as CheckDynamicStreamModel, 175) 176from airbyte_cdk.sources.declarative.models.declarative_component_schema import ( 177 CheckStream as CheckStreamModel, 178) 179from airbyte_cdk.sources.declarative.models.declarative_component_schema import ( 180 CombinedExtractor as CombinedExtractorModel, 181) 182from airbyte_cdk.sources.declarative.models.declarative_component_schema import ( 183 ComplexFieldType as ComplexFieldTypeModel, 184) 185from airbyte_cdk.sources.declarative.models.declarative_component_schema import ( 186 ComponentMappingDefinition as ComponentMappingDefinitionModel, 187) 188from airbyte_cdk.sources.declarative.models.declarative_component_schema import ( 189 CompositeErrorHandler as CompositeErrorHandlerModel, 190) 191from airbyte_cdk.sources.declarative.models.declarative_component_schema import ( 192 ConcurrencyLevel as ConcurrencyLevelModel, 193) 194from airbyte_cdk.sources.declarative.models.declarative_component_schema import ( 195 ConfigAddFields as ConfigAddFieldsModel, 196) 197from airbyte_cdk.sources.declarative.models.declarative_component_schema import ( 198 ConfigComponentsResolver as ConfigComponentsResolverModel, 199) 200from airbyte_cdk.sources.declarative.models.declarative_component_schema import ( 201 ConfigMigration as ConfigMigrationModel, 202) 203from airbyte_cdk.sources.declarative.models.declarative_component_schema import ( 204 ConfigRemapField as ConfigRemapFieldModel, 205) 206from airbyte_cdk.sources.declarative.models.declarative_component_schema import ( 207 ConfigRemoveFields as ConfigRemoveFieldsModel, 208) 209from airbyte_cdk.sources.declarative.models.declarative_component_schema import ( 210 ConstantBackoffStrategy as ConstantBackoffStrategyModel, 211) 212from airbyte_cdk.sources.declarative.models.declarative_component_schema import ( 213 CsvDecoder as CsvDecoderModel, 214) 215from airbyte_cdk.sources.declarative.models.declarative_component_schema import ( 216 CursorPagination as CursorPaginationModel, 217) 218from airbyte_cdk.sources.declarative.models.declarative_component_schema import ( 219 CustomAuthenticator as CustomAuthenticatorModel, 220) 221from airbyte_cdk.sources.declarative.models.declarative_component_schema import ( 222 CustomBackoffStrategy as CustomBackoffStrategyModel, 223) 224from airbyte_cdk.sources.declarative.models.declarative_component_schema import ( 225 CustomConfigTransformation as CustomConfigTransformationModel, 226) 227from airbyte_cdk.sources.declarative.models.declarative_component_schema import ( 228 CustomDecoder as CustomDecoderModel, 229) 230from airbyte_cdk.sources.declarative.models.declarative_component_schema import ( 231 CustomErrorHandler as CustomErrorHandlerModel, 232) 233from airbyte_cdk.sources.declarative.models.declarative_component_schema import ( 234 CustomPaginationStrategy as CustomPaginationStrategyModel, 235) 236from airbyte_cdk.sources.declarative.models.declarative_component_schema import ( 237 CustomPartitionRouter as CustomPartitionRouterModel, 238) 239from airbyte_cdk.sources.declarative.models.declarative_component_schema import ( 240 CustomRecordExtractor as CustomRecordExtractorModel, 241) 242from airbyte_cdk.sources.declarative.models.declarative_component_schema import ( 243 CustomRecordFilter as CustomRecordFilterModel, 244) 245from airbyte_cdk.sources.declarative.models.declarative_component_schema import ( 246 CustomRequester as CustomRequesterModel, 247) 248from airbyte_cdk.sources.declarative.models.declarative_component_schema import ( 249 CustomRetriever as CustomRetrieverModel, 250) 251from airbyte_cdk.sources.declarative.models.declarative_component_schema import ( 252 CustomSchemaLoader as CustomSchemaLoader, 253) 254from airbyte_cdk.sources.declarative.models.declarative_component_schema import ( 255 CustomSchemaNormalization as CustomSchemaNormalizationModel, 256) 257from airbyte_cdk.sources.declarative.models.declarative_component_schema import ( 258 CustomTransformation as CustomTransformationModel, 259) 260from airbyte_cdk.sources.declarative.models.declarative_component_schema import ( 261 CustomValidationStrategy as CustomValidationStrategyModel, 262) 263from airbyte_cdk.sources.declarative.models.declarative_component_schema import ( 264 DatetimeBasedCursor as DatetimeBasedCursorModel, 265) 266from airbyte_cdk.sources.declarative.models.declarative_component_schema import ( 267 DeclarativeStream as DeclarativeStreamModel, 268) 269from airbyte_cdk.sources.declarative.models.declarative_component_schema import ( 270 DefaultErrorHandler as DefaultErrorHandlerModel, 271) 272from airbyte_cdk.sources.declarative.models.declarative_component_schema import ( 273 DefaultPaginator as DefaultPaginatorModel, 274) 275from airbyte_cdk.sources.declarative.models.declarative_component_schema import ( 276 DpathExtractor as DpathExtractorModel, 277) 278from airbyte_cdk.sources.declarative.models.declarative_component_schema import ( 279 DpathFlattenFields as DpathFlattenFieldsModel, 280) 281from airbyte_cdk.sources.declarative.models.declarative_component_schema import ( 282 DpathValidator as DpathValidatorModel, 283) 284from airbyte_cdk.sources.declarative.models.declarative_component_schema import ( 285 DynamicSchemaLoader as DynamicSchemaLoaderModel, 286) 287from airbyte_cdk.sources.declarative.models.declarative_component_schema import ( 288 DynamicStreamCheckConfig as DynamicStreamCheckConfigModel, 289) 290from airbyte_cdk.sources.declarative.models.declarative_component_schema import ( 291 ExponentialBackoffStrategy as ExponentialBackoffStrategyModel, 292) 293from airbyte_cdk.sources.declarative.models.declarative_component_schema import ( 294 FileUploader as FileUploaderModel, 295) 296from airbyte_cdk.sources.declarative.models.declarative_component_schema import ( 297 FixedWindowCallRatePolicy as FixedWindowCallRatePolicyModel, 298) 299from airbyte_cdk.sources.declarative.models.declarative_component_schema import ( 300 FlattenFields as FlattenFieldsModel, 301) 302from airbyte_cdk.sources.declarative.models.declarative_component_schema import ( 303 GroupByKeyMergeStrategy as GroupByKeyMergeStrategyModel, 304) 305from airbyte_cdk.sources.declarative.models.declarative_component_schema import ( 306 GroupingPartitionRouter as GroupingPartitionRouterModel, 307) 308from airbyte_cdk.sources.declarative.models.declarative_component_schema import ( 309 GzipDecoder as GzipDecoderModel, 310) 311from airbyte_cdk.sources.declarative.models.declarative_component_schema import ( 312 HTTPAPIBudget as HTTPAPIBudgetModel, 313) 314from airbyte_cdk.sources.declarative.models.declarative_component_schema import ( 315 HttpComponentsResolver as HttpComponentsResolverModel, 316) 317from airbyte_cdk.sources.declarative.models.declarative_component_schema import ( 318 HttpRequester as HttpRequesterModel, 319) 320from airbyte_cdk.sources.declarative.models.declarative_component_schema import ( 321 HttpRequestRegexMatcher as HttpRequestRegexMatcherModel, 322) 323from airbyte_cdk.sources.declarative.models.declarative_component_schema import ( 324 HttpResponseFilter as HttpResponseFilterModel, 325) 326from airbyte_cdk.sources.declarative.models.declarative_component_schema import ( 327 IncrementingCountCursor as IncrementingCountCursorModel, 328) 329from airbyte_cdk.sources.declarative.models.declarative_component_schema import ( 330 InlineSchemaLoader as InlineSchemaLoaderModel, 331) 332from airbyte_cdk.sources.declarative.models.declarative_component_schema import ( 333 IterableDecoder as IterableDecoderModel, 334) 335from airbyte_cdk.sources.declarative.models.declarative_component_schema import ( 336 JsonDecoder as JsonDecoderModel, 337) 338from airbyte_cdk.sources.declarative.models.declarative_component_schema import ( 339 JsonFileSchemaLoader as JsonFileSchemaLoaderModel, 340) 341from airbyte_cdk.sources.declarative.models.declarative_component_schema import ( 342 JsonItemsDecoder as JsonItemsDecoderModel, 343) 344from airbyte_cdk.sources.declarative.models.declarative_component_schema import ( 345 JsonlDecoder as JsonlDecoderModel, 346) 347from airbyte_cdk.sources.declarative.models.declarative_component_schema import ( 348 JsonSchemaPropertySelector as JsonSchemaPropertySelectorModel, 349) 350from airbyte_cdk.sources.declarative.models.declarative_component_schema import ( 351 JwtAuthenticator as JwtAuthenticatorModel, 352) 353from airbyte_cdk.sources.declarative.models.declarative_component_schema import ( 354 JwtHeaders as JwtHeadersModel, 355) 356from airbyte_cdk.sources.declarative.models.declarative_component_schema import ( 357 JwtPayload as JwtPayloadModel, 358) 359from airbyte_cdk.sources.declarative.models.declarative_component_schema import ( 360 KeysReplace as KeysReplaceModel, 361) 362from airbyte_cdk.sources.declarative.models.declarative_component_schema import ( 363 KeysToLower as KeysToLowerModel, 364) 365from airbyte_cdk.sources.declarative.models.declarative_component_schema import ( 366 KeysToSnakeCase as KeysToSnakeCaseModel, 367) 368from airbyte_cdk.sources.declarative.models.declarative_component_schema import ( 369 LegacySessionTokenAuthenticator as LegacySessionTokenAuthenticatorModel, 370) 371from airbyte_cdk.sources.declarative.models.declarative_component_schema import ( 372 LegacyToPerPartitionStateMigration as LegacyToPerPartitionStateMigrationModel, 373) 374from airbyte_cdk.sources.declarative.models.declarative_component_schema import ( 375 ListPartitionRouter as ListPartitionRouterModel, 376) 377from airbyte_cdk.sources.declarative.models.declarative_component_schema import ( 378 MinMaxDatetime as MinMaxDatetimeModel, 379) 380from airbyte_cdk.sources.declarative.models.declarative_component_schema import ( 381 MovingWindowCallRatePolicy as MovingWindowCallRatePolicyModel, 382) 383from airbyte_cdk.sources.declarative.models.declarative_component_schema import ( 384 NoAuth as NoAuthModel, 385) 386from airbyte_cdk.sources.declarative.models.declarative_component_schema import ( 387 NoPagination as NoPaginationModel, 388) 389from airbyte_cdk.sources.declarative.models.declarative_component_schema import ( 390 OAuthAuthenticator as OAuthAuthenticatorModel, 391) 392from airbyte_cdk.sources.declarative.models.declarative_component_schema import ( 393 OffsetIncrement as OffsetIncrementModel, 394) 395from airbyte_cdk.sources.declarative.models.declarative_component_schema import ( 396 PageIncrement as PageIncrementModel, 397) 398from airbyte_cdk.sources.declarative.models.declarative_component_schema import ( 399 PageSizeReduction as PageSizeReductionModel, 400) 401from airbyte_cdk.sources.declarative.models.declarative_component_schema import ( 402 PaginationReset as PaginationResetModel, 403) 404from airbyte_cdk.sources.declarative.models.declarative_component_schema import ( 405 ParametrizedComponentsResolver as ParametrizedComponentsResolverModel, 406) 407from airbyte_cdk.sources.declarative.models.declarative_component_schema import ( 408 ParentFieldPath as ParentFieldPathModel, 409) 410from airbyte_cdk.sources.declarative.models.declarative_component_schema import ( 411 ParentStreamConfig as ParentStreamConfigModel, 412) 413from airbyte_cdk.sources.declarative.models.declarative_component_schema import ( 414 PredicateValidator as PredicateValidatorModel, 415) 416from airbyte_cdk.sources.declarative.models.declarative_component_schema import ( 417 PropertiesFromEndpoint as PropertiesFromEndpointModel, 418) 419from airbyte_cdk.sources.declarative.models.declarative_component_schema import ( 420 PropertyChunking as PropertyChunkingModel, 421) 422from airbyte_cdk.sources.declarative.models.declarative_component_schema import ( 423 PropertyLimitType as PropertyLimitTypeModel, 424) 425from airbyte_cdk.sources.declarative.models.declarative_component_schema import ( 426 QueryProperties as QueryPropertiesModel, 427) 428from airbyte_cdk.sources.declarative.models.declarative_component_schema import ( 429 Rate as RateModel, 430) 431from airbyte_cdk.sources.declarative.models.declarative_component_schema import ( 432 RateLimitedMultipleTokenAuthenticator as RateLimitedMultipleTokenAuthenticatorModel, 433) 434from airbyte_cdk.sources.declarative.models.declarative_component_schema import ( 435 RecordExpander as RecordExpanderModel, 436) 437from airbyte_cdk.sources.declarative.models.declarative_component_schema import ( 438 RecordFilter as RecordFilterModel, 439) 440from airbyte_cdk.sources.declarative.models.declarative_component_schema import ( 441 RecordSelector as RecordSelectorModel, 442) 443from airbyte_cdk.sources.declarative.models.declarative_component_schema import ( 444 RefreshTokenUpdater as RefreshTokenUpdaterModel, 445) 446from airbyte_cdk.sources.declarative.models.declarative_component_schema import ( 447 RemoveFields as RemoveFieldsModel, 448) 449from airbyte_cdk.sources.declarative.models.declarative_component_schema import ( 450 RequestOption as RequestOptionModel, 451) 452from airbyte_cdk.sources.declarative.models.declarative_component_schema import ( 453 RequestPath as RequestPathModel, 454) 455from airbyte_cdk.sources.declarative.models.declarative_component_schema import ( 456 ResponseToFileExtractor as ResponseToFileExtractorModel, 457) 458from airbyte_cdk.sources.declarative.models.declarative_component_schema import ( 459 SchemaNormalization as SchemaNormalizationModel, 460) 461from airbyte_cdk.sources.declarative.models.declarative_component_schema import ( 462 SchemaTypeIdentifier as SchemaTypeIdentifierModel, 463) 464from airbyte_cdk.sources.declarative.models.declarative_component_schema import ( 465 SelectiveAuthenticator as SelectiveAuthenticatorModel, 466) 467from airbyte_cdk.sources.declarative.models.declarative_component_schema import ( 468 SessionTokenAuthenticator as SessionTokenAuthenticatorModel, 469) 470from airbyte_cdk.sources.declarative.models.declarative_component_schema import ( 471 SimpleRetriever as SimpleRetrieverModel, 472) 473from airbyte_cdk.sources.declarative.models.declarative_component_schema import Spec as SpecModel 474from airbyte_cdk.sources.declarative.models.declarative_component_schema import ( 475 StateDelegatingStream as StateDelegatingStreamModel, 476) 477from airbyte_cdk.sources.declarative.models.declarative_component_schema import ( 478 StreamConfig as StreamConfigModel, 479) 480from airbyte_cdk.sources.declarative.models.declarative_component_schema import ( 481 SubstreamPartitionRouter as SubstreamPartitionRouterModel, 482) 483from airbyte_cdk.sources.declarative.models.declarative_component_schema import ( 484 SyncTokenCursor as SyncTokenCursorModel, 485) 486from airbyte_cdk.sources.declarative.models.declarative_component_schema import ( 487 TypesMap as TypesMapModel, 488) 489from airbyte_cdk.sources.declarative.models.declarative_component_schema import ( 490 UnionPartitionRouter as UnionPartitionRouterModel, 491) 492from airbyte_cdk.sources.declarative.models.declarative_component_schema import ( 493 UnlimitedCallRatePolicy as UnlimitedCallRatePolicyModel, 494) 495from airbyte_cdk.sources.declarative.models.declarative_component_schema import ( 496 ValidateAdheresToSchema as ValidateAdheresToSchemaModel, 497) 498from airbyte_cdk.sources.declarative.models.declarative_component_schema import ValueType 499from airbyte_cdk.sources.declarative.models.declarative_component_schema import ( 500 WaitTimeFromHeader as WaitTimeFromHeaderModel, 501) 502from airbyte_cdk.sources.declarative.models.declarative_component_schema import ( 503 WaitTimeFromResponse as WaitTimeFromResponseModel, 504) 505from airbyte_cdk.sources.declarative.models.declarative_component_schema import ( 506 WaitUntilTimeFromHeader as WaitUntilTimeFromHeaderModel, 507) 508from airbyte_cdk.sources.declarative.models.declarative_component_schema import ( 509 XmlDecoder as XmlDecoderModel, 510) 511from airbyte_cdk.sources.declarative.models.declarative_component_schema import ( 512 ZipfileDecoder as ZipfileDecoderModel, 513) 514from airbyte_cdk.sources.declarative.parsers.custom_code_compiler import ( 515 INJECTED_MANIFEST, 516 AirbyteCustomCodeNotPermittedError, 517 custom_code_execution_permitted, 518) 519from airbyte_cdk.sources.declarative.parsers.stop_condition_safety import ( 520 StopConditionSafety, 521 classify_stop_condition, 522) 523from airbyte_cdk.sources.declarative.partition_routers import ( 524 CartesianProductStreamSlicer, 525 GroupingPartitionRouter, 526 ListPartitionRouter, 527 PartitionRouter, 528 SinglePartitionRouter, 529 SubstreamPartitionRouter, 530 UnionPartitionRouter, 531) 532from airbyte_cdk.sources.declarative.partition_routers.async_job_partition_router import ( 533 AsyncJobPartitionRouter, 534) 535from airbyte_cdk.sources.declarative.partition_routers.substream_partition_router import ( 536 ParentStreamConfig, 537) 538from airbyte_cdk.sources.declarative.requesters import HttpRequester, RequestOption 539from airbyte_cdk.sources.declarative.requesters.error_handlers import ( 540 CompositeErrorHandler, 541 DefaultErrorHandler, 542 HttpResponseFilter, 543) 544from airbyte_cdk.sources.declarative.requesters.error_handlers.backoff_strategies import ( 545 ConstantBackoffStrategy, 546 ExponentialBackoffStrategy, 547 WaitTimeFromHeaderBackoffStrategy, 548 WaitTimeFromResponseBackoffStrategy, 549 WaitUntilTimeFromHeaderBackoffStrategy, 550) 551from airbyte_cdk.sources.declarative.requesters.http_job_repository import AsyncHttpJobRepository 552from airbyte_cdk.sources.declarative.requesters.paginators import ( 553 DefaultPaginator, 554 NoPagination, 555 PaginatorTestReadDecorator, 556) 557from airbyte_cdk.sources.declarative.requesters.paginators.strategies import ( 558 CursorPaginationStrategy, 559 CursorStopCondition, 560 OffsetIncrement, 561 PageIncrement, 562 StopConditionPaginationStrategyDecorator, 563) 564from airbyte_cdk.sources.declarative.requesters.query_properties import ( 565 PropertiesFromEndpoint, 566 PropertyChunking, 567 QueryProperties, 568) 569from airbyte_cdk.sources.declarative.requesters.query_properties.property_chunking import ( 570 PropertyLimitType, 571) 572from airbyte_cdk.sources.declarative.requesters.query_properties.property_selector import ( 573 JsonSchemaPropertySelector, 574) 575from airbyte_cdk.sources.declarative.requesters.query_properties.strategies import ( 576 GroupByKey, 577) 578from airbyte_cdk.sources.declarative.requesters.request_option import RequestOptionType 579from airbyte_cdk.sources.declarative.requesters.request_options import ( 580 DatetimeBasedRequestOptionsProvider, 581 DefaultRequestOptionsProvider, 582 InterpolatedRequestOptionsProvider, 583 RequestOptionsProvider, 584 SyncTokenRequestOptionsProvider, 585) 586from airbyte_cdk.sources.declarative.requesters.request_options.per_partition_request_option_provider import ( 587 PerPartitionRequestOptionsProvider, 588) 589from airbyte_cdk.sources.declarative.requesters.request_path import RequestPath 590from airbyte_cdk.sources.declarative.requesters.requester import HttpMethod, Requester 591from airbyte_cdk.sources.declarative.resolvers import ( 592 ComponentMappingDefinition, 593 ConfigComponentsResolver, 594 HttpComponentsResolver, 595 ParametrizedComponentsResolver, 596 StreamConfig, 597 StreamParametersDefinition, 598) 599from airbyte_cdk.sources.declarative.retrievers import ( 600 AsyncRetriever, 601 LazySimpleRetriever, 602 SimpleRetriever, 603) 604from airbyte_cdk.sources.declarative.retrievers.file_uploader import ( 605 ConnectorBuilderFileUploader, 606 DefaultFileUploader, 607 FileUploader, 608 LocalFileSystemFileWriter, 609 NoopFileWriter, 610) 611from airbyte_cdk.sources.declarative.retrievers.page_size_reducer import ( 612 PageSizeReduction, 613 PageSizeResetPolicy, 614) 615from airbyte_cdk.sources.declarative.retrievers.pagination_tracker import PaginationTracker 616from airbyte_cdk.sources.declarative.retrievers.request_window_splitting import ( 617 RequestWindowSplitting, 618) 619from airbyte_cdk.sources.declarative.schema import ( 620 ComplexFieldType, 621 DefaultSchemaLoader, 622 DynamicSchemaLoader, 623 InlineSchemaLoader, 624 JsonFileSchemaLoader, 625 SchemaLoader, 626 SchemaTypeIdentifier, 627 TypesMap, 628) 629from airbyte_cdk.sources.declarative.schema.caching_schema_loader_decorator import ( 630 CachingSchemaLoaderDecorator, 631) 632from airbyte_cdk.sources.declarative.schema.composite_schema_loader import CompositeSchemaLoader 633from airbyte_cdk.sources.declarative.spec import ConfigMigration, Spec 634from airbyte_cdk.sources.declarative.stream_slicers import ( 635 StreamSlicer, 636 StreamSlicerTestReadDecorator, 637) 638from airbyte_cdk.sources.declarative.stream_slicers.declarative_partition_generator import ( 639 DeclarativePartitionFactory, 640 StreamSlicerPartitionGenerator, 641) 642from airbyte_cdk.sources.declarative.transformations import ( 643 AddFields, 644 RecordTransformation, 645 RemoveFields, 646) 647from airbyte_cdk.sources.declarative.transformations.add_fields import AddedFieldDefinition 648from airbyte_cdk.sources.declarative.transformations.config_transformations import ( 649 ConfigAddFields, 650 ConfigRemapField, 651 ConfigRemoveFields, 652) 653from airbyte_cdk.sources.declarative.transformations.config_transformations.config_transformation import ( 654 ConfigTransformation, 655) 656from airbyte_cdk.sources.declarative.transformations.dpath_flatten_fields import ( 657 DpathFlattenFields, 658 KeyTransformation, 659) 660from airbyte_cdk.sources.declarative.transformations.flatten_fields import ( 661 FlattenFields, 662) 663from airbyte_cdk.sources.declarative.transformations.keys_replace_transformation import ( 664 KeysReplaceTransformation, 665) 666from airbyte_cdk.sources.declarative.transformations.keys_to_lower_transformation import ( 667 KeysToLowerTransformation, 668) 669from airbyte_cdk.sources.declarative.transformations.keys_to_snake_transformation import ( 670 KeysToSnakeCaseTransformation, 671) 672from airbyte_cdk.sources.declarative.validators import ( 673 DpathValidator, 674 PredicateValidator, 675 ValidateAdheresToSchema, 676) 677from airbyte_cdk.sources.http_logger import format_http_message 678from airbyte_cdk.sources.message import ( 679 InMemoryMessageRepository, 680 LogAppenderMessageRepositoryDecorator, 681 MessageRepository, 682 NoopMessageRepository, 683) 684from airbyte_cdk.sources.message.repository import StateFilteringMessageRepository 685from airbyte_cdk.sources.streams import NO_CURSOR_STATE_KEY 686from airbyte_cdk.sources.streams.call_rate import ( 687 APIBudget, 688 FixedWindowCallRatePolicy, 689 HttpAPIBudget, 690 HttpRequestRegexMatcher, 691 MovingWindowCallRatePolicy, 692 Rate, 693 UnlimitedCallRatePolicy, 694) 695from airbyte_cdk.sources.streams.concurrent.abstract_stream import AbstractStream 696from airbyte_cdk.sources.streams.concurrent.clamping import ( 697 ClampingEndProvider, 698 ClampingStrategy, 699 DayClampingStrategy, 700 MonthClampingStrategy, 701 NoClamping, 702 WeekClampingStrategy, 703 Weekday, 704) 705from airbyte_cdk.sources.streams.concurrent.cursor import ( 706 ConcurrentCursor, 707 Cursor, 708 CursorField, 709 FinalStateCursor, 710) 711from airbyte_cdk.sources.streams.concurrent.default_stream import DefaultStream 712from airbyte_cdk.sources.streams.concurrent.helpers import get_primary_key_from_stream 713from airbyte_cdk.sources.streams.concurrent.partitions.stream_slicer import ( 714 StreamSlicer as ConcurrentStreamSlicer, 715) 716from airbyte_cdk.sources.streams.concurrent.state_converters.datetime_stream_state_converter import ( 717 CustomFormatConcurrentStreamStateConverter, 718 DateTimeStreamStateConverter, 719) 720from airbyte_cdk.sources.streams.concurrent.state_converters.incrementing_count_stream_state_converter import ( 721 IncrementingCountStreamStateConverter, 722) 723from airbyte_cdk.sources.streams.http.error_handlers.response_models import ResponseAction 724from airbyte_cdk.sources.types import Config 725from airbyte_cdk.sources.utils.transform import TransformConfig, TypeTransformer 726from airbyte_cdk.utils.traced_exception import AirbyteTracedException 727 728ComponentDefinition = Mapping[str, Any] 729 730SCHEMA_TRANSFORMER_TYPE_MAPPING = { 731 SchemaNormalizationModel.None_: TransformConfig.NoTransform, 732 SchemaNormalizationModel.Default: TransformConfig.DefaultSchemaNormalization, 733} 734_NO_STREAM_SLICING = SinglePartitionRouter(parameters={}) 735 736# Ideally this should use the value defined in ConcurrentDeclarativeSource, but 737# this would be a circular import 738MAX_SLICES = 5 739 740# Smallest duration each `DatetimeParser`/`strftime` directive can distinguish, used by 741# `_smallest_datetime_format_unit` to check `datetime_format` can represent `cursor_granularity`. A format with 742# no entry here (e.g. a bare `%Y`/`%m`) is treated as "cannot determine" rather than rejected. 743_DATETIME_FORMAT_DIRECTIVE_GRANULARITY: List[Tuple[str, datetime.timedelta]] = [ 744 ("%epoch_microseconds", datetime.timedelta(microseconds=1)), 745 ("%s_as_float", datetime.timedelta(microseconds=1)), 746 ("%f", datetime.timedelta(microseconds=1)), 747 ("%_ms", datetime.timedelta(milliseconds=1)), 748 ("%ms", datetime.timedelta(milliseconds=1)), 749 ("%S", datetime.timedelta(seconds=1)), 750 ("%s", datetime.timedelta(seconds=1)), 751 ("%M", datetime.timedelta(minutes=1)), 752 ("%H", datetime.timedelta(hours=1)), 753 ("%I", datetime.timedelta(hours=1)), 754 ("%d", datetime.timedelta(days=1)), 755] 756 757LOGGER = logging.getLogger(f"airbyte.model_to_component_factory") 758 759 760class ModelToComponentFactory: 761 EPOCH_DATETIME_FORMAT = "%s" 762 763 def __init__( 764 self, 765 limit_pages_fetched_per_slice: Optional[int] = None, 766 limit_slices_fetched: Optional[int] = None, 767 emit_connector_builder_messages: bool = False, 768 disable_retries: bool = False, 769 disable_cache: bool = False, 770 message_repository: Optional[MessageRepository] = None, 771 connector_state_manager: Optional[ConnectorStateManager] = None, 772 max_concurrent_async_job_count: Optional[int] = None, 773 configured_catalog: Optional[ConfiguredAirbyteCatalog] = None, 774 api_budget: Optional[APIBudget] = None, 775 rate_limited_authenticators: Optional[ 776 Dict[str, RateLimitedMultipleTokenAuthenticator] 777 ] = None, 778 custom_components_trusted: bool = True, 779 ): 780 self._init_mappings() 781 self._custom_components_trusted = custom_components_trusted 782 self._limit_pages_fetched_per_slice = limit_pages_fetched_per_slice 783 self._limit_slices_fetched = limit_slices_fetched 784 self._emit_connector_builder_messages = emit_connector_builder_messages 785 self._disable_retries = disable_retries 786 self._disable_cache = disable_cache 787 self._message_repository = message_repository or InMemoryMessageRepository( 788 self._evaluate_log_level(emit_connector_builder_messages) 789 ) 790 self._stream_name_to_configured_stream = self._create_stream_name_to_configured_stream( 791 configured_catalog 792 ) 793 self._connector_state_manager = connector_state_manager or ConnectorStateManager() 794 self._api_budget: Optional[Union[APIBudget]] = api_budget 795 # Shared instances so all streams see the same token quota counters (like api_budget) 796 self._rate_limited_authenticators: Dict[str, RateLimitedMultipleTokenAuthenticator] = ( 797 rate_limited_authenticators if rate_limited_authenticators is not None else {} 798 ) 799 self._job_tracker: JobTracker = JobTracker(max_concurrent_async_job_count or 1) 800 # placeholder for deprecation warnings 801 self._collected_deprecation_logs: List[ConnectorBuilderLogMessage] = [] 802 803 def _init_mappings(self) -> None: 804 self.PYDANTIC_MODEL_TO_CONSTRUCTOR: Mapping[Type[BaseModel], Callable[..., Any]] = { 805 AddedFieldDefinitionModel: self.create_added_field_definition, 806 AddFieldsModel: self.create_add_fields, 807 ApiKeyAuthenticatorModel: self.create_api_key_authenticator, 808 BasicHttpAuthenticatorModel: self.create_basic_http_authenticator, 809 BearerAuthenticatorModel: self.create_bearer_authenticator, 810 CheckStreamModel: self.create_check_stream, 811 DynamicStreamCheckConfigModel: self.create_dynamic_stream_check_config, 812 CheckDynamicStreamModel: self.create_check_dynamic_stream, 813 CompositeErrorHandlerModel: self.create_composite_error_handler, 814 ConcurrencyLevelModel: self.create_concurrency_level, 815 ConfigMigrationModel: self.create_config_migration, 816 ConfigAddFieldsModel: self.create_config_add_fields, 817 ConfigRemapFieldModel: self.create_config_remap_field, 818 ConfigRemoveFieldsModel: self.create_config_remove_fields, 819 ConstantBackoffStrategyModel: self.create_constant_backoff_strategy, 820 CsvDecoderModel: self.create_csv_decoder, 821 CursorPaginationModel: self.create_cursor_pagination, 822 CustomAuthenticatorModel: self.create_custom_component, 823 CustomBackoffStrategyModel: self.create_custom_component, 824 CustomDecoderModel: self.create_custom_component, 825 CustomErrorHandlerModel: self.create_custom_component, 826 CustomRecordExtractorModel: self.create_custom_component, 827 CustomRecordFilterModel: self.create_custom_component, 828 CustomRequesterModel: self.create_custom_component, 829 CustomRetrieverModel: self.create_custom_component, 830 CustomSchemaLoader: self.create_custom_component, 831 CustomSchemaNormalizationModel: self.create_custom_component, 832 CustomStateMigration: self.create_custom_component, 833 CustomPaginationStrategyModel: self.create_custom_component, 834 CustomPartitionRouterModel: self.create_custom_component, 835 CustomTransformationModel: self.create_custom_component, 836 CustomValidationStrategyModel: self.create_custom_component, 837 CustomConfigTransformationModel: self.create_custom_component, 838 DeclarativeStreamModel: self.create_default_stream, 839 DefaultErrorHandlerModel: self.create_default_error_handler, 840 DefaultPaginatorModel: self.create_default_paginator, 841 CombinedExtractorModel: self.create_combined_extractor, 842 DpathExtractorModel: self.create_dpath_extractor, 843 DpathValidatorModel: self.create_dpath_validator, 844 ResponseToFileExtractorModel: self.create_response_to_file_extractor, 845 ExponentialBackoffStrategyModel: self.create_exponential_backoff_strategy, 846 SessionTokenAuthenticatorModel: self.create_session_token_authenticator, 847 GroupByKeyMergeStrategyModel: self.create_group_by_key, 848 HttpRequesterModel: self.create_http_requester, 849 HttpResponseFilterModel: self.create_http_response_filter, 850 InlineSchemaLoaderModel: self.create_inline_schema_loader, 851 JsonDecoderModel: self.create_json_decoder, 852 JsonItemsDecoderModel: self.create_json_items_decoder, 853 JsonlDecoderModel: self.create_jsonl_decoder, 854 JsonSchemaPropertySelectorModel: self.create_json_schema_property_selector, 855 GzipDecoderModel: self.create_gzip_decoder, 856 KeysToLowerModel: self.create_keys_to_lower_transformation, 857 KeysToSnakeCaseModel: self.create_keys_to_snake_transformation, 858 KeysReplaceModel: self.create_keys_replace_transformation, 859 FlattenFieldsModel: self.create_flatten_fields, 860 DpathFlattenFieldsModel: self.create_dpath_flatten_fields, 861 IterableDecoderModel: self.create_iterable_decoder, 862 XmlDecoderModel: self.create_xml_decoder, 863 JsonFileSchemaLoaderModel: self.create_json_file_schema_loader, 864 DynamicSchemaLoaderModel: self.create_dynamic_schema_loader, 865 SchemaTypeIdentifierModel: self.create_schema_type_identifier, 866 TypesMapModel: self.create_types_map, 867 ComplexFieldTypeModel: self.create_complex_field_type, 868 JwtAuthenticatorModel: self.create_jwt_authenticator, 869 LegacyToPerPartitionStateMigrationModel: self.create_legacy_to_per_partition_state_migration, 870 ListPartitionRouterModel: self.create_list_partition_router, 871 MinMaxDatetimeModel: self.create_min_max_datetime, 872 NoAuthModel: self.create_no_auth, 873 NoPaginationModel: self.create_no_pagination, 874 OAuthAuthenticatorModel: self.create_oauth_authenticator, 875 OffsetIncrementModel: self.create_offset_increment, 876 PageIncrementModel: self.create_page_increment, 877 ParentStreamConfigModel: self.create_parent_stream_config_with_substream_wrapper, 878 PredicateValidatorModel: self.create_predicate_validator, 879 PropertiesFromEndpointModel: self.create_properties_from_endpoint, 880 PropertyChunkingModel: self.create_property_chunking, 881 QueryPropertiesModel: self.create_query_properties, 882 ParentFieldPathModel: self.create_parent_field_path, 883 RecordExpanderModel: self.create_record_expander, 884 RecordFilterModel: self.create_record_filter, 885 RecordSelectorModel: self.create_record_selector, 886 RemoveFieldsModel: self.create_remove_fields, 887 RequestPathModel: self.create_request_path, 888 RequestOptionModel: self.create_request_option, 889 LegacySessionTokenAuthenticatorModel: self.create_legacy_session_token_authenticator, 890 SelectiveAuthenticatorModel: self.create_selective_authenticator, 891 SimpleRetrieverModel: self.create_simple_retriever, 892 StateDelegatingStreamModel: self.create_state_delegating_stream, 893 SpecModel: self.create_spec, 894 SubstreamPartitionRouterModel: self.create_substream_partition_router, 895 ValidateAdheresToSchemaModel: self.create_validate_adheres_to_schema, 896 WaitTimeFromHeaderModel: self.create_wait_time_from_header, 897 WaitTimeFromResponseModel: self.create_wait_time_from_response, 898 WaitUntilTimeFromHeaderModel: self.create_wait_until_time_from_header, 899 AsyncRetrieverModel: self.create_async_retriever, 900 HttpComponentsResolverModel: self.create_http_components_resolver, 901 ConfigComponentsResolverModel: self.create_config_components_resolver, 902 ParametrizedComponentsResolverModel: self.create_parametrized_components_resolver, 903 StreamConfigModel: self.create_stream_config, 904 ComponentMappingDefinitionModel: self.create_components_mapping_definition, 905 ZipfileDecoderModel: self.create_zipfile_decoder, 906 HTTPAPIBudgetModel: self.create_http_api_budget, 907 FileUploaderModel: self.create_file_uploader, 908 FixedWindowCallRatePolicyModel: self.create_fixed_window_call_rate_policy, 909 MovingWindowCallRatePolicyModel: self.create_moving_window_call_rate_policy, 910 UnlimitedCallRatePolicyModel: self.create_unlimited_call_rate_policy, 911 RateModel: self.create_rate, 912 HttpRequestRegexMatcherModel: self.create_http_request_matcher, 913 RateLimitedMultipleTokenAuthenticatorModel: self.create_rate_limited_multiple_token_authenticator, 914 GroupingPartitionRouterModel: self.create_grouping_partition_router, 915 UnionPartitionRouterModel: self.create_union_partition_router, 916 } 917 918 # Needed for the case where we need to perform a second parse on the fields of a custom component 919 self.TYPE_NAME_TO_MODEL = {cls.__name__: cls for cls in self.PYDANTIC_MODEL_TO_CONSTRUCTOR} 920 921 @staticmethod 922 def _create_stream_name_to_configured_stream( 923 configured_catalog: Optional[ConfiguredAirbyteCatalog], 924 ) -> Mapping[str, ConfiguredAirbyteStream]: 925 return ( 926 {stream.stream.name: stream for stream in configured_catalog.streams} 927 if configured_catalog 928 else {} 929 ) 930 931 def create_component( 932 self, 933 model_type: Type[BaseModel], 934 component_definition: ComponentDefinition, 935 config: Config, 936 **kwargs: Any, 937 ) -> Any: 938 """ 939 Takes a given Pydantic model type and Mapping representing a component definition and creates a declarative component and 940 subcomponents which will be used at runtime. This is done by first parsing the mapping into a Pydantic model and then creating 941 creating declarative components from that model. 942 943 :param model_type: The type of declarative component that is being initialized 944 :param component_definition: The mapping that represents a declarative component 945 :param config: The connector config that is provided by the customer 946 :return: The declarative component to be used at runtime 947 """ 948 949 component_type = component_definition.get("type") 950 if component_definition.get("type") != model_type.__name__: 951 raise ValueError( 952 f"Expected manifest component of type {model_type.__name__}, but received {component_type} instead" 953 ) 954 955 declarative_component_model = model_type.parse_obj(component_definition) 956 957 if not isinstance(declarative_component_model, model_type): 958 raise ValueError( 959 f"Expected {model_type.__name__} component, but received {declarative_component_model.__class__.__name__}" 960 ) 961 962 return self._create_component_from_model( 963 model=declarative_component_model, config=config, **kwargs 964 ) 965 966 def _create_component_from_model(self, model: BaseModel, config: Config, **kwargs: Any) -> Any: 967 if model.__class__ not in self.PYDANTIC_MODEL_TO_CONSTRUCTOR: 968 raise ValueError( 969 f"{model.__class__} with attributes {model} is not a valid component type" 970 ) 971 component_constructor = self.PYDANTIC_MODEL_TO_CONSTRUCTOR.get(model.__class__) 972 if not component_constructor: 973 raise ValueError(f"Could not find constructor for {model.__class__}") 974 975 # collect deprecation warnings for supported models. 976 if isinstance(model, BaseModelWithDeprecations): 977 self._collect_model_deprecations(model) 978 979 return component_constructor(model=model, config=config, **kwargs) 980 981 def get_model_deprecations(self) -> List[ConnectorBuilderLogMessage]: 982 """ 983 Returns the deprecation warnings that were collected during the creation of components. 984 """ 985 return self._collected_deprecation_logs 986 987 def _collect_model_deprecations(self, model: BaseModelWithDeprecations) -> None: 988 """ 989 Collects deprecation logs from the given model and appends any new logs to the internal collection. 990 991 This method checks if the provided model has deprecation logs (identified by the presence of the DEPRECATION_LOGS_TAG attribute and a non-None `_deprecation_logs` property). It iterates through each deprecation log in the model and appends it to the `_collected_deprecation_logs` list if it has not already been collected, ensuring that duplicate logs are avoided. 992 993 Args: 994 model (BaseModelWithDeprecations): The model instance from which to collect deprecation logs. 995 """ 996 if hasattr(model, DEPRECATION_LOGS_TAG) and model._deprecation_logs is not None: 997 for log in model._deprecation_logs: 998 # avoid duplicates for deprecation logs observed. 999 if log not in self._collected_deprecation_logs: 1000 self._collected_deprecation_logs.append(log) 1001 1002 def create_config_migration( 1003 self, model: ConfigMigrationModel, config: Config 1004 ) -> ConfigMigration: 1005 transformations: List[ConfigTransformation] = [ 1006 self._create_component_from_model(transformation, config) 1007 for transformation in model.transformations 1008 ] 1009 1010 return ConfigMigration( 1011 description=model.description, 1012 transformations=transformations, 1013 ) 1014 1015 def create_config_add_fields( 1016 self, model: ConfigAddFieldsModel, config: Config, **kwargs: Any 1017 ) -> ConfigAddFields: 1018 fields = [self._create_component_from_model(field, config) for field in model.fields] 1019 return ConfigAddFields( 1020 fields=fields, 1021 condition=model.condition or "", 1022 ) 1023 1024 @staticmethod 1025 def create_config_remove_fields( 1026 model: ConfigRemoveFieldsModel, config: Config, **kwargs: Any 1027 ) -> ConfigRemoveFields: 1028 return ConfigRemoveFields( 1029 field_pointers=model.field_pointers, 1030 condition=model.condition or "", 1031 ) 1032 1033 @staticmethod 1034 def create_config_remap_field( 1035 model: ConfigRemapFieldModel, config: Config, **kwargs: Any 1036 ) -> ConfigRemapField: 1037 mapping = cast(Mapping[str, Any], model.map) 1038 return ConfigRemapField( 1039 map=mapping, 1040 field_path=model.field_path, 1041 config=config, 1042 ) 1043 1044 def create_dpath_validator(self, model: DpathValidatorModel, config: Config) -> DpathValidator: 1045 strategy = self._create_component_from_model(model.validation_strategy, config) 1046 1047 return DpathValidator( 1048 field_path=model.field_path, 1049 strategy=strategy, 1050 ) 1051 1052 def create_predicate_validator( 1053 self, model: PredicateValidatorModel, config: Config 1054 ) -> PredicateValidator: 1055 strategy = self._create_component_from_model(model.validation_strategy, config) 1056 1057 return PredicateValidator( 1058 value=model.value, 1059 strategy=strategy, 1060 ) 1061 1062 @staticmethod 1063 def create_validate_adheres_to_schema( 1064 model: ValidateAdheresToSchemaModel, config: Config, **kwargs: Any 1065 ) -> ValidateAdheresToSchema: 1066 base_schema = cast(Mapping[str, Any], model.base_schema) 1067 return ValidateAdheresToSchema( 1068 schema=base_schema, 1069 ) 1070 1071 @staticmethod 1072 def create_added_field_definition( 1073 model: AddedFieldDefinitionModel, config: Config, **kwargs: Any 1074 ) -> AddedFieldDefinition: 1075 interpolated_value = InterpolatedString.create( 1076 model.value, parameters=model.parameters or {} 1077 ) 1078 return AddedFieldDefinition( 1079 path=model.path, 1080 value=interpolated_value, 1081 value_type=ModelToComponentFactory._json_schema_type_name_to_type(model.value_type), 1082 parameters=model.parameters or {}, 1083 ) 1084 1085 def create_add_fields(self, model: AddFieldsModel, config: Config, **kwargs: Any) -> AddFields: 1086 added_field_definitions = [ 1087 self._create_component_from_model( 1088 model=added_field_definition_model, 1089 value_type=ModelToComponentFactory._json_schema_type_name_to_type( 1090 added_field_definition_model.value_type 1091 ), 1092 config=config, 1093 ) 1094 for added_field_definition_model in model.fields 1095 ] 1096 return AddFields( 1097 fields=added_field_definitions, 1098 condition=model.condition or "", 1099 parameters=model.parameters or {}, 1100 ) 1101 1102 def create_keys_to_lower_transformation( 1103 self, model: KeysToLowerModel, config: Config, **kwargs: Any 1104 ) -> KeysToLowerTransformation: 1105 return KeysToLowerTransformation() 1106 1107 def create_keys_to_snake_transformation( 1108 self, model: KeysToSnakeCaseModel, config: Config, **kwargs: Any 1109 ) -> KeysToSnakeCaseTransformation: 1110 return KeysToSnakeCaseTransformation() 1111 1112 def create_keys_replace_transformation( 1113 self, model: KeysReplaceModel, config: Config, **kwargs: Any 1114 ) -> KeysReplaceTransformation: 1115 return KeysReplaceTransformation( 1116 old=model.old, new=model.new, parameters=model.parameters or {} 1117 ) 1118 1119 def create_flatten_fields( 1120 self, model: FlattenFieldsModel, config: Config, **kwargs: Any 1121 ) -> FlattenFields: 1122 return FlattenFields( 1123 flatten_lists=model.flatten_lists if model.flatten_lists is not None else True 1124 ) 1125 1126 def create_dpath_flatten_fields( 1127 self, model: DpathFlattenFieldsModel, config: Config, **kwargs: Any 1128 ) -> DpathFlattenFields: 1129 model_field_path: List[Union[InterpolatedString, str]] = [x for x in model.field_path] 1130 key_transformation = ( 1131 KeyTransformation( 1132 config=config, 1133 prefix=model.key_transformation.prefix, 1134 suffix=model.key_transformation.suffix, 1135 parameters=model.parameters or {}, 1136 ) 1137 if model.key_transformation is not None 1138 else None 1139 ) 1140 return DpathFlattenFields( 1141 config=config, 1142 field_path=model_field_path, 1143 delete_origin_value=model.delete_origin_value 1144 if model.delete_origin_value is not None 1145 else False, 1146 replace_record=model.replace_record if model.replace_record is not None else False, 1147 key_transformation=key_transformation, 1148 parameters=model.parameters or {}, 1149 ) 1150 1151 @staticmethod 1152 def _json_schema_type_name_to_type(value_type: Optional[ValueType]) -> Optional[Type[Any]]: 1153 if not value_type: 1154 return None 1155 names_to_types = { 1156 ValueType.string: str, 1157 ValueType.number: float, 1158 ValueType.integer: int, 1159 ValueType.boolean: bool, 1160 } 1161 return names_to_types[value_type] 1162 1163 def create_api_key_authenticator( 1164 self, 1165 model: ApiKeyAuthenticatorModel, 1166 config: Config, 1167 token_provider: Optional[TokenProvider] = None, 1168 **kwargs: Any, 1169 ) -> ApiKeyAuthenticator: 1170 if model.inject_into is None and model.header is None: 1171 raise ValueError( 1172 "Expected either inject_into or header to be set for ApiKeyAuthenticator" 1173 ) 1174 1175 if model.inject_into is not None and model.header is not None: 1176 raise ValueError( 1177 "inject_into and header cannot be set both for ApiKeyAuthenticator - remove the deprecated header option" 1178 ) 1179 1180 if token_provider is not None and model.api_token != "": 1181 raise ValueError( 1182 "If token_provider is set, api_token is ignored and has to be set to empty string." 1183 ) 1184 1185 request_option = ( 1186 self._create_component_from_model( 1187 model.inject_into, config, parameters=model.parameters or {} 1188 ) 1189 if model.inject_into 1190 else RequestOption( 1191 inject_into=RequestOptionType.header, 1192 field_name=model.header or "", 1193 parameters=model.parameters or {}, 1194 ) 1195 ) 1196 1197 return ApiKeyAuthenticator( 1198 token_provider=( 1199 token_provider 1200 if token_provider is not None 1201 else InterpolatedStringTokenProvider( 1202 api_token=model.api_token or "", 1203 config=config, 1204 parameters=model.parameters or {}, 1205 ) 1206 ), 1207 request_option=request_option, 1208 config=config, 1209 parameters=model.parameters or {}, 1210 ) 1211 1212 def create_legacy_to_per_partition_state_migration( 1213 self, 1214 model: LegacyToPerPartitionStateMigrationModel, 1215 config: Mapping[str, Any], 1216 declarative_stream: DeclarativeStreamModel, 1217 ) -> LegacyToPerPartitionStateMigration: 1218 retriever = declarative_stream.retriever 1219 if not isinstance(retriever, (SimpleRetrieverModel, AsyncRetrieverModel)): 1220 raise ValueError( 1221 f"LegacyToPerPartitionStateMigrations can only be applied on a DeclarativeStream with a SimpleRetriever or AsyncRetriever. Got {type(retriever)}" 1222 ) 1223 partition_router = retriever.partition_router 1224 if not isinstance( 1225 partition_router, 1226 ( 1227 SubstreamPartitionRouterModel, 1228 CustomPartitionRouterModel, 1229 UnionPartitionRouterModel, 1230 ), 1231 ): 1232 raise ValueError( 1233 f"LegacyToPerPartitionStateMigrations can only be applied on a SimpleRetriever with a SubstreamPartitionRouter, UnionPartitionRouter or CustomPartitionRouter. Got {type(partition_router)}" 1234 ) 1235 if not isinstance(partition_router, UnionPartitionRouterModel) and not hasattr( 1236 partition_router, "parent_stream_configs" 1237 ): 1238 raise ValueError( 1239 "LegacyToPerPartitionStateMigrations can only be applied with a parent stream configuration." 1240 ) 1241 1242 if not hasattr(declarative_stream, "incremental_sync"): 1243 raise ValueError( 1244 "LegacyToPerPartitionStateMigrations can only be applied with an incremental_sync configuration." 1245 ) 1246 1247 return LegacyToPerPartitionStateMigration( 1248 partition_router, # type: ignore # was already checked above 1249 declarative_stream.incremental_sync, # type: ignore # was already checked. Migration can be applied only to incremental streams. 1250 config, 1251 declarative_stream.parameters, # type: ignore # different type is expected here Mapping[str, Any], got Dict[str, Any] 1252 ) 1253 1254 def create_session_token_authenticator( 1255 self, model: SessionTokenAuthenticatorModel, config: Config, *, name: str, **kwargs: Any 1256 ) -> Union[ApiKeyAuthenticator, BearerAuthenticator]: 1257 self._reject_reduce_page_size_action( 1258 model.login_requester, f"`login_requester` of the SessionTokenAuthenticator of {name}" 1259 ) 1260 self._reject_split_request_window_action( 1261 model.login_requester, f"`login_requester` of the SessionTokenAuthenticator of {name}" 1262 ) 1263 decoder = ( 1264 self._create_component_from_model(model=model.decoder, config=config) 1265 if model.decoder 1266 else JsonDecoder(parameters={}) 1267 ) 1268 login_requester = self._create_component_from_model( 1269 model=model.login_requester, 1270 config=config, 1271 name=f"{name}_login_requester", 1272 decoder=decoder, 1273 ) 1274 token_provider = SessionTokenProvider( 1275 login_requester=login_requester, 1276 session_token_path=model.session_token_path, 1277 expiration_duration=parse_duration(model.expiration_duration) 1278 if model.expiration_duration 1279 else None, 1280 parameters=model.parameters or {}, 1281 message_repository=self._message_repository, 1282 decoder=decoder, 1283 ) 1284 if model.request_authentication.type == "Bearer": 1285 return ModelToComponentFactory.create_bearer_authenticator( 1286 BearerAuthenticatorModel(type="BearerAuthenticator", api_token=""), # type: ignore # $parameters has a default value 1287 config, 1288 token_provider=token_provider, 1289 ) 1290 else: 1291 # Get the api_token template if specified, default to just the session token 1292 api_token_template = ( 1293 getattr(model.request_authentication, "api_token", None) or "{{ session_token }}" 1294 ) 1295 final_token_provider: TokenProvider = InterpolatedSessionTokenProvider( 1296 config=config, 1297 api_token=api_token_template, 1298 session_token_provider=token_provider, 1299 parameters=model.parameters or {}, 1300 ) 1301 return self.create_api_key_authenticator( 1302 ApiKeyAuthenticatorModel( 1303 type="ApiKeyAuthenticator", 1304 api_token="", 1305 inject_into=model.request_authentication.inject_into, 1306 ), # type: ignore # $parameters and headers default to None 1307 config=config, 1308 token_provider=final_token_provider, 1309 ) 1310 1311 @staticmethod 1312 def create_basic_http_authenticator( 1313 model: BasicHttpAuthenticatorModel, config: Config, **kwargs: Any 1314 ) -> BasicHttpAuthenticator: 1315 return BasicHttpAuthenticator( 1316 password=model.password or "", 1317 username=model.username, 1318 config=config, 1319 parameters=model.parameters or {}, 1320 ) 1321 1322 @staticmethod 1323 def create_bearer_authenticator( 1324 model: BearerAuthenticatorModel, 1325 config: Config, 1326 token_provider: Optional[TokenProvider] = None, 1327 **kwargs: Any, 1328 ) -> BearerAuthenticator: 1329 if token_provider is not None and model.api_token != "": 1330 raise ValueError( 1331 "If token_provider is set, api_token is ignored and has to be set to empty string." 1332 ) 1333 return BearerAuthenticator( 1334 token_provider=( 1335 token_provider 1336 if token_provider is not None 1337 else InterpolatedStringTokenProvider( 1338 api_token=model.api_token or "", 1339 config=config, 1340 parameters=model.parameters or {}, 1341 ) 1342 ), 1343 config=config, 1344 parameters=model.parameters or {}, 1345 ) 1346 1347 @staticmethod 1348 def create_dynamic_stream_check_config( 1349 model: DynamicStreamCheckConfigModel, config: Config, **kwargs: Any 1350 ) -> DynamicStreamCheckConfig: 1351 return DynamicStreamCheckConfig( 1352 dynamic_stream_name=model.dynamic_stream_name, 1353 stream_count=model.stream_count, 1354 ) 1355 1356 def create_check_stream( 1357 self, model: CheckStreamModel, config: Config, **kwargs: Any 1358 ) -> CheckStream: 1359 if model.dynamic_streams_check_configs is None and model.stream_names is None: 1360 raise ValueError( 1361 "Expected either stream_names or dynamic_streams_check_configs to be set for CheckStream" 1362 ) 1363 1364 dynamic_streams_check_configs = ( 1365 [ 1366 self._create_component_from_model(model=dynamic_stream_check_config, config=config) 1367 for dynamic_stream_check_config in model.dynamic_streams_check_configs 1368 ] 1369 if model.dynamic_streams_check_configs 1370 else [] 1371 ) 1372 1373 # `model.config_overrides` is deliberately not read here. The source applies it around the whole 1374 # check operation (`ConcurrentDeclarativeSource._config_overridden_for_check`), which is what makes 1375 # it work for every checker type rather than only this one. Do not wire it in a second time. 1376 return CheckStream( 1377 stream_names=model.stream_names or [], 1378 dynamic_streams_check_configs=dynamic_streams_check_configs, 1379 parameters={}, 1380 ) 1381 1382 @staticmethod 1383 def create_check_dynamic_stream( 1384 model: CheckDynamicStreamModel, config: Config, **kwargs: Any 1385 ) -> CheckDynamicStream: 1386 assert model.use_check_availability is not None # for mypy 1387 1388 use_check_availability = model.use_check_availability 1389 1390 # See `create_check_stream`: `model.config_overrides` is applied by the source, not here. 1391 return CheckDynamicStream( 1392 stream_count=model.stream_count, 1393 use_check_availability=use_check_availability, 1394 parameters={}, 1395 ) 1396 1397 def create_composite_error_handler( 1398 self, model: CompositeErrorHandlerModel, config: Config, **kwargs: Any 1399 ) -> CompositeErrorHandler: 1400 error_handlers = [ 1401 self._create_component_from_model(model=error_handler_model, config=config) 1402 for error_handler_model in model.error_handlers 1403 ] 1404 return CompositeErrorHandler( 1405 error_handlers=error_handlers, parameters=model.parameters or {} 1406 ) 1407 1408 @staticmethod 1409 def create_concurrency_level( 1410 model: ConcurrencyLevelModel, config: Config, **kwargs: Any 1411 ) -> ConcurrencyLevel: 1412 return ConcurrencyLevel( 1413 default_concurrency=model.default_concurrency, 1414 max_concurrency=model.max_concurrency, 1415 config=config, 1416 parameters={}, 1417 ) 1418 1419 @staticmethod 1420 def apply_stream_state_migrations( 1421 stream_state_migrations: List[Any] | None, stream_state: MutableMapping[str, Any] 1422 ) -> MutableMapping[str, Any]: 1423 if stream_state_migrations: 1424 for state_migration in stream_state_migrations: 1425 if state_migration.should_migrate(stream_state): 1426 # The state variable is expected to be mutable but the migrate method returns an immutable mapping. 1427 stream_state = dict(state_migration.migrate(stream_state)) 1428 return stream_state 1429 1430 def create_concurrent_cursor_from_datetime_based_cursor( 1431 self, 1432 model_type: Type[BaseModel], 1433 component_definition: ComponentDefinition, 1434 stream_name: str, 1435 stream_namespace: Optional[str], 1436 stream_state: MutableMapping[str, Any], 1437 config: Config, 1438 message_repository: Optional[MessageRepository] = None, 1439 runtime_lookback_window: Optional[datetime.timedelta] = None, 1440 **kwargs: Any, 1441 ) -> ConcurrentCursor: 1442 component_type = component_definition.get("type") 1443 if component_definition.get("type") != model_type.__name__: 1444 raise ValueError( 1445 f"Expected manifest component of type {model_type.__name__}, but received {component_type} instead" 1446 ) 1447 1448 # FIXME the interfaces of the concurrent cursor are kind of annoying as they take a `ComponentDefinition` instead of the actual model. This was done because the ConcurrentDeclarativeSource didn't have access to the models [here for example](https://github.com/airbytehq/airbyte-python-cdk/blob/f525803b3fec9329e4cc8478996a92bf884bfde9/airbyte_cdk/sources/declarative/concurrent_declarative_source.py#L354C54-L354C91). So now we have two cases: 1449 # * The ComponentDefinition comes from model.__dict__ in which case we have `parameters` 1450 # * The ComponentDefinition comes from the manifest as a dict in which case we have `$parameters` 1451 # We should change those interfaces to use the model once we clean up the code in CDS at which point the parameter propagation should happen as part of the ModelToComponentFactory. 1452 if "$parameters" not in component_definition and "parameters" in component_definition: 1453 component_definition["$parameters"] = component_definition.get("parameters") # type: ignore # This is a dict 1454 datetime_based_cursor_model = model_type.parse_obj(component_definition) 1455 1456 if not isinstance(datetime_based_cursor_model, DatetimeBasedCursorModel): 1457 raise ValueError( 1458 f"Expected {model_type.__name__} component, but received {datetime_based_cursor_model.__class__.__name__}" 1459 ) 1460 1461 model_parameters = datetime_based_cursor_model.parameters or {} 1462 1463 cursor_field = self._get_catalog_defined_cursor_field( 1464 stream_name=stream_name, 1465 allow_catalog_defined_cursor_field=datetime_based_cursor_model.allow_catalog_defined_cursor_field 1466 or False, 1467 ) 1468 1469 if not cursor_field: 1470 interpolated_cursor_field = InterpolatedString.create( 1471 datetime_based_cursor_model.cursor_field, 1472 parameters=model_parameters, 1473 ) 1474 cursor_field = CursorField( 1475 cursor_field_key=interpolated_cursor_field.eval(config=config), 1476 supports_catalog_defined_cursor_field=datetime_based_cursor_model.allow_catalog_defined_cursor_field 1477 or False, 1478 ) 1479 1480 interpolated_partition_field_start = InterpolatedString.create( 1481 datetime_based_cursor_model.partition_field_start or "start_time", 1482 parameters=model_parameters, 1483 ) 1484 interpolated_partition_field_end = InterpolatedString.create( 1485 datetime_based_cursor_model.partition_field_end or "end_time", 1486 parameters=model_parameters, 1487 ) 1488 1489 slice_boundary_fields = ( 1490 interpolated_partition_field_start.eval(config=config), 1491 interpolated_partition_field_end.eval(config=config), 1492 ) 1493 1494 datetime_format = datetime_based_cursor_model.datetime_format 1495 1496 cursor_granularity = ( 1497 parse_duration(datetime_based_cursor_model.cursor_granularity) 1498 if datetime_based_cursor_model.cursor_granularity 1499 else None 1500 ) 1501 1502 lookback_window = None 1503 interpolated_lookback_window = ( 1504 InterpolatedString.create( 1505 datetime_based_cursor_model.lookback_window, 1506 parameters=model_parameters, 1507 ) 1508 if datetime_based_cursor_model.lookback_window 1509 else None 1510 ) 1511 if interpolated_lookback_window: 1512 evaluated_lookback_window = interpolated_lookback_window.eval(config=config) 1513 if evaluated_lookback_window: 1514 lookback_window = parse_duration(evaluated_lookback_window) 1515 1516 connector_state_converter: DateTimeStreamStateConverter 1517 connector_state_converter = CustomFormatConcurrentStreamStateConverter( 1518 datetime_format=datetime_format, 1519 input_datetime_formats=datetime_based_cursor_model.cursor_datetime_formats, 1520 is_sequential_state=True, # ConcurrentPerPartitionCursor only works with sequential state 1521 cursor_granularity=cursor_granularity, 1522 ) 1523 1524 # Adjusts the stream state by applying the runtime lookback window. 1525 # This is used to ensure correct state handling in case of failed partitions. 1526 stream_state_value = stream_state.get(cursor_field.cursor_field_key) 1527 if runtime_lookback_window and stream_state_value: 1528 new_stream_state = ( 1529 connector_state_converter.parse_timestamp(stream_state_value) 1530 - runtime_lookback_window 1531 ) 1532 stream_state[cursor_field.cursor_field_key] = connector_state_converter.output_format( 1533 new_stream_state 1534 ) 1535 1536 start_date_runtime_value: Union[InterpolatedString, str, MinMaxDatetime] 1537 if isinstance(datetime_based_cursor_model.start_datetime, MinMaxDatetimeModel): 1538 start_date_runtime_value = self.create_min_max_datetime( 1539 model=datetime_based_cursor_model.start_datetime, config=config 1540 ) 1541 else: 1542 start_date_runtime_value = datetime_based_cursor_model.start_datetime 1543 1544 end_date_runtime_value: Optional[Union[InterpolatedString, str, MinMaxDatetime]] 1545 if isinstance(datetime_based_cursor_model.end_datetime, MinMaxDatetimeModel): 1546 end_date_runtime_value = self.create_min_max_datetime( 1547 model=datetime_based_cursor_model.end_datetime, config=config 1548 ) 1549 else: 1550 end_date_runtime_value = datetime_based_cursor_model.end_datetime 1551 1552 interpolated_start_date = MinMaxDatetime.create( 1553 interpolated_string_or_min_max_datetime=start_date_runtime_value, 1554 parameters=datetime_based_cursor_model.parameters, 1555 ) 1556 interpolated_end_date = ( 1557 None 1558 if not end_date_runtime_value 1559 else MinMaxDatetime.create( 1560 end_date_runtime_value, datetime_based_cursor_model.parameters 1561 ) 1562 ) 1563 1564 # If datetime format is not specified then start/end datetime should inherit it from the stream slicer 1565 if not interpolated_start_date.datetime_format: 1566 interpolated_start_date.datetime_format = datetime_format 1567 if interpolated_end_date and not interpolated_end_date.datetime_format: 1568 interpolated_end_date.datetime_format = datetime_format 1569 1570 start_date = interpolated_start_date.get_datetime(config=config) 1571 end_date_provider = ( 1572 partial(interpolated_end_date.get_datetime, config) 1573 if interpolated_end_date 1574 else connector_state_converter.get_end_provider() 1575 ) 1576 1577 if ( 1578 datetime_based_cursor_model.step and not datetime_based_cursor_model.cursor_granularity 1579 ) or ( 1580 not datetime_based_cursor_model.step and datetime_based_cursor_model.cursor_granularity 1581 ): 1582 raise ValueError( 1583 f"If step is defined, cursor_granularity should be as well and vice-versa. " 1584 f"Right now, step is `{datetime_based_cursor_model.step}` and cursor_granularity is `{datetime_based_cursor_model.cursor_granularity}`" 1585 ) 1586 1587 # When step is not defined, default to a step size from the starting date to the present moment 1588 step_length = datetime.timedelta.max 1589 interpolated_step = ( 1590 InterpolatedString.create( 1591 datetime_based_cursor_model.step, 1592 parameters=model_parameters, 1593 ) 1594 if datetime_based_cursor_model.step 1595 else None 1596 ) 1597 if interpolated_step: 1598 evaluated_step = interpolated_step.eval(config) 1599 if evaluated_step: 1600 step_length = parse_duration(evaluated_step) 1601 1602 clamping_strategy: ClampingStrategy = NoClamping() 1603 if datetime_based_cursor_model.clamping: 1604 # While it is undesirable to interpolate within the model factory (as opposed to at runtime), 1605 # it is still better than shifting interpolation low-code concept into the ConcurrentCursor runtime 1606 # object which we want to keep agnostic of being low-code 1607 target = InterpolatedString( 1608 string=datetime_based_cursor_model.clamping.target, 1609 parameters=model_parameters, 1610 ) 1611 evaluated_target = target.eval(config=config) 1612 match evaluated_target: 1613 case "DAY": 1614 clamping_strategy = DayClampingStrategy() 1615 end_date_provider = ClampingEndProvider( 1616 DayClampingStrategy(is_ceiling=False), 1617 end_date_provider, # type: ignore # Having issues w/ inspection for GapType and CursorValueType as shown in existing tests. Confirmed functionality is working in practice 1618 granularity=cursor_granularity or datetime.timedelta(seconds=1), 1619 ) 1620 case "WEEK": 1621 if ( 1622 not datetime_based_cursor_model.clamping.target_details 1623 or "weekday" not in datetime_based_cursor_model.clamping.target_details 1624 ): 1625 raise ValueError( 1626 "Given WEEK clamping, weekday needs to be provided as target_details" 1627 ) 1628 weekday = self._assemble_weekday( 1629 datetime_based_cursor_model.clamping.target_details["weekday"] 1630 ) 1631 clamping_strategy = WeekClampingStrategy(weekday) 1632 end_date_provider = ClampingEndProvider( 1633 WeekClampingStrategy(weekday, is_ceiling=False), 1634 end_date_provider, # type: ignore # Having issues w/ inspection for GapType and CursorValueType as shown in existing tests. Confirmed functionality is working in practice 1635 granularity=cursor_granularity or datetime.timedelta(days=1), 1636 ) 1637 case "MONTH": 1638 clamping_strategy = MonthClampingStrategy() 1639 end_date_provider = ClampingEndProvider( 1640 MonthClampingStrategy(is_ceiling=False), 1641 end_date_provider, # type: ignore # Having issues w/ inspection for GapType and CursorValueType as shown in existing tests. Confirmed functionality is working in practice 1642 granularity=cursor_granularity or datetime.timedelta(days=1), 1643 ) 1644 case _: 1645 raise ValueError( 1646 f"Invalid clamping target {evaluated_target}, expected DAY, WEEK, MONTH" 1647 ) 1648 1649 return ConcurrentCursor( 1650 stream_name=stream_name, 1651 stream_namespace=stream_namespace, 1652 stream_state=stream_state, 1653 message_repository=message_repository or self._message_repository, 1654 connector_state_manager=self._connector_state_manager, 1655 connector_state_converter=connector_state_converter, 1656 cursor_field=cursor_field, 1657 slice_boundary_fields=slice_boundary_fields, 1658 start=start_date, # type: ignore # Having issues w/ inspection for GapType and CursorValueType as shown in existing tests. Confirmed functionality is working in practice 1659 end_provider=end_date_provider, # type: ignore # Having issues w/ inspection for GapType and CursorValueType as shown in existing tests. Confirmed functionality is working in practice 1660 lookback_window=lookback_window, 1661 slice_range=step_length, 1662 cursor_granularity=cursor_granularity, 1663 clamping_strategy=clamping_strategy, 1664 is_compare_strictly=datetime_based_cursor_model.is_compare_strictly or False, 1665 ) 1666 1667 def create_concurrent_cursor_from_incrementing_count_cursor( 1668 self, 1669 model_type: Type[BaseModel], 1670 component_definition: ComponentDefinition, 1671 stream_name: str, 1672 stream_namespace: Optional[str], 1673 stream_state: MutableMapping[str, Any], 1674 config: Config, 1675 message_repository: Optional[MessageRepository] = None, 1676 **kwargs: Any, 1677 ) -> ConcurrentCursor: 1678 component_type = component_definition.get("type") 1679 if component_definition.get("type") != model_type.__name__: 1680 raise ValueError( 1681 f"Expected manifest component of type {model_type.__name__}, but received {component_type} instead" 1682 ) 1683 1684 incrementing_count_cursor_model = model_type.parse_obj(component_definition) 1685 1686 if not isinstance(incrementing_count_cursor_model, IncrementingCountCursorModel): 1687 raise ValueError( 1688 f"Expected {model_type.__name__} component, but received {incrementing_count_cursor_model.__class__.__name__}" 1689 ) 1690 1691 start_value: Union[int, str, None] = incrementing_count_cursor_model.start_value 1692 # Pydantic Union type coercion can convert int 0 to string '0' depending on Union order. 1693 # We need to handle both int and str representations of numeric values. 1694 # Evaluate the InterpolatedString and convert to int for the ConcurrentCursor. 1695 if start_value is not None: 1696 interpolated_start_value = InterpolatedString.create( 1697 str(start_value), # Ensure we pass a string to InterpolatedString.create 1698 parameters=incrementing_count_cursor_model.parameters or {}, 1699 ) 1700 evaluated_start_value: int = int(interpolated_start_value.eval(config=config)) 1701 else: 1702 evaluated_start_value = 0 1703 1704 cursor_field = self._get_catalog_defined_cursor_field( 1705 stream_name=stream_name, 1706 allow_catalog_defined_cursor_field=incrementing_count_cursor_model.allow_catalog_defined_cursor_field 1707 or False, 1708 ) 1709 1710 if not cursor_field: 1711 interpolated_cursor_field = InterpolatedString.create( 1712 incrementing_count_cursor_model.cursor_field, 1713 parameters=incrementing_count_cursor_model.parameters or {}, 1714 ) 1715 cursor_field = CursorField( 1716 cursor_field_key=interpolated_cursor_field.eval(config=config), 1717 supports_catalog_defined_cursor_field=incrementing_count_cursor_model.allow_catalog_defined_cursor_field 1718 or False, 1719 ) 1720 1721 connector_state_converter = IncrementingCountStreamStateConverter( 1722 is_sequential_state=True, # ConcurrentPerPartitionCursor only works with sequential state 1723 ) 1724 1725 return ConcurrentCursor( 1726 stream_name=stream_name, 1727 stream_namespace=stream_namespace, 1728 stream_state=stream_state, 1729 message_repository=message_repository or self._message_repository, 1730 connector_state_manager=self._connector_state_manager, 1731 connector_state_converter=connector_state_converter, 1732 cursor_field=cursor_field, 1733 slice_boundary_fields=None, 1734 start=evaluated_start_value, # type: ignore # Having issues w/ inspection for GapType and CursorValueType as shown in existing tests. Confirmed functionality is working in practice 1735 end_provider=connector_state_converter.get_end_provider(), # type: ignore # Having issues w/ inspection for GapType and CursorValueType as shown in existing tests. Confirmed functionality is working in practice 1736 ) 1737 1738 def _assemble_weekday(self, weekday: str) -> Weekday: 1739 match weekday: 1740 case "MONDAY": 1741 return Weekday.MONDAY 1742 case "TUESDAY": 1743 return Weekday.TUESDAY 1744 case "WEDNESDAY": 1745 return Weekday.WEDNESDAY 1746 case "THURSDAY": 1747 return Weekday.THURSDAY 1748 case "FRIDAY": 1749 return Weekday.FRIDAY 1750 case "SATURDAY": 1751 return Weekday.SATURDAY 1752 case "SUNDAY": 1753 return Weekday.SUNDAY 1754 case _: 1755 raise ValueError(f"Unknown weekday {weekday}") 1756 1757 def create_concurrent_cursor_from_perpartition_cursor( 1758 self, 1759 state_manager: ConnectorStateManager, 1760 model_type: Type[BaseModel], 1761 component_definition: ComponentDefinition, 1762 stream_name: str, 1763 stream_namespace: Optional[str], 1764 config: Config, 1765 stream_state: MutableMapping[str, Any], 1766 partition_router: PartitionRouter, 1767 attempt_to_create_cursor_if_not_provided: bool = False, 1768 **kwargs: Any, 1769 ) -> ConcurrentPerPartitionCursor: 1770 component_type = component_definition.get("type") 1771 if component_definition.get("type") != model_type.__name__: 1772 raise ValueError( 1773 f"Expected manifest component of type {model_type.__name__}, but received {component_type} instead" 1774 ) 1775 1776 # FIXME the interfaces of the concurrent cursor are kind of annoying as they take a `ComponentDefinition` instead of the actual model. This was done because the ConcurrentDeclarativeSource didn't have access to the models [here for example](https://github.com/airbytehq/airbyte-python-cdk/blob/f525803b3fec9329e4cc8478996a92bf884bfde9/airbyte_cdk/sources/declarative/concurrent_declarative_source.py#L354C54-L354C91). So now we have two cases: 1777 # * The ComponentDefinition comes from model.__dict__ in which case we have `parameters` 1778 # * The ComponentDefinition comes from the manifest as a dict in which case we have `$parameters` 1779 # We should change those interfaces to use the model once we clean up the code in CDS at which point the parameter propagation should happen as part of the ModelToComponentFactory. 1780 if "$parameters" not in component_definition and "parameters" in component_definition: 1781 component_definition["$parameters"] = component_definition.get("parameters") # type: ignore # This is a dict 1782 datetime_based_cursor_model = model_type.parse_obj(component_definition) 1783 1784 if not isinstance(datetime_based_cursor_model, DatetimeBasedCursorModel): 1785 raise ValueError( 1786 f"Expected {model_type.__name__} component, but received {datetime_based_cursor_model.__class__.__name__}" 1787 ) 1788 1789 cursor_field = self._get_catalog_defined_cursor_field( 1790 stream_name=stream_name, 1791 allow_catalog_defined_cursor_field=datetime_based_cursor_model.allow_catalog_defined_cursor_field 1792 or False, 1793 ) 1794 1795 if not cursor_field: 1796 interpolated_cursor_field = InterpolatedString.create( 1797 datetime_based_cursor_model.cursor_field, 1798 # FIXME the interfaces of the concurrent cursor are kind of annoying as they take a `ComponentDefinition` instead of the actual model. This was done because the ConcurrentDeclarativeSource didn't have access to the models [here for example](https://github.com/airbytehq/airbyte-python-cdk/blob/f525803b3fec9329e4cc8478996a92bf884bfde9/airbyte_cdk/sources/declarative/concurrent_declarative_source.py#L354C54-L354C91). So now we have two cases: 1799 # * The ComponentDefinition comes from model.__dict__ in which case we have `parameters` 1800 # * The ComponentDefinition comes from the manifest as a dict in which case we have `$parameters` 1801 # We should change those interfaces to use the model once we clean up the code in CDS at which point the parameter propagation should happen as part of the ModelToComponentFactory. 1802 parameters=datetime_based_cursor_model.parameters or {}, 1803 ) 1804 cursor_field = CursorField( 1805 cursor_field_key=interpolated_cursor_field.eval(config=config), 1806 supports_catalog_defined_cursor_field=datetime_based_cursor_model.allow_catalog_defined_cursor_field 1807 or False, 1808 ) 1809 1810 datetime_format = datetime_based_cursor_model.datetime_format 1811 1812 cursor_granularity = ( 1813 parse_duration(datetime_based_cursor_model.cursor_granularity) 1814 if datetime_based_cursor_model.cursor_granularity 1815 else None 1816 ) 1817 1818 connector_state_converter: DateTimeStreamStateConverter 1819 connector_state_converter = CustomFormatConcurrentStreamStateConverter( 1820 datetime_format=datetime_format, 1821 input_datetime_formats=datetime_based_cursor_model.cursor_datetime_formats, 1822 is_sequential_state=True, # ConcurrentPerPartitionCursor only works with sequential state 1823 cursor_granularity=cursor_granularity, 1824 ) 1825 1826 # Create the cursor factory 1827 cursor_factory = ConcurrentCursorFactory( 1828 partial( 1829 self.create_concurrent_cursor_from_datetime_based_cursor, 1830 state_manager=state_manager, 1831 model_type=model_type, 1832 component_definition=component_definition, 1833 stream_name=stream_name, 1834 stream_namespace=stream_namespace, 1835 config=config, 1836 message_repository=NoopMessageRepository(), 1837 ) 1838 ) 1839 1840 # Per-partition state doesn't make sense for GroupingPartitionRouter, so force the global state 1841 use_global_cursor = isinstance( 1842 partition_router, GroupingPartitionRouter 1843 ) or component_definition.get("global_substream_cursor", False) 1844 1845 # Return the concurrent cursor and state converter 1846 return ConcurrentPerPartitionCursor( 1847 cursor_factory=cursor_factory, 1848 partition_router=partition_router, 1849 stream_name=stream_name, 1850 stream_namespace=stream_namespace, 1851 stream_state=stream_state, 1852 message_repository=self._message_repository, # type: ignore 1853 connector_state_manager=state_manager, 1854 connector_state_converter=connector_state_converter, 1855 cursor_field=cursor_field, 1856 use_global_cursor=use_global_cursor, 1857 attempt_to_create_cursor_if_not_provided=attempt_to_create_cursor_if_not_provided, 1858 ) 1859 1860 @staticmethod 1861 def create_constant_backoff_strategy( 1862 model: ConstantBackoffStrategyModel, config: Config, **kwargs: Any 1863 ) -> ConstantBackoffStrategy: 1864 ModelToComponentFactory._validate_jitter_range(model.jitter_range_in_seconds) 1865 return ConstantBackoffStrategy( 1866 backoff_time_in_seconds=model.backoff_time_in_seconds, 1867 jitter_range_in_seconds=model.jitter_range_in_seconds, 1868 config=config, 1869 parameters=model.parameters or {}, 1870 ) 1871 1872 @staticmethod 1873 def _validate_jitter_range(jitter_range_in_seconds: Optional[float]) -> None: 1874 if jitter_range_in_seconds is not None and jitter_range_in_seconds < 0: 1875 raise ValueError("jitter_range_in_seconds must be greater than or equal to 0") 1876 1877 def create_cursor_pagination( 1878 self, 1879 model: CursorPaginationModel, 1880 config: Config, 1881 decoder: Optional[Decoder] = None, 1882 **kwargs: Any, 1883 ) -> CursorPaginationStrategy: 1884 # None when nested under a custom component, which cannot pass a decoder 1885 if decoder is None: 1886 decoder = JsonDecoder(parameters={}) 1887 if isinstance(decoder, PaginationDecoderDecorator): 1888 inner_decoder = decoder.decoder 1889 else: 1890 inner_decoder = decoder 1891 decoder = PaginationDecoderDecorator(decoder=decoder) 1892 1893 if self._is_supported_decoder_for_pagination(inner_decoder): 1894 decoder_to_use = decoder 1895 else: 1896 raise ValueError( 1897 self._UNSUPPORTED_DECODER_ERROR.format(decoder_type=type(inner_decoder)) 1898 ) 1899 1900 # Pydantic v1 Union type coercion can convert int to string depending on Union order. 1901 # If page_size is a string that represents an integer (not an interpolation), convert it back. 1902 page_size = model.page_size 1903 if isinstance(page_size, str) and page_size.isdigit(): 1904 page_size = int(page_size) 1905 1906 return CursorPaginationStrategy( 1907 cursor_value=model.cursor_value, 1908 decoder=decoder_to_use, 1909 page_size=page_size, 1910 stop_condition=model.stop_condition, 1911 config=config, 1912 parameters=model.parameters or {}, 1913 ) 1914 1915 def create_custom_component(self, model: Any, config: Config, **kwargs: Any) -> Any: 1916 """ 1917 Generically creates a custom component based on the model type and a class_name reference to the custom Python class being 1918 instantiated. Only the model's additional properties that match the custom class definition are passed to the constructor 1919 :param model: The Pydantic model of the custom component being created 1920 :param config: The custom defined connector config 1921 :return: The declarative component built from the Pydantic model to be used at runtime 1922 """ 1923 # Instantiating a custom component means importing and executing arbitrary code referenced 1924 # by `class_name`. Manifests supplied by a caller, whether through the config or directly to 1925 # the manifest server, are untrusted input and could point `class_name` at any importable 1926 # callable, so they honor the same `AIRBYTE_ENABLE_UNSAFE_CODE` gate as injected 1927 # `components.py` code. Manifests bundled in a published connector image are trusted and may 1928 # always use their bundled custom components. 1929 manifest_is_untrusted = not self._custom_components_trusted or bool( 1930 config.get(INJECTED_MANIFEST) 1931 ) 1932 if manifest_is_untrusted and not custom_code_execution_permitted(): 1933 raise AirbyteCustomCodeNotPermittedError 1934 1935 custom_component_class = self._get_class_from_fully_qualified_class_name(model.class_name) 1936 component_fields = get_type_hints(custom_component_class) 1937 model_args = model.dict() 1938 model_args["config"] = config 1939 1940 # There are cases where a parent component will pass arguments to a child component via kwargs. When there are field collisions 1941 # we defer to these arguments over the component's definition 1942 for key, arg in kwargs.items(): 1943 model_args[key] = arg 1944 1945 # Pydantic is unable to parse a custom component's fields that are subcomponents into models because their fields and types are not 1946 # defined in the schema. The fields and types are defined within the Python class implementation. Pydantic can only parse down to 1947 # the custom component and this code performs a second parse to convert the sub-fields first into models, then declarative components 1948 for model_field, model_value in model_args.items(): 1949 # If a custom component field doesn't have a type set, we try to use the type hints to infer the type 1950 if ( 1951 isinstance(model_value, dict) 1952 and "type" not in model_value 1953 and model_field in component_fields 1954 ): 1955 derived_type = self._derive_component_type_from_type_hints( 1956 component_fields.get(model_field) 1957 ) 1958 if derived_type: 1959 model_value["type"] = derived_type 1960 1961 if self._is_component(model_value): 1962 model_args[model_field] = self._create_nested_component( 1963 model, 1964 model_field, 1965 model_value, 1966 config, 1967 **kwargs, 1968 ) 1969 elif isinstance(model_value, list): 1970 vals = [] 1971 for v in model_value: 1972 if isinstance(v, dict) and "type" not in v and model_field in component_fields: 1973 derived_type = self._derive_component_type_from_type_hints( 1974 component_fields.get(model_field) 1975 ) 1976 if derived_type: 1977 v["type"] = derived_type 1978 if self._is_component(v): 1979 vals.append( 1980 self._create_nested_component( 1981 model, 1982 model_field, 1983 v, 1984 config, 1985 **kwargs, 1986 ) 1987 ) 1988 else: 1989 vals.append(v) 1990 model_args[model_field] = vals 1991 1992 kwargs = { 1993 class_field: model_args[class_field] 1994 for class_field in component_fields.keys() 1995 if class_field in model_args 1996 } 1997 1998 if "api_budget" in component_fields and kwargs.get("api_budget") is None: 1999 kwargs["api_budget"] = self._api_budget 2000 2001 return custom_component_class(**kwargs) 2002 2003 @staticmethod 2004 def _get_class_from_fully_qualified_class_name( 2005 full_qualified_class_name: str, 2006 ) -> Any: 2007 """Get a class from its fully qualified name. 2008 2009 If a custom components module is needed, we assume it is already registered - probably 2010 as `source_declarative_manifest.components` or `components`. 2011 2012 Args: 2013 full_qualified_class_name (str): The fully qualified name of the class (e.g., "module.ClassName"). 2014 2015 Returns: 2016 Any: The class object. 2017 2018 Raises: 2019 ValueError: If the class cannot be loaded. 2020 """ 2021 split = full_qualified_class_name.split(".") 2022 module_name_full = ".".join(split[:-1]) 2023 class_name = split[-1] 2024 2025 try: 2026 module_ref = importlib.import_module(module_name_full) 2027 except ModuleNotFoundError as e: 2028 if split[0] == "source_declarative_manifest": 2029 # During testing, the modules containing the custom components are not moved to source_declarative_manifest. In order to run the test, add the source folder to your PYTHONPATH or add it runtime using sys.path.append 2030 try: 2031 import os 2032 2033 module_name_with_source_declarative_manifest = ".".join(split[1:-1]) 2034 module_ref = importlib.import_module( 2035 module_name_with_source_declarative_manifest 2036 ) 2037 except ModuleNotFoundError: 2038 raise ValueError(f"Could not load module `{module_name_full}`.") from e 2039 else: 2040 raise ValueError(f"Could not load module `{module_name_full}`.") from e 2041 2042 try: 2043 return getattr(module_ref, class_name) 2044 except AttributeError as e: 2045 raise ValueError( 2046 f"Could not load class `{class_name}` from module `{module_name_full}`.", 2047 ) from e 2048 2049 @staticmethod 2050 def _derive_component_type_from_type_hints(field_type: Any) -> Optional[str]: 2051 interface = field_type 2052 while True: 2053 origin = get_origin(interface) 2054 if origin: 2055 # Unnest types until we reach the raw type 2056 # List[T] -> T 2057 # Optional[List[T]] -> T 2058 args = get_args(interface) 2059 interface = args[0] 2060 else: 2061 break 2062 if isinstance(interface, type) and not ModelToComponentFactory.is_builtin_type(interface): 2063 return interface.__name__ 2064 return None 2065 2066 @staticmethod 2067 def is_builtin_type(cls: Optional[Type[Any]]) -> bool: 2068 if not cls: 2069 return False 2070 return cls.__module__ == "builtins" 2071 2072 @staticmethod 2073 def _extract_missing_parameters(error: TypeError) -> List[str]: 2074 parameter_search = re.search(r"keyword-only.*:\s(.*)", str(error)) 2075 if parameter_search: 2076 return re.findall(r"\'(.+?)\'", parameter_search.group(1)) 2077 else: 2078 return [] 2079 2080 def _create_nested_component( 2081 self, model: Any, model_field: str, model_value: Any, config: Config, **kwargs: Any 2082 ) -> Any: 2083 type_name = model_value.get("type", None) 2084 if not type_name: 2085 # If no type is specified, we can assume this is a dictionary object which can be returned instead of a subcomponent 2086 return model_value 2087 2088 model_type = self.TYPE_NAME_TO_MODEL.get(type_name, None) 2089 if model_type: 2090 parsed_model = model_type.parse_obj(model_value) 2091 try: 2092 # To improve usability of the language, certain fields are shared between components. This can come in the form of 2093 # a parent component passing some of its fields to a child component or the parent extracting fields from other child 2094 # components and passing it to others. One example is the DefaultPaginator referencing the HttpRequester url_base 2095 # while constructing a SimpleRetriever. However, custom components don't support this behavior because they are created 2096 # generically in create_custom_component(). This block allows developers to specify extra arguments in $parameters that 2097 # are needed by a component and could not be shared. 2098 model_constructor = self.PYDANTIC_MODEL_TO_CONSTRUCTOR.get(parsed_model.__class__) 2099 constructor_kwargs = inspect.getfullargspec(model_constructor).kwonlyargs 2100 model_parameters = model_value.get("$parameters", {}) 2101 matching_parameters = { 2102 kwarg: model_parameters[kwarg] 2103 for kwarg in constructor_kwargs 2104 if kwarg in model_parameters 2105 } 2106 matching_kwargs = { 2107 kwarg: kwargs[kwarg] for kwarg in constructor_kwargs if kwarg in kwargs 2108 } 2109 return self._create_component_from_model( 2110 model=parsed_model, config=config, **(matching_parameters | matching_kwargs) 2111 ) 2112 except TypeError as error: 2113 missing_parameters = self._extract_missing_parameters(error) 2114 if missing_parameters: 2115 raise ValueError( 2116 f"Error creating component '{type_name}' with parent custom component {model.class_name}: Please provide " 2117 + ", ".join( 2118 ( 2119 f"{type_name}.$parameters.{parameter}" 2120 for parameter in missing_parameters 2121 ) 2122 ) 2123 ) 2124 raise TypeError( 2125 f"Error creating component '{type_name}' with parent custom component {model.class_name}: {error}" 2126 ) 2127 else: 2128 raise ValueError( 2129 f"Error creating custom component {model.class_name}. Subcomponent creation has not been implemented for '{type_name}'" 2130 ) 2131 2132 @staticmethod 2133 def _is_component(model_value: Any) -> bool: 2134 return isinstance(model_value, dict) and model_value.get("type") is not None 2135 2136 def create_default_stream( 2137 self, model: DeclarativeStreamModel, config: Config, is_parent: bool = False, **kwargs: Any 2138 ) -> AbstractStream: 2139 primary_key = model.primary_key.__root__ if model.primary_key else None 2140 self._validate_sync_token_stream(model) 2141 self._migrate_state(model, config) 2142 self._warn_on_ineffective_incremental_dependency(model) 2143 2144 partition_router = self._build_stream_slicer_from_partition_router( 2145 model.retriever, 2146 config, 2147 stream_name=model.name, 2148 **kwargs, 2149 ) 2150 concurrent_cursor = self._build_concurrent_cursor(model, partition_router, config) 2151 request_options_provider: Optional[RequestOptionsProvider] 2152 if model.incremental_sync and isinstance(model.incremental_sync, DatetimeBasedCursorModel): 2153 cursor_model: DatetimeBasedCursorModel = model.incremental_sync 2154 2155 end_time_option = ( 2156 self._create_component_from_model( 2157 cursor_model.end_time_option, config, parameters=cursor_model.parameters or {} 2158 ) 2159 if cursor_model.end_time_option 2160 else None 2161 ) 2162 start_time_option = ( 2163 self._create_component_from_model( 2164 cursor_model.start_time_option, config, parameters=cursor_model.parameters or {} 2165 ) 2166 if cursor_model.start_time_option 2167 else None 2168 ) 2169 2170 datetime_request_options_provider = DatetimeBasedRequestOptionsProvider( 2171 start_time_option=start_time_option, 2172 end_time_option=end_time_option, 2173 partition_field_start=cursor_model.partition_field_start, 2174 partition_field_end=cursor_model.partition_field_end, 2175 config=config, 2176 parameters=model.parameters or {}, 2177 ) 2178 request_options_provider = ( 2179 datetime_request_options_provider 2180 if not isinstance(concurrent_cursor, ConcurrentPerPartitionCursor) 2181 else PerPartitionRequestOptionsProvider( 2182 partition_router, datetime_request_options_provider 2183 ) 2184 ) 2185 elif isinstance(model.incremental_sync, SyncTokenCursorModel): 2186 sync_token_cursor_model = model.incremental_sync 2187 token_option = self._create_component_from_model( 2188 sync_token_cursor_model.token_option, 2189 config, 2190 parameters=sync_token_cursor_model.parameters or {}, 2191 ) 2192 sync_token_request_options_provider = SyncTokenRequestOptionsProvider( 2193 token_option=token_option, 2194 config=config, 2195 parameters=sync_token_cursor_model.parameters or {}, 2196 initial_token=sync_token_cursor_model.initial_token, 2197 ) 2198 request_options_provider = ( 2199 PerPartitionRequestOptionsProvider( 2200 partition_router, sync_token_request_options_provider 2201 ) 2202 if isinstance(concurrent_cursor, SyncTokenCursor) and concurrent_cursor.partitioned 2203 else sync_token_request_options_provider 2204 ) 2205 elif model.incremental_sync and isinstance( 2206 model.incremental_sync, IncrementingCountCursorModel 2207 ): 2208 if isinstance(concurrent_cursor, ConcurrentPerPartitionCursor): 2209 raise ValueError( 2210 "PerPartition does not support per partition states because switching to global state is time based" 2211 ) 2212 2213 cursor_model: IncrementingCountCursorModel = model.incremental_sync # type: ignore 2214 2215 start_time_option = ( 2216 self._create_component_from_model( 2217 cursor_model.start_value_option, # type: ignore # mypy still thinks cursor_model of type DatetimeBasedCursor 2218 config, 2219 parameters=cursor_model.parameters or {}, 2220 ) 2221 if cursor_model.start_value_option # type: ignore # mypy still thinks cursor_model of type DatetimeBasedCursor 2222 else None 2223 ) 2224 2225 # The concurrent engine defaults the start/end fields on the slice to "start" and "end", but 2226 # the default DatetimeBasedRequestOptionsProvider() sets them to start_time/end_time 2227 partition_field_start = "start" 2228 2229 request_options_provider = DatetimeBasedRequestOptionsProvider( 2230 start_time_option=start_time_option, 2231 partition_field_start=partition_field_start, 2232 config=config, 2233 parameters=model.parameters or {}, 2234 ) 2235 else: 2236 request_options_provider = None 2237 2238 transformations = [] 2239 if model.transformations: 2240 for transformation_model in model.transformations: 2241 transformations.append( 2242 self._create_component_from_model(model=transformation_model, config=config) 2243 ) 2244 file_uploader = None 2245 if model.file_uploader: 2246 file_uploader = self._create_component_from_model( 2247 model=model.file_uploader, config=config 2248 ) 2249 2250 stream_slicer: ConcurrentStreamSlicer = ( 2251 partition_router 2252 if isinstance(concurrent_cursor, FinalStateCursor) 2253 else concurrent_cursor 2254 ) 2255 2256 retriever = self._create_component_from_model( 2257 model=model.retriever, 2258 config=config, 2259 name=model.name, 2260 primary_key=primary_key, 2261 request_options_provider=request_options_provider, 2262 stream_slicer=stream_slicer, 2263 partition_router=partition_router, 2264 has_stop_condition_cursor=self._is_stop_condition_on_cursor(model), 2265 is_client_side_incremental_sync=self._is_client_side_filtering_enabled(model), 2266 cursor=concurrent_cursor, 2267 transformations=transformations, 2268 file_uploader=file_uploader, 2269 incremental_sync=model.incremental_sync, 2270 ) 2271 if isinstance(retriever, AsyncRetriever): 2272 stream_slicer = retriever.stream_slicer 2273 2274 schema_loader: SchemaLoader 2275 if model.schema_loader and isinstance(model.schema_loader, list): 2276 nested_schema_loaders = [ 2277 self._create_component_from_model(model=nested_schema_loader, config=config) 2278 for nested_schema_loader in model.schema_loader 2279 ] 2280 schema_loader = CompositeSchemaLoader( 2281 schema_loaders=nested_schema_loaders, parameters={} 2282 ) 2283 elif model.schema_loader: 2284 schema_loader = self._create_component_from_model( 2285 model=model.schema_loader, # type: ignore # If defined, schema_loader is guaranteed not to be a list and will be one of the existing base models 2286 config=config, 2287 ) 2288 else: 2289 options = model.parameters or {} 2290 if "name" not in options: 2291 options["name"] = model.name 2292 schema_loader = DefaultSchemaLoader(config=config, parameters=options) 2293 schema_loader = CachingSchemaLoaderDecorator(schema_loader) 2294 2295 stream_name = model.name or "" 2296 return DefaultStream( 2297 partition_generator=StreamSlicerPartitionGenerator( 2298 DeclarativePartitionFactory( 2299 stream_name, 2300 schema_loader, 2301 retriever, 2302 self._message_repository, 2303 ), 2304 stream_slicer, 2305 slice_limit=self._limit_slices_fetched, 2306 ), 2307 name=stream_name, 2308 json_schema=schema_loader.get_json_schema, 2309 primary_key=get_primary_key_from_stream(primary_key), 2310 cursor_field=( 2311 concurrent_cursor.cursor_field 2312 if hasattr(concurrent_cursor, "cursor_field") 2313 else None 2314 ), 2315 logger=logging.getLogger(f"airbyte.{stream_name}"), 2316 cursor=concurrent_cursor, 2317 supports_file_transfer=hasattr(model, "file_uploader") and bool(model.file_uploader), 2318 source_defined_incremental=isinstance(model.incremental_sync, SyncTokenCursorModel), 2319 ) 2320 2321 @staticmethod 2322 def _flatten_partition_router_models(router_model: Any) -> list[Any]: 2323 if router_model is None: 2324 return [] 2325 if isinstance(router_model, list): 2326 return [ 2327 nested_router 2328 for router in router_model 2329 for nested_router in ModelToComponentFactory._flatten_partition_router_models( 2330 router 2331 ) 2332 ] 2333 if isinstance(router_model, UnionPartitionRouterModel): 2334 return [ 2335 router_model, 2336 *ModelToComponentFactory._flatten_partition_router_models( 2337 getattr(router_model, "partition_routers", None) 2338 ), 2339 ] 2340 if isinstance(router_model, GroupingPartitionRouterModel): 2341 return [ 2342 router_model, 2343 *ModelToComponentFactory._flatten_partition_router_models( 2344 getattr(router_model, "underlying_partition_router", None) 2345 ), 2346 ] 2347 return [router_model] 2348 2349 @staticmethod 2350 def _validate_sync_token_stream(model: DeclarativeStreamModel) -> None: 2351 cursor_model = model.incremental_sync 2352 stream_name = model.name or "" 2353 retriever_model = model.retriever 2354 router_model = getattr(retriever_model, "partition_router", None) 2355 routers = ModelToComponentFactory._flatten_partition_router_models(router_model) 2356 for router in routers: 2357 if isinstance(router, SubstreamPartitionRouterModel): 2358 for parent in router.parent_stream_configs: 2359 parent_sync = getattr(parent.stream, "incremental_sync", None) 2360 if isinstance(parent.stream, StateDelegatingStreamModel): 2361 parent_sync = parent.stream.incremental_stream.incremental_sync 2362 if parent.incremental_dependency and isinstance( 2363 parent_sync, SyncTokenCursorModel 2364 ): 2365 raise ValueError( 2366 f"Stream {stream_name}: a SyncTokenCursor parent cannot be used with " 2367 "`incremental_dependency`." 2368 ) 2369 if not isinstance(cursor_model, SyncTokenCursorModel): 2370 return 2371 datetime_only_keys = { 2372 "start_datetime", 2373 "end_datetime", 2374 "step", 2375 "cursor_granularity", 2376 "datetime_format", 2377 "cursor_datetime_formats", 2378 "start_time_option", 2379 "end_time_option", 2380 "lookback_window", 2381 "partition_field_start", 2382 "partition_field_end", 2383 "is_data_feed", 2384 "is_client_side_incremental", 2385 "is_compare_strictly", 2386 "global_substream_cursor", 2387 "clamping", 2388 } 2389 unsupported = sorted( 2390 datetime_only_keys.intersection(cursor_model.__dict__).difference( 2391 cursor_model.parameters or {} 2392 ) 2393 ) 2394 if unsupported: 2395 raise ValueError( 2396 f"Stream {stream_name}: SyncTokenCursor does not support datetime-only setting(s): " 2397 f"{', '.join(unsupported)}." 2398 ) 2399 if isinstance(model.retriever, AsyncRetrieverModel): 2400 raise ValueError( 2401 f"Stream {stream_name}: SyncTokenCursor is not supported with AsyncRetriever; " 2402 "use SimpleRetriever so response pages can be observed." 2403 ) 2404 if isinstance(retriever_model, CustomRetrieverModel): 2405 LOGGER.warning( 2406 "Stream %s: CustomRetriever does not expose response pages to SyncTokenCursor; " 2407 "no sync token will be captured.", 2408 stream_name, 2409 ) 2410 2411 for router in routers: 2412 if isinstance(router, GroupingPartitionRouterModel): 2413 raise ValueError( 2414 f"Stream {stream_name}: SyncTokenCursor does not support GroupingPartitionRouter; " 2415 "use an ungrouped partition router." 2416 ) 2417 if isinstance(router, ListPartitionRouterModel) and router.cursor_field == "token": 2418 raise ValueError( 2419 f"Stream {stream_name}: partition router field 'token' conflicts with SyncTokenCursor." 2420 ) 2421 if isinstance(router, SubstreamPartitionRouterModel): 2422 for parent in router.parent_stream_configs: 2423 if parent.partition_field == "token": 2424 raise ValueError( 2425 f"Stream {stream_name}: parent partition field 'token' conflicts with SyncTokenCursor." 2426 ) 2427 if parent.lazy_read_pointer: 2428 raise ValueError( 2429 f"Stream {stream_name}: SyncTokenCursor is not supported with parent " 2430 "`lazy_read_pointer`; remove the lazy pointer." 2431 ) 2432 if parent.incremental_dependency: 2433 raise ValueError( 2434 f"Stream {stream_name}: SyncTokenCursor is not supported with " 2435 "`incremental_dependency` on a SubstreamPartitionRouter." 2436 ) 2437 pagination_reset = getattr(retriever_model, "pagination_reset", None) 2438 if pagination_reset and pagination_reset.limits: 2439 raise ValueError( 2440 f"Stream {stream_name}: pagination reset `limits` are not supported with SyncTokenCursor." 2441 ) 2442 requester_model = getattr(retriever_model, "requester", None) 2443 query_properties = [getattr(requester_model, "query_properties", None)] 2444 request_parameters = getattr(requester_model, "request_parameters", None) 2445 if isinstance(request_parameters, Mapping): 2446 query_properties.extend( 2447 value 2448 for value in request_parameters.values() 2449 if isinstance(value, QueryPropertiesModel) 2450 or (isinstance(value, Mapping) and value.get("type") == "QueryProperties") 2451 ) 2452 for properties in query_properties: 2453 property_chunking = ( 2454 properties.get("property_chunking") 2455 if isinstance(properties, Mapping) 2456 else getattr(properties, "property_chunking", None) 2457 ) 2458 if property_chunking: 2459 raise ValueError( 2460 f"Stream {stream_name}: query property chunking is not supported with " 2461 "SyncTokenCursor." 2462 ) 2463 2464 def _warn_on_ineffective_incremental_dependency(self, model: DeclarativeStreamModel) -> None: 2465 """ 2466 `incremental_dependency: true` only takes effect when the substream defines its own 2467 `incremental_sync`: the parent cursor is persisted under the `parent_state` key of the 2468 substream's state, which is only emitted by incremental substreams. On a stream without 2469 `incremental_sync`, the setting is silently ignored and all parent records are re-read on 2470 every sync, so we warn about the misconfiguration instead. 2471 """ 2472 if model.incremental_sync: 2473 return 2474 2475 partition_router = getattr(model.retriever, "partition_router", None) 2476 if not partition_router: 2477 return 2478 2479 routers = partition_router if isinstance(partition_router, list) else [partition_router] 2480 for router in routers: 2481 if isinstance(router, GroupingPartitionRouterModel): 2482 router = router.underlying_partition_router 2483 if isinstance(router, SubstreamPartitionRouterModel) and any( 2484 parent_stream_config.incremental_dependency 2485 for parent_stream_config in router.parent_stream_configs 2486 ): 2487 LOGGER.warning( 2488 f"Stream `{model.name}` has `incremental_dependency: true` in its parent stream configuration but does not define `incremental_sync`. " 2489 "The parent stream's cursor is only persisted in the state of an incremental substream, so this setting has no effect and all parent records will be re-read on every sync. " 2490 "Define `incremental_sync` on this stream or remove `incremental_dependency`." 2491 ) 2492 return 2493 2494 def _migrate_state(self, model: DeclarativeStreamModel, config: Config) -> None: 2495 stream_name = model.name or "" 2496 stream_state = self._connector_state_manager.get_stream_state( 2497 stream_name=stream_name, namespace=None 2498 ) 2499 if model.state_migrations: 2500 state_transformations = [ 2501 self._create_component_from_model(state_migration, config, declarative_stream=model) 2502 for state_migration in model.state_migrations 2503 ] 2504 else: 2505 state_transformations = [] 2506 stream_state = self.apply_stream_state_migrations(state_transformations, stream_state) 2507 self._connector_state_manager.update_state_for_stream( 2508 stream_name=stream_name, namespace=None, value=stream_state 2509 ) 2510 2511 def _is_stop_condition_on_cursor(self, model: DeclarativeStreamModel) -> bool: 2512 return bool( 2513 model.incremental_sync 2514 and hasattr(model.incremental_sync, "is_data_feed") 2515 and model.incremental_sync.is_data_feed 2516 ) 2517 2518 def _is_client_side_filtering_enabled(self, model: DeclarativeStreamModel) -> bool: 2519 return bool( 2520 model.incremental_sync 2521 and hasattr(model.incremental_sync, "is_client_side_incremental") 2522 and model.incremental_sync.is_client_side_incremental 2523 ) 2524 2525 def _build_stream_slicer_from_partition_router( 2526 self, 2527 model: Union[ 2528 AsyncRetrieverModel, 2529 CustomRetrieverModel, 2530 SimpleRetrieverModel, 2531 ], 2532 config: Config, 2533 stream_name: Optional[str] = None, 2534 **kwargs: Any, 2535 ) -> PartitionRouter: 2536 if ( 2537 hasattr(model, "partition_router") 2538 and isinstance(model, (SimpleRetrieverModel, AsyncRetrieverModel, CustomRetrieverModel)) 2539 and model.partition_router 2540 ): 2541 stream_slicer_model = model.partition_router 2542 if isinstance(stream_slicer_model, list): 2543 return CartesianProductStreamSlicer( 2544 [ 2545 self._create_component_from_model( 2546 model=slicer, config=config, stream_name=stream_name or "" 2547 ) 2548 for slicer in stream_slicer_model 2549 ], 2550 parameters={}, 2551 ) 2552 elif isinstance(stream_slicer_model, dict): 2553 # partition router comes from CustomRetrieverModel therefore has not been parsed as a model 2554 params = stream_slicer_model.get("$parameters") 2555 if not isinstance(params, dict): 2556 params = {} 2557 stream_slicer_model["$parameters"] = params 2558 2559 if stream_name is not None: 2560 params["stream_name"] = stream_name 2561 2562 return self._create_nested_component( # type: ignore[no-any-return] # There is no guarantee that this will return a stream slicer. If not, we expect an AttributeError during the call to `stream_slices` 2563 model, 2564 "partition_router", 2565 stream_slicer_model, 2566 config, 2567 **kwargs, 2568 ) 2569 else: 2570 return self._create_component_from_model( # type: ignore[no-any-return] # Will be created PartitionRouter as stream_slicer_model is model.partition_router 2571 model=stream_slicer_model, config=config, stream_name=stream_name or "" 2572 ) 2573 return SinglePartitionRouter(parameters={}) 2574 2575 def _build_concurrent_cursor( 2576 self, 2577 model: DeclarativeStreamModel, 2578 stream_slicer: Optional[PartitionRouter], 2579 config: Config, 2580 ) -> Cursor: 2581 stream_name = model.name or "" 2582 stream_state = self._connector_state_manager.get_stream_state(stream_name, None) 2583 2584 if isinstance(model.incremental_sync, SyncTokenCursorModel): 2585 return self.create_sync_token_cursor( 2586 model.incremental_sync, 2587 config=config, 2588 stream_name=stream_name, 2589 stream_state=stream_state, 2590 partition_router=stream_slicer or SinglePartitionRouter(parameters={}), 2591 retriever_model=model.retriever, 2592 ) 2593 2594 if ( 2595 model.incremental_sync 2596 and stream_slicer 2597 and not isinstance(stream_slicer, SinglePartitionRouter) 2598 ): 2599 if isinstance(model.incremental_sync, IncrementingCountCursorModel): 2600 # We don't currently support usage of partition routing and IncrementingCountCursor at the 2601 # same time because we didn't solve for design questions like what the lookback window would 2602 # be as well as global cursor fall backs. We have not seen customers that have needed both 2603 # at the same time yet and are currently punting on this until we need to solve it. 2604 raise ValueError( 2605 f"The low-code framework does not currently support usage of a PartitionRouter and an IncrementingCountCursor at the same time. Please specify only one of these options for stream {stream_name}." 2606 ) 2607 return self.create_concurrent_cursor_from_perpartition_cursor( # type: ignore # This is a known issue that we are creating and returning a ConcurrentCursor which does not technically implement the (low-code) StreamSlicer. However, (low-code) StreamSlicer and ConcurrentCursor both implement StreamSlicer.stream_slices() which is the primary method needed for checkpointing 2608 state_manager=self._connector_state_manager, 2609 model_type=DatetimeBasedCursorModel, 2610 component_definition=model.incremental_sync.__dict__, 2611 stream_name=stream_name, 2612 stream_state=stream_state, 2613 stream_namespace=None, 2614 config=config or {}, 2615 partition_router=stream_slicer, 2616 attempt_to_create_cursor_if_not_provided=True, # FIXME can we remove that now? 2617 ) 2618 elif model.incremental_sync: 2619 if type(model.incremental_sync) == IncrementingCountCursorModel: 2620 return self.create_concurrent_cursor_from_incrementing_count_cursor( # type: ignore # This is a known issue that we are creating and returning a ConcurrentCursor which does not technically implement the (low-code) StreamSlicer. However, (low-code) StreamSlicer and ConcurrentCursor both implement StreamSlicer.stream_slices() which is the primary method needed for checkpointing 2621 model_type=IncrementingCountCursorModel, 2622 component_definition=model.incremental_sync.__dict__, 2623 stream_name=stream_name, 2624 stream_namespace=None, 2625 stream_state=stream_state, 2626 config=config or {}, 2627 ) 2628 elif type(model.incremental_sync) == DatetimeBasedCursorModel: 2629 return self.create_concurrent_cursor_from_datetime_based_cursor( # type: ignore # This is a known issue that we are creating and returning a ConcurrentCursor which does not technically implement the (low-code) StreamSlicer. However, (low-code) StreamSlicer and ConcurrentCursor both implement StreamSlicer.stream_slices() which is the primary method needed for checkpointing 2630 model_type=type(model.incremental_sync), 2631 component_definition=model.incremental_sync.__dict__, 2632 stream_name=stream_name, 2633 stream_namespace=None, 2634 stream_state=stream_state, 2635 config=config or {}, 2636 attempt_to_create_cursor_if_not_provided=True, 2637 ) 2638 else: 2639 raise ValueError( 2640 f"Incremental sync of type {type(model.incremental_sync)} is not supported" 2641 ) 2642 return FinalStateCursor(stream_name, None, self._message_repository) 2643 2644 def create_sync_token_cursor( 2645 self, 2646 model: SyncTokenCursorModel, 2647 *, 2648 config: Config, 2649 stream_name: str, 2650 stream_state: Optional[Mapping[str, Any]], 2651 partition_router: PartitionRouter, 2652 retriever_model: Any, 2653 ) -> SyncTokenCursor: 2654 decoder_model = getattr(retriever_model, "decoder", None) 2655 decoder = ( 2656 self._create_component_from_model(model=decoder_model, config=config) 2657 if isinstance(decoder_model, BaseModel) 2658 else JsonDecoder(parameters={}) 2659 ) 2660 if self._decoder_streams_in_production(decoder): 2661 raise AirbyteTracedException( 2662 message=f"Stream {stream_name}: SyncTokenCursor is not supported with a streaming decoder.", 2663 internal_message=( 2664 f"Stream {stream_name} uses {type(decoder).__name__}, which streams the response " 2665 f"and can only read it once. SyncTokenCursor reads the token from the response " 2666 f"after the record selector has consumed it. Use a non-streaming decoder such " 2667 f"as JsonDecoder or XmlDecoder." 2668 ), 2669 failure_type=FailureType.config_error, 2670 ) 2671 return SyncTokenCursor( 2672 stream_name=stream_name, 2673 stream_namespace=None, 2674 stream_state=stream_state, 2675 message_repository=self._message_repository, 2676 connector_state_manager=self._connector_state_manager, 2677 partition_router=partition_router, 2678 token_path=model.token_path, 2679 decoder=decoder, 2680 config=config or {}, 2681 parameters=model.parameters or {}, 2682 completion_condition=model.completion_condition, 2683 cursor_field=( 2684 InterpolatedString.create( 2685 model.cursor_field, parameters=model.parameters or {} 2686 ).eval(config=config) 2687 if model.cursor_field 2688 else None 2689 ), 2690 ) 2691 2692 def create_default_error_handler( 2693 self, model: DefaultErrorHandlerModel, config: Config, **kwargs: Any 2694 ) -> DefaultErrorHandler: 2695 backoff_strategies = [] 2696 if model.backoff_strategies: 2697 for backoff_strategy_model in model.backoff_strategies: 2698 backoff_strategies.append( 2699 self._create_component_from_model(model=backoff_strategy_model, config=config) 2700 ) 2701 2702 response_filters = [] 2703 if model.response_filters: 2704 for response_filter_model in model.response_filters: 2705 response_filters.append( 2706 self._create_component_from_model(model=response_filter_model, config=config) 2707 ) 2708 response_filters.append( 2709 HttpResponseFilter(config=config, parameters=model.parameters or {}) 2710 ) 2711 2712 return DefaultErrorHandler( 2713 backoff_strategies=backoff_strategies, 2714 max_retries=model.max_retries, 2715 response_filters=response_filters, 2716 config=config, 2717 parameters=model.parameters or {}, 2718 ) 2719 2720 def create_default_paginator( 2721 self, 2722 model: DefaultPaginatorModel, 2723 config: Config, 2724 *, 2725 url_base: str, 2726 extractor_model: Optional[ 2727 Union[CustomRecordExtractorModel, DpathExtractorModel, CombinedExtractorModel] 2728 ] = None, 2729 decoder: Optional[Decoder] = None, 2730 cursor_used_for_stop_condition: Optional[Cursor] = None, 2731 ) -> Union[DefaultPaginator, PaginatorTestReadDecorator]: 2732 if decoder: 2733 if self._is_supported_decoder_for_pagination(decoder): 2734 decoder_to_use = PaginationDecoderDecorator(decoder=decoder) 2735 else: 2736 raise ValueError(self._UNSUPPORTED_DECODER_ERROR.format(decoder_type=type(decoder))) 2737 else: 2738 decoder_to_use = PaginationDecoderDecorator(decoder=JsonDecoder(parameters={})) 2739 page_size_option = ( 2740 self._create_component_from_model(model=model.page_size_option, config=config) 2741 if model.page_size_option 2742 else None 2743 ) 2744 page_token_option = ( 2745 self._create_component_from_model(model=model.page_token_option, config=config) 2746 if model.page_token_option 2747 else None 2748 ) 2749 pagination_strategy = self._create_component_from_model( 2750 model=model.pagination_strategy, 2751 config=config, 2752 decoder=decoder_to_use, 2753 extractor_model=extractor_model, 2754 ) 2755 if cursor_used_for_stop_condition: 2756 pagination_strategy = StopConditionPaginationStrategyDecorator( 2757 pagination_strategy, CursorStopCondition(cursor_used_for_stop_condition) 2758 ) 2759 paginator = DefaultPaginator( 2760 decoder=decoder_to_use, 2761 page_size_option=page_size_option, 2762 page_token_option=page_token_option, 2763 pagination_strategy=pagination_strategy, 2764 url_base=url_base, 2765 config=config, 2766 parameters=model.parameters or {}, 2767 ) 2768 if self._limit_pages_fetched_per_slice: 2769 return PaginatorTestReadDecorator(paginator, self._limit_pages_fetched_per_slice) 2770 return paginator 2771 2772 @staticmethod 2773 def _is_decoder_downgraded_by_connector_builder(decoder: Decoder) -> bool: 2774 """Is this the buffered stand-in the Connector Builder builds for a streaming decoder? 2775 2776 `create_csv_decoder`, `create_jsonl_decoder`, `create_json_items_decoder` and 2777 `create_gzip_decoder` build a `CompositeRawDecoder` with `stream_response=False` when 2778 `_emit_connector_builder_messages` is set, so a Builder test read can be replayed. Those 2779 four instances are the only ones that stream in production while reporting 2780 `is_stream_response() == False` in the Builder. 2781 2782 The match is on the exact class and on the parser, not on `isinstance`: a `CustomDecoder` 2783 subclassing `CompositeRawDecoder` with `stream_response=False`, or a buffered 2784 `CompositeRawDecoder` built around a `JsonParser`, reads the body from `response.content` 2785 in production too and must not be rejected. 2786 """ 2787 return type(decoder) is CompositeRawDecoder and isinstance( 2788 decoder.parser, (CsvParser, JsonLineParser, JsonItemsParser, GzipParser) 2789 ) 2790 2791 def _decoder_streams_in_production(self, decoder: Decoder) -> bool: 2792 """Can this decoder read a response body only once in production? 2793 2794 True for decoders that stream (`is_stream_response()`), and, in the Connector Builder, for 2795 the buffered stand-ins it builds for them (`_is_decoder_downgraded_by_connector_builder`). 2796 """ 2797 inner_decoder = ( 2798 decoder.decoder if isinstance(decoder, PaginationDecoderDecorator) else decoder 2799 ) 2800 return inner_decoder.is_stream_response() or ( 2801 self._emit_connector_builder_messages 2802 and self._is_decoder_downgraded_by_connector_builder(inner_decoder) 2803 ) 2804 2805 def _reject_combined_extractor_over_streaming_decoder(self, decoder: Optional[Decoder]) -> None: 2806 """Refuse to build a `CombinedExtractor` whose response body can only be read once. 2807 2808 Every sub-extractor is handed the same `requests.Response`. A streaming decoder consumes 2809 and closes `response.raw`, so every sub-extractor after the first reads a closed 2810 `urllib3.HTTPResponse`, which returns an empty body instead of raising: `union` emits only 2811 the first sub-extractor's records and `first_match` emits nothing when the first path 2812 misses. Rejecting the manifest is the only way to make that loud. 2813 2814 The Connector Builder downgrades those same decoders to `stream_response=False` so a test 2815 read can be replayed, which would let a manifest look correct in the Builder and lose 2816 records once published. The downgraded decoders are therefore rejected in the Builder too. 2817 """ 2818 if decoder is None or not self._decoder_streams_in_production(decoder): 2819 return 2820 inner_decoder = ( 2821 decoder.decoder if isinstance(decoder, PaginationDecoderDecorator) else decoder 2822 ) 2823 raise AirbyteTracedException( 2824 message="CombinedExtractor is not supported with a streaming decoder.", 2825 internal_message=( 2826 f"CombinedExtractor was configured with {type(inner_decoder).__name__}, which " 2827 f"streams the response and can only be read once. Sub-extractors after the first " 2828 f"would read a closed response and silently return no records. The streaming " 2829 f"decoders are CsvDecoder, JsonlDecoder, JsonItemsDecoder, GzipDecoder and " 2830 f"IterableDecoder; use JsonDecoder, XmlDecoder or ZipfileDecoder, or declare a " 2831 f"single extractor." 2832 ), 2833 failure_type=FailureType.config_error, 2834 ) 2835 2836 def create_combined_extractor( 2837 self, 2838 model: CombinedExtractorModel, 2839 config: Config, 2840 decoder: Optional[Decoder] = None, 2841 **kwargs: Any, 2842 ) -> CombinedExtractor: 2843 self._reject_combined_extractor_over_streaming_decoder(decoder) 2844 extractors = [ 2845 self._create_component_from_model(model=sub_extractor, config=config, decoder=decoder) 2846 for sub_extractor in model.extractors 2847 ] 2848 return CombinedExtractor( 2849 extractors=extractors, 2850 mode=CombineMode(model.mode.value) if model.mode else CombineMode.union, 2851 skip_empty_records=bool(model.skip_empty_records), 2852 parameters=model.parameters or {}, 2853 ) 2854 2855 def create_dpath_extractor( 2856 self, 2857 model: DpathExtractorModel, 2858 config: Config, 2859 decoder: Optional[Decoder] = None, 2860 **kwargs: Any, 2861 ) -> DpathExtractor: 2862 if decoder: 2863 decoder_to_use = decoder 2864 else: 2865 decoder_to_use = JsonDecoder(parameters={}) 2866 model_field_path: List[Union[InterpolatedString, str]] = [x for x in model.field_path] 2867 2868 record_expander = None 2869 if model.record_expander: 2870 record_expander = self._create_component_from_model( 2871 model=model.record_expander, 2872 config=config, 2873 ) 2874 2875 return DpathExtractor( 2876 decoder=decoder_to_use, 2877 field_path=model_field_path, 2878 config=config, 2879 parameters=model.parameters or {}, 2880 record_expander=record_expander, 2881 ) 2882 2883 def create_record_expander( 2884 self, 2885 model: RecordExpanderModel, 2886 config: Config, 2887 **kwargs: Any, 2888 ) -> RecordExpander: 2889 truncated_list_retriever = None 2890 suppress_incomplete_fetch_warning = False 2891 if model.truncated_list_retriever: 2892 retriever_model = model.truncated_list_retriever 2893 name = "record_expander_truncated_list" 2894 # `CustomRetriever` allows extra fields, so read from the dumped model to cover both types. 2895 retriever_fields = retriever_model.dict() 2896 for unsupported_option in ("partition_router", "pagination_reset"): 2897 if retriever_fields.get(unsupported_option): 2898 raise ValueError( 2899 f"`{unsupported_option}` is not supported on `truncated_list_retriever`." 2900 ) 2901 log_formatter = lambda response: format_http_message( 2902 response, 2903 f"Record expander '{name}' request", 2904 "Request performed in order to fetch the complete nested list of a truncated record.", 2905 name, 2906 is_auxiliary=True, 2907 ) 2908 # `name`/`primary_key`/`transformations` are also what a `CustomRetriever` forwards to its 2909 # nested `requester`/`record_selector`, so they are passed for both retriever types. 2910 truncated_list_retriever = self._create_component_from_model( 2911 model=retriever_model, 2912 config=config, 2913 name=name, 2914 primary_key=None, 2915 transformations=[], 2916 log_formatter=log_formatter, 2917 ) 2918 # Only a capped paginator makes a shortfall expected; `NoPagination` and retrievers 2919 # without a paginator are never capped. 2920 suppress_incomplete_fetch_warning = isinstance( 2921 truncated_list_retriever, SimpleRetriever 2922 ) and isinstance(truncated_list_retriever.paginator, PaginatorTestReadDecorator) 2923 return RecordExpander( 2924 expand_records_from_field=model.expand_records_from_field, 2925 config=config, 2926 parameters=model.parameters or {}, 2927 remain_original_record=model.remain_original_record or False, 2928 parent_fields=[ 2929 self._create_component_from_model(model=parent_field, config=config) 2930 for parent_field in model.parent_fields 2931 ] 2932 if model.parent_fields 2933 else None, 2934 merge_parent=model.merge_parent or False, 2935 on_no_records=OnNoRecords(model.on_no_records.value) 2936 if model.on_no_records 2937 else OnNoRecords.skip, 2938 truncation_indicator_path=model.truncation_indicator_path, 2939 truncated_list_retriever=truncated_list_retriever, 2940 message_repository=self._message_repository, 2941 suppress_incomplete_fetch_warning=suppress_incomplete_fetch_warning, 2942 ) 2943 2944 @staticmethod 2945 def create_parent_field_path( 2946 model: ParentFieldPathModel, 2947 config: Config, 2948 **kwargs: Any, 2949 ) -> ParentFieldPath: 2950 return ParentFieldPath( 2951 parent_path=model.parent_path, 2952 record_path=model.record_path, 2953 config=config, 2954 parameters=model.parameters or {}, 2955 ) 2956 2957 @staticmethod 2958 def create_response_to_file_extractor( 2959 model: ResponseToFileExtractorModel, 2960 **kwargs: Any, 2961 ) -> ResponseToFileExtractor: 2962 return ResponseToFileExtractor( 2963 parameters=model.parameters or {}, 2964 preserve_na_values=model.preserve_na_values or False, 2965 ) 2966 2967 @staticmethod 2968 def create_exponential_backoff_strategy( 2969 model: ExponentialBackoffStrategyModel, config: Config 2970 ) -> ExponentialBackoffStrategy: 2971 ModelToComponentFactory._validate_jitter_range(model.jitter_range_in_seconds) 2972 return ExponentialBackoffStrategy( 2973 factor=model.factor or 5, 2974 jitter_range_in_seconds=model.jitter_range_in_seconds, 2975 parameters=model.parameters or {}, 2976 config=config, 2977 ) 2978 2979 @staticmethod 2980 def create_group_by_key(model: GroupByKeyMergeStrategyModel, config: Config) -> GroupByKey: 2981 return GroupByKey(model.key, config=config, parameters=model.parameters or {}) 2982 2983 def create_http_requester( 2984 self, 2985 model: HttpRequesterModel, 2986 config: Config, 2987 decoder: Decoder = JsonDecoder(parameters={}), 2988 query_properties_key: Optional[str] = None, 2989 use_cache: Optional[bool] = None, 2990 *, 2991 name: str, 2992 ) -> HttpRequester: 2993 authenticator = ( 2994 self._create_component_from_model( 2995 model=model.authenticator, 2996 config=config, 2997 url_base=model.url or model.url_base, 2998 name=name, 2999 decoder=decoder, 3000 ) 3001 if model.authenticator 3002 else None 3003 ) 3004 error_handler = ( 3005 self._create_component_from_model(model=model.error_handler, config=config) 3006 if model.error_handler 3007 else DefaultErrorHandler( 3008 backoff_strategies=[], 3009 response_filters=[], 3010 config=config, 3011 parameters=model.parameters or {}, 3012 ) 3013 ) 3014 3015 api_budget = self._api_budget 3016 3017 request_options_provider = InterpolatedRequestOptionsProvider( 3018 request_body=model.request_body, 3019 request_body_data=model.request_body_data, 3020 request_body_json=model.request_body_json, 3021 request_headers=model.request_headers, 3022 request_parameters=model.request_parameters, # type: ignore # QueryProperties have been removed in `create_simple_retriever` 3023 query_properties_key=query_properties_key, 3024 config=config, 3025 parameters=model.parameters or {}, 3026 ) 3027 3028 assert model.use_cache is not None # for mypy 3029 assert model.http_method is not None # for mypy 3030 3031 should_use_cache = (model.use_cache or bool(use_cache)) and not self._disable_cache 3032 3033 return HttpRequester( 3034 name=name, 3035 url=model.url, 3036 url_base=model.url_base, 3037 path=model.path, 3038 authenticator=authenticator, 3039 error_handler=error_handler, 3040 api_budget=api_budget, 3041 http_method=HttpMethod[model.http_method.value], 3042 request_options_provider=request_options_provider, 3043 config=config, 3044 disable_retries=self._disable_retries, 3045 parameters=model.parameters or {}, 3046 message_repository=self._message_repository, 3047 use_cache=should_use_cache, 3048 connect_timeout_in_seconds=model.connect_timeout_in_seconds, 3049 read_timeout_in_seconds=model.read_timeout_in_seconds, 3050 use_tcp_keepalive=model.use_tcp_keepalive or False, 3051 decoder=decoder, 3052 stream_response=decoder.is_stream_response() if decoder else False, 3053 ) 3054 3055 @staticmethod 3056 def create_http_response_filter( 3057 model: HttpResponseFilterModel, config: Config, **kwargs: Any 3058 ) -> HttpResponseFilter: 3059 if model.action: 3060 action = ResponseAction(model.action.value) 3061 else: 3062 action = None 3063 3064 failure_type = FailureType(model.failure_type.value) if model.failure_type else None 3065 3066 http_codes = ( 3067 set(model.http_codes) if model.http_codes else set() 3068 ) # JSON schema notation has no set data type. The schema enforces an array of unique elements 3069 3070 return HttpResponseFilter( 3071 action=action, 3072 failure_type=failure_type, 3073 error_message=model.error_message or "", 3074 error_message_contains=model.error_message_contains or "", 3075 http_codes=http_codes, 3076 predicate=model.predicate or "", 3077 config=config, 3078 parameters=model.parameters or {}, 3079 ) 3080 3081 @staticmethod 3082 def create_inline_schema_loader( 3083 model: InlineSchemaLoaderModel, config: Config, **kwargs: Any 3084 ) -> InlineSchemaLoader: 3085 return InlineSchemaLoader(schema=model.schema_ or {}, parameters={}) 3086 3087 def create_complex_field_type( 3088 self, model: ComplexFieldTypeModel, config: Config, **kwargs: Any 3089 ) -> ComplexFieldType: 3090 items = ( 3091 self._create_component_from_model(model=model.items, config=config) 3092 if isinstance(model.items, ComplexFieldTypeModel) 3093 else model.items 3094 ) 3095 3096 return ComplexFieldType(field_type=model.field_type, items=items) 3097 3098 def create_types_map(self, model: TypesMapModel, config: Config, **kwargs: Any) -> TypesMap: 3099 target_type = ( 3100 self._create_component_from_model(model=model.target_type, config=config) 3101 if isinstance(model.target_type, ComplexFieldTypeModel) 3102 else model.target_type 3103 ) 3104 3105 return TypesMap( 3106 target_type=target_type, 3107 current_type=model.current_type, 3108 condition=model.condition if model.condition is not None else "True", 3109 ) 3110 3111 def create_schema_type_identifier( 3112 self, model: SchemaTypeIdentifierModel, config: Config, **kwargs: Any 3113 ) -> SchemaTypeIdentifier: 3114 types_mapping = [] 3115 if model.types_mapping: 3116 types_mapping.extend( 3117 [ 3118 self._create_component_from_model(types_map, config=config) 3119 for types_map in model.types_mapping 3120 ] 3121 ) 3122 model_schema_pointer: List[Union[InterpolatedString, str]] = ( 3123 [x for x in model.schema_pointer] if model.schema_pointer else [] 3124 ) 3125 model_key_pointer: List[Union[InterpolatedString, str]] = [x for x in model.key_pointer] 3126 model_type_pointer: Optional[List[Union[InterpolatedString, str]]] = ( 3127 [x for x in model.type_pointer] if model.type_pointer else None 3128 ) 3129 3130 return SchemaTypeIdentifier( 3131 schema_pointer=model_schema_pointer, 3132 key_pointer=model_key_pointer, 3133 type_pointer=model_type_pointer, 3134 types_mapping=types_mapping, 3135 parameters=model.parameters or {}, 3136 ) 3137 3138 def create_dynamic_schema_loader( 3139 self, model: DynamicSchemaLoaderModel, config: Config, **kwargs: Any 3140 ) -> DynamicSchemaLoader: 3141 schema_transformations = [] 3142 if model.schema_transformations: 3143 for transformation_model in model.schema_transformations: 3144 schema_transformations.append( 3145 self._create_component_from_model(model=transformation_model, config=config) 3146 ) 3147 name = "dynamic_properties" 3148 partition_router = self._build_stream_slicer_from_partition_router(model.retriever, config) 3149 retriever_kwargs: Dict[str, Any] = {} 3150 if isinstance(model.retriever, AsyncRetrieverModel): 3151 if model.retriever.partition_router: 3152 raise ValueError( 3153 "DynamicSchemaLoader does not support an AsyncRetriever with a partition_router: only the first record is read, so jobs for other partitions would be orphaned. Remove the partition_router from the schema loader's AsyncRetriever." 3154 ) 3155 # create_async_retriever requires stream_slicer; passing it to a CustomRetriever would override its own 3156 retriever_kwargs["stream_slicer"] = partition_router 3157 retriever = self._create_component_from_model( 3158 model=model.retriever, 3159 config=config, 3160 name=name, 3161 primary_key=None, 3162 partition_router=partition_router, 3163 **retriever_kwargs, 3164 transformations=[], 3165 use_cache=True, 3166 log_formatter=( 3167 lambda response: format_http_message( 3168 response, 3169 f"Schema loader '{name}' request", 3170 f"Request performed in order to extract schema.", 3171 name, 3172 is_auxiliary=True, 3173 ) 3174 ), 3175 ) 3176 schema_type_identifier = self._create_component_from_model( 3177 model.schema_type_identifier, config=config, parameters=model.parameters or {} 3178 ) 3179 schema_filter = ( 3180 self._create_component_from_model( 3181 model.schema_filter, config=config, parameters=model.parameters or {} 3182 ) 3183 if model.schema_filter is not None 3184 else None 3185 ) 3186 3187 return DynamicSchemaLoader( 3188 retriever=retriever, 3189 config=config, 3190 schema_transformations=schema_transformations, 3191 schema_filter=schema_filter, 3192 schema_type_identifier=schema_type_identifier, 3193 parameters=model.parameters or {}, 3194 ) 3195 3196 @staticmethod 3197 def create_json_decoder(model: JsonDecoderModel, config: Config, **kwargs: Any) -> Decoder: 3198 return JsonDecoder(parameters={}) 3199 3200 def create_csv_decoder(self, model: CsvDecoderModel, config: Config, **kwargs: Any) -> Decoder: 3201 return CompositeRawDecoder( 3202 parser=ModelToComponentFactory._get_parser(model, config), 3203 stream_response=False if self._emit_connector_builder_messages else True, 3204 ) 3205 3206 def create_jsonl_decoder( 3207 self, model: JsonlDecoderModel, config: Config, **kwargs: Any 3208 ) -> Decoder: 3209 return CompositeRawDecoder( 3210 parser=ModelToComponentFactory._get_parser(model, config), 3211 stream_response=False if self._emit_connector_builder_messages else True, 3212 ) 3213 3214 def create_json_items_decoder( 3215 self, model: JsonItemsDecoderModel, config: Config, **kwargs: Any 3216 ) -> Decoder: 3217 return CompositeRawDecoder( 3218 parser=ModelToComponentFactory._get_parser(model, config), 3219 stream_response=False if self._emit_connector_builder_messages else True, 3220 ) 3221 3222 def create_gzip_decoder( 3223 self, model: GzipDecoderModel, config: Config, **kwargs: Any 3224 ) -> Decoder: 3225 _compressed_response_types = { 3226 "gzip", 3227 "x-gzip", 3228 "gzip, deflate", 3229 "x-gzip, deflate", 3230 "application/zip", 3231 "application/gzip", 3232 "application/x-gzip", 3233 "application/x-zip-compressed", 3234 } 3235 3236 gzip_parser: GzipParser = ModelToComponentFactory._get_parser(model, config) # type: ignore # based on the model, we know this will be a GzipParser 3237 3238 if self._emit_connector_builder_messages: 3239 return CompositeRawDecoder(gzip_parser, False) 3240 3241 transport_gzip_parser = GzipParser(inner_parser=gzip_parser) 3242 return CompositeRawDecoder.by_headers( 3243 [ 3244 ({"Content-Encoding"}, {"gzip"}, transport_gzip_parser), 3245 ({"Content-Type"}, _compressed_response_types, gzip_parser), 3246 ], 3247 stream_response=True, 3248 fallback_parser=gzip_parser, 3249 ) 3250 3251 @staticmethod 3252 def create_iterable_decoder( 3253 model: IterableDecoderModel, config: Config, **kwargs: Any 3254 ) -> IterableDecoder: 3255 return IterableDecoder(parameters={}) 3256 3257 @staticmethod 3258 def create_xml_decoder(model: XmlDecoderModel, config: Config, **kwargs: Any) -> XmlDecoder: 3259 return XmlDecoder(parameters={}) 3260 3261 def create_zipfile_decoder( 3262 self, model: ZipfileDecoderModel, config: Config, **kwargs: Any 3263 ) -> ZipfileDecoder: 3264 return ZipfileDecoder(parser=ModelToComponentFactory._get_parser(model.decoder, config)) 3265 3266 @staticmethod 3267 def _get_parser(model: BaseModel, config: Config) -> Parser: 3268 if isinstance(model, JsonDecoderModel): 3269 # Note that the logic is a bit different from the JsonDecoder as there is some legacy that is maintained to return {} on error cases 3270 return JsonParser() 3271 elif isinstance(model, JsonItemsDecoderModel): 3272 return JsonItemsParser( 3273 items_path=model.items_path, 3274 encoding=model.encoding, 3275 ) 3276 elif isinstance(model, JsonlDecoderModel): 3277 return JsonLineParser() 3278 elif isinstance(model, CsvDecoderModel): 3279 return CsvParser( 3280 encoding=model.encoding, 3281 delimiter=model.delimiter, 3282 set_values_to_none=model.set_values_to_none, 3283 ) 3284 elif isinstance(model, GzipDecoderModel): 3285 return GzipParser( 3286 inner_parser=ModelToComponentFactory._get_parser(model.decoder, config) 3287 ) 3288 elif isinstance( 3289 model, (CustomDecoderModel, IterableDecoderModel, XmlDecoderModel, ZipfileDecoderModel) 3290 ): 3291 raise ValueError(f"Decoder type {model} does not have parser associated to it") 3292 3293 raise ValueError(f"Unknown decoder type {model}") 3294 3295 @staticmethod 3296 def create_json_file_schema_loader( 3297 model: JsonFileSchemaLoaderModel, config: Config, **kwargs: Any 3298 ) -> JsonFileSchemaLoader: 3299 return JsonFileSchemaLoader( 3300 file_path=model.file_path or "", config=config, parameters=model.parameters or {} 3301 ) 3302 3303 def create_jwt_authenticator( 3304 self, model: JwtAuthenticatorModel, config: Config, **kwargs: Any 3305 ) -> JwtAuthenticator: 3306 jwt_headers = model.jwt_headers or JwtHeadersModel(kid=None, typ="JWT", cty=None) 3307 jwt_payload = model.jwt_payload or JwtPayloadModel(iss=None, sub=None, aud=None) 3308 request_option = ( 3309 self._create_component_from_model(model.request_option, config) 3310 if model.request_option 3311 else None 3312 ) 3313 return JwtAuthenticator( 3314 config=config, 3315 parameters=model.parameters or {}, 3316 algorithm=JwtAlgorithm(model.algorithm.value), 3317 secret_key=model.secret_key, 3318 base64_encode_secret_key=model.base64_encode_secret_key, 3319 token_duration=model.token_duration, 3320 header_prefix=model.header_prefix, 3321 kid=jwt_headers.kid, 3322 typ=jwt_headers.typ, 3323 cty=jwt_headers.cty, 3324 iss=jwt_payload.iss, 3325 sub=jwt_payload.sub, 3326 aud=jwt_payload.aud, 3327 additional_jwt_headers=model.additional_jwt_headers, 3328 additional_jwt_payload=model.additional_jwt_payload, 3329 passphrase=model.passphrase, 3330 request_option=request_option, 3331 ) 3332 3333 def create_list_partition_router( 3334 self, model: ListPartitionRouterModel, config: Config, **kwargs: Any 3335 ) -> ListPartitionRouter: 3336 request_option = ( 3337 self._create_component_from_model(model.request_option, config) 3338 if model.request_option 3339 else None 3340 ) 3341 return ListPartitionRouter( 3342 cursor_field=model.cursor_field, 3343 request_option=request_option, 3344 values=model.values, 3345 config=config, 3346 parameters=model.parameters or {}, 3347 ) 3348 3349 @staticmethod 3350 def create_min_max_datetime( 3351 model: MinMaxDatetimeModel, config: Config, **kwargs: Any 3352 ) -> MinMaxDatetime: 3353 return MinMaxDatetime( 3354 datetime=model.datetime, 3355 datetime_format=model.datetime_format or "", 3356 max_datetime=model.max_datetime or "", 3357 min_datetime=model.min_datetime or "", 3358 parameters=model.parameters or {}, 3359 ) 3360 3361 @staticmethod 3362 def create_no_auth(model: NoAuthModel, config: Config, **kwargs: Any) -> NoAuth: 3363 return NoAuth(parameters=model.parameters or {}) 3364 3365 @staticmethod 3366 def create_no_pagination( 3367 model: NoPaginationModel, config: Config, **kwargs: Any 3368 ) -> NoPagination: 3369 return NoPagination(parameters={}) 3370 3371 def create_oauth_authenticator( 3372 self, model: OAuthAuthenticatorModel, config: Config, **kwargs: Any 3373 ) -> DeclarativeOauth2Authenticator: 3374 profile_assertion = ( 3375 self._create_component_from_model(model.profile_assertion, config=config) 3376 if model.profile_assertion 3377 else None 3378 ) 3379 3380 refresh_token_error_status_codes, refresh_token_error_key, refresh_token_error_values = ( 3381 self._get_refresh_token_error_information(model) 3382 ) 3383 if model.refresh_token_updater: 3384 # ignore type error because fixing it would have a lot of dependencies, revisit later 3385 return DeclarativeSingleUseRefreshTokenOauth2Authenticator( # type: ignore 3386 config, 3387 InterpolatedString.create( 3388 model.token_refresh_endpoint, # type: ignore 3389 parameters=model.parameters or {}, 3390 ).eval(config), 3391 access_token_name=InterpolatedString.create( 3392 model.access_token_name or "access_token", parameters=model.parameters or {} 3393 ).eval(config), 3394 refresh_token_name=model.refresh_token_updater.refresh_token_name, 3395 expires_in_name=InterpolatedString.create( 3396 model.expires_in_name or "expires_in", parameters=model.parameters or {} 3397 ).eval(config), 3398 client_id_name=InterpolatedString.create( 3399 model.client_id_name or "client_id", parameters=model.parameters or {} 3400 ).eval(config), 3401 client_id=InterpolatedString.create( 3402 model.client_id, parameters=model.parameters or {} 3403 ).eval(config) 3404 if model.client_id 3405 else model.client_id, 3406 client_secret_name=InterpolatedString.create( 3407 model.client_secret_name or "client_secret", parameters=model.parameters or {} 3408 ).eval(config), 3409 client_secret=InterpolatedString.create( 3410 model.client_secret, parameters=model.parameters or {} 3411 ).eval(config) 3412 if model.client_secret 3413 else model.client_secret, 3414 access_token_config_path=model.refresh_token_updater.access_token_config_path, 3415 refresh_token_config_path=model.refresh_token_updater.refresh_token_config_path, 3416 token_expiry_date_config_path=model.refresh_token_updater.token_expiry_date_config_path, 3417 grant_type_name=InterpolatedString.create( 3418 model.grant_type_name or "grant_type", parameters=model.parameters or {} 3419 ).eval(config), 3420 grant_type=InterpolatedString.create( 3421 model.grant_type or "refresh_token", parameters=model.parameters or {} 3422 ).eval(config), 3423 refresh_request_body=InterpolatedMapping( 3424 model.refresh_request_body or {}, parameters=model.parameters or {} 3425 ).eval(config), 3426 refresh_request_headers=InterpolatedMapping( 3427 model.refresh_request_headers or {}, parameters=model.parameters or {} 3428 ).eval(config), 3429 send_refresh_request_as_query_params=bool( 3430 model.send_refresh_request_as_query_params 3431 ), 3432 scopes=model.scopes, 3433 token_expiry_date_format=model.token_expiry_date_format, 3434 token_expiry_is_time_of_expiration=bool(model.token_expiry_date_format), 3435 message_repository=self._message_repository, 3436 emit_control_message_to_message_repository=self._emit_connector_builder_messages, 3437 refresh_token_error_status_codes=refresh_token_error_status_codes, 3438 refresh_token_error_key=refresh_token_error_key, 3439 refresh_token_error_values=refresh_token_error_values, 3440 refresh_token_error_messages=self._get_refresh_token_error_messages(model), 3441 ) 3442 # ignore type error because fixing it would have a lot of dependencies, revisit later 3443 return DeclarativeOauth2Authenticator( # type: ignore 3444 access_token_name=model.access_token_name or "access_token", 3445 access_token_value=model.access_token_value, 3446 client_id_name=model.client_id_name or "client_id", 3447 client_id=model.client_id, 3448 client_secret_name=model.client_secret_name or "client_secret", 3449 client_secret=model.client_secret, 3450 expires_in_name=model.expires_in_name or "expires_in", 3451 grant_type_name=model.grant_type_name or "grant_type", 3452 grant_type=model.grant_type or "refresh_token", 3453 refresh_request_body=model.refresh_request_body, 3454 refresh_request_headers=model.refresh_request_headers, 3455 send_refresh_request_as_query_params=bool(model.send_refresh_request_as_query_params), 3456 refresh_token_name=model.refresh_token_name or "refresh_token", 3457 refresh_token=model.refresh_token, 3458 scopes=model.scopes, 3459 token_expiry_date=model.token_expiry_date, 3460 token_expiry_date_format=model.token_expiry_date_format, 3461 token_expiry_is_time_of_expiration=bool(model.token_expiry_date_format), 3462 token_refresh_endpoint=model.token_refresh_endpoint, 3463 config=config, 3464 parameters=model.parameters or {}, 3465 message_repository=self._message_repository, 3466 profile_assertion=profile_assertion, 3467 use_profile_assertion=model.use_profile_assertion, 3468 refresh_token_error_status_codes=refresh_token_error_status_codes, 3469 refresh_token_error_key=refresh_token_error_key, 3470 refresh_token_error_values=refresh_token_error_values, 3471 refresh_token_error_messages=self._get_refresh_token_error_messages(model), 3472 ) 3473 3474 @staticmethod 3475 def _get_refresh_token_error_information( 3476 model: OAuthAuthenticatorModel, 3477 ) -> Tuple[Tuple[int, ...], str, Tuple[str, ...]]: 3478 """ 3479 In a previous version of the CDK, the auth error as config_error was only done if a refresh token updater was 3480 defined. As a transition, we added those fields on the OAuthAuthenticatorModel. This method ensures that the 3481 information is defined only once and return the right fields. 3482 """ 3483 refresh_token_updater = model.refresh_token_updater 3484 is_defined_on_refresh_token_updated = refresh_token_updater and ( 3485 refresh_token_updater.refresh_token_error_status_codes 3486 or refresh_token_updater.refresh_token_error_key 3487 or refresh_token_updater.refresh_token_error_values 3488 ) 3489 is_defined_on_oauth_authenticator = ( 3490 model.refresh_token_error_status_codes 3491 or model.refresh_token_error_key 3492 or model.refresh_token_error_values 3493 ) 3494 if is_defined_on_refresh_token_updated and is_defined_on_oauth_authenticator: 3495 raise ValueError( 3496 "refresh_token_error should either be defined on the OAuthAuthenticatorModel or the RefreshTokenUpdaterModel, not both" 3497 ) 3498 3499 if is_defined_on_refresh_token_updated: 3500 not_optional_refresh_token_updater: RefreshTokenUpdaterModel = refresh_token_updater # type: ignore # we know from the condition that this is not None 3501 return ( 3502 tuple(not_optional_refresh_token_updater.refresh_token_error_status_codes) 3503 if not_optional_refresh_token_updater.refresh_token_error_status_codes 3504 else (), 3505 not_optional_refresh_token_updater.refresh_token_error_key or "", 3506 tuple(not_optional_refresh_token_updater.refresh_token_error_values) 3507 if not_optional_refresh_token_updater.refresh_token_error_values 3508 else (), 3509 ) 3510 elif is_defined_on_oauth_authenticator: 3511 return ( 3512 tuple(model.refresh_token_error_status_codes) 3513 if model.refresh_token_error_status_codes 3514 else (), 3515 model.refresh_token_error_key or "", 3516 tuple(model.refresh_token_error_values) if model.refresh_token_error_values else (), 3517 ) 3518 3519 # returning default values we think cover most cases 3520 return (400,), "error", ("invalid_grant", "invalid_permissions") 3521 3522 @staticmethod 3523 def _get_refresh_token_error_messages(model: OAuthAuthenticatorModel) -> Dict[str, str]: 3524 """Resolve messages separately so they never disable the default gate.""" 3525 return dict(model.refresh_token_error_messages or {}) 3526 3527 @staticmethod 3528 def _reject_union_combined_extractor_for_offset_increment( 3529 extractor_model: Optional[BaseModel], 3530 ) -> None: 3531 """Refuse an `OffsetIncrement` paginator driven by a `union` `CombinedExtractor`. 3532 3533 `OffsetIncrement.next_page_token` advances the offset by the number of records its 3534 extractor returns for the page. Under `union` that number is the sum over all 3535 sub-extractors, so the offset overshoots the API page size and every page after the first 3536 starts past the records that were never read: with two sub-extractors returning two records 3537 each and `page_size: 2`, the requested offsets are 0, 4, 8 instead of 0, 2, 4 and two 3538 thirds of the records are silently dropped. 3539 3540 `first_match` returns the winning sub-extractor's count, which does not inflate the count, 3541 and is left alone. A `union` nested anywhere in the tree inflates the count of the node 3542 above it, so the whole tree is walked. 3543 """ 3544 if not isinstance(extractor_model, CombinedExtractorModel): 3545 return 3546 if not ModelToComponentFactory._combined_extractor_tree_contains_union(extractor_model): 3547 return 3548 raise AirbyteTracedException( 3549 message=( 3550 'CombinedExtractor mode "union" is not supported with an OffsetIncrement paginator.' 3551 ), 3552 internal_message=( 3553 "OffsetIncrement counts the records of its extractor to advance the offset. A " 3554 "`union` CombinedExtractor returns the sum of its sub-extractors' records, which " 3555 "overshoots the page the API returned, so records would be skipped. Use the " 3556 "`first_match` mode, a CursorPagination or PageIncrement paginator, or a single " 3557 "extractor." 3558 ), 3559 failure_type=FailureType.config_error, 3560 ) 3561 3562 @staticmethod 3563 def _combined_extractor_tree_contains_union(model: CombinedExtractorModel) -> bool: 3564 mode = CombineMode(model.mode.value) if model.mode else CombineMode.union 3565 if mode == CombineMode.union: 3566 return True 3567 return any( 3568 isinstance(sub_extractor, CombinedExtractorModel) 3569 and ModelToComponentFactory._combined_extractor_tree_contains_union(sub_extractor) 3570 for sub_extractor in model.extractors 3571 ) 3572 3573 @staticmethod 3574 def _count_dropped_empty_records(extractor: RecordExtractor) -> None: 3575 """Make the record selector count the empty records a `CombinedExtractor` drops. 3576 3577 `OffsetIncrement` and `PageIncrement` stop when that count falls below the page size, so a 3578 full page holding a null would otherwise look short and silently end pagination. 3579 """ 3580 if not isinstance(extractor, CombinedExtractor): 3581 return 3582 extractor.count_dropped_empty_records = True 3583 for sub_extractor in extractor.extractors: 3584 ModelToComponentFactory._count_dropped_empty_records(sub_extractor) 3585 3586 def create_offset_increment( 3587 self, 3588 model: OffsetIncrementModel, 3589 config: Config, 3590 decoder: Optional[Decoder] = None, 3591 extractor_model: Optional[ 3592 Union[CustomRecordExtractorModel, DpathExtractorModel, CombinedExtractorModel] 3593 ] = None, 3594 **kwargs: Any, 3595 ) -> OffsetIncrement: 3596 # None when nested under a custom component, which cannot pass a decoder 3597 if decoder is None: 3598 decoder = JsonDecoder(parameters={}) 3599 if isinstance(decoder, PaginationDecoderDecorator): 3600 inner_decoder = decoder.decoder 3601 else: 3602 inner_decoder = decoder 3603 decoder = PaginationDecoderDecorator(decoder=decoder) 3604 3605 if self._is_supported_decoder_for_pagination(inner_decoder): 3606 decoder_to_use = decoder 3607 else: 3608 raise ValueError( 3609 self._UNSUPPORTED_DECODER_ERROR.format(decoder_type=type(inner_decoder)) 3610 ) 3611 3612 self._reject_union_combined_extractor_for_offset_increment(extractor_model) 3613 3614 # Pydantic v1 Union type coercion can convert int to string depending on Union order. 3615 # If page_size is a string that represents an integer (not an interpolation), convert it back. 3616 page_size = model.page_size 3617 if isinstance(page_size, str) and page_size.isdigit(): 3618 page_size = int(page_size) 3619 3620 return OffsetIncrement( 3621 page_size=page_size, 3622 config=config, 3623 decoder=decoder_to_use, 3624 inject_on_first_request=model.inject_on_first_request or False, 3625 parameters=model.parameters or {}, 3626 ) 3627 3628 @staticmethod 3629 def create_page_increment( 3630 model: PageIncrementModel, config: Config, **kwargs: Any 3631 ) -> PageIncrement: 3632 # Pydantic v1 Union type coercion can convert int to string depending on Union order. 3633 # If page_size is a string that represents an integer (not an interpolation), convert it back. 3634 page_size = model.page_size 3635 if isinstance(page_size, str) and page_size.isdigit(): 3636 page_size = int(page_size) 3637 3638 return PageIncrement( 3639 page_size=page_size, 3640 config=config, 3641 start_from_page=model.start_from_page or 0, 3642 inject_on_first_request=model.inject_on_first_request or False, 3643 parameters=model.parameters or {}, 3644 ) 3645 3646 def create_parent_stream_config( 3647 self, model: ParentStreamConfigModel, config: Config, *, stream_name: str, **kwargs: Any 3648 ) -> ParentStreamConfig: 3649 declarative_stream = self._create_component_from_model( 3650 model.stream, 3651 config=config, 3652 is_parent=True, 3653 **kwargs, 3654 ) 3655 request_option = ( 3656 self._create_component_from_model(model.request_option, config=config) 3657 if model.request_option 3658 else None 3659 ) 3660 3661 if model.lazy_read_pointer and any("*" in pointer for pointer in model.lazy_read_pointer): 3662 raise ValueError( 3663 "The '*' wildcard in 'lazy_read_pointer' is not supported — only direct paths are allowed." 3664 ) 3665 3666 model_lazy_read_pointer: List[Union[InterpolatedString, str]] = ( 3667 [x for x in model.lazy_read_pointer] if model.lazy_read_pointer else [] 3668 ) 3669 3670 return ParentStreamConfig( 3671 parent_key=model.parent_key, 3672 request_option=request_option, 3673 stream=declarative_stream, 3674 partition_field=model.partition_field, 3675 config=config, 3676 incremental_dependency=model.incremental_dependency or False, 3677 parameters=model.parameters or {}, 3678 extra_fields=model.extra_fields, 3679 lazy_read_pointer=model_lazy_read_pointer, 3680 ) 3681 3682 def create_properties_from_endpoint( 3683 self, model: PropertiesFromEndpointModel, config: Config, **kwargs: Any 3684 ) -> PropertiesFromEndpoint: 3685 retriever = self._create_component_from_model( 3686 model=model.retriever, 3687 config=config, 3688 name="dynamic_properties", 3689 primary_key=None, 3690 stream_slicer=None, 3691 transformations=[], 3692 use_cache=True, # Enable caching on the HttpRequester/HttpClient because the properties endpoint will be called for every slice being processed, and it is highly unlikely for the response to different 3693 ) 3694 return PropertiesFromEndpoint( 3695 property_field_path=model.property_field_path, 3696 retriever=retriever, 3697 config=config, 3698 parameters=model.parameters or {}, 3699 ) 3700 3701 def create_property_chunking( 3702 self, model: PropertyChunkingModel, config: Config, **kwargs: Any 3703 ) -> PropertyChunking: 3704 record_merge_strategy = ( 3705 self._create_component_from_model( 3706 model=model.record_merge_strategy, config=config, **kwargs 3707 ) 3708 if model.record_merge_strategy 3709 else None 3710 ) 3711 3712 property_limit_type: PropertyLimitType 3713 match model.property_limit_type: 3714 case PropertyLimitTypeModel.property_count: 3715 property_limit_type = PropertyLimitType.property_count 3716 case PropertyLimitTypeModel.characters: 3717 property_limit_type = PropertyLimitType.characters 3718 case _: 3719 raise ValueError(f"Invalid PropertyLimitType {property_limit_type}") 3720 3721 return PropertyChunking( 3722 property_limit_type=property_limit_type, 3723 property_limit=model.property_limit, 3724 record_merge_strategy=record_merge_strategy, 3725 config=config, 3726 parameters=model.parameters or {}, 3727 ) 3728 3729 def create_query_properties( 3730 self, model: QueryPropertiesModel, config: Config, *, stream_name: str, **kwargs: Any 3731 ) -> QueryProperties: 3732 if isinstance(model.property_list, list): 3733 property_list = model.property_list 3734 else: 3735 property_list = self._create_component_from_model( 3736 model=model.property_list, config=config, **kwargs 3737 ) 3738 3739 property_chunking = ( 3740 self._create_component_from_model( 3741 model=model.property_chunking, config=config, **kwargs 3742 ) 3743 if model.property_chunking 3744 else None 3745 ) 3746 3747 property_selector = ( 3748 self._create_component_from_model( 3749 model=model.property_selector, config=config, stream_name=stream_name, **kwargs 3750 ) 3751 if model.property_selector 3752 else None 3753 ) 3754 3755 return QueryProperties( 3756 property_list=property_list, 3757 always_include_properties=model.always_include_properties, 3758 property_chunking=property_chunking, 3759 property_selector=property_selector, 3760 config=config, 3761 parameters=model.parameters or {}, 3762 ) 3763 3764 def create_json_schema_property_selector( 3765 self, 3766 model: JsonSchemaPropertySelectorModel, 3767 config: Config, 3768 *, 3769 stream_name: str, 3770 **kwargs: Any, 3771 ) -> JsonSchemaPropertySelector: 3772 configured_stream = self._stream_name_to_configured_stream.get(stream_name) 3773 3774 transformations = [] 3775 if model.transformations: 3776 for transformation_model in model.transformations: 3777 transformations.append( 3778 self._create_component_from_model(model=transformation_model, config=config) 3779 ) 3780 3781 return JsonSchemaPropertySelector( 3782 configured_stream=configured_stream, 3783 properties_transformations=transformations, 3784 config=config, 3785 parameters=model.parameters or {}, 3786 ) 3787 3788 @staticmethod 3789 def create_record_filter( 3790 model: RecordFilterModel, config: Config, **kwargs: Any 3791 ) -> RecordFilter: 3792 return RecordFilter( 3793 condition=model.condition or "", config=config, parameters=model.parameters or {} 3794 ) 3795 3796 @staticmethod 3797 def create_request_path(model: RequestPathModel, config: Config, **kwargs: Any) -> RequestPath: 3798 return RequestPath(parameters={}) 3799 3800 @staticmethod 3801 def create_request_option( 3802 model: RequestOptionModel, config: Config, **kwargs: Any 3803 ) -> RequestOption: 3804 inject_into = RequestOptionType(model.inject_into.value) 3805 field_path: Optional[List[Union[InterpolatedString, str]]] = ( 3806 [ 3807 InterpolatedString.create(segment, parameters=kwargs.get("parameters", {})) 3808 for segment in model.field_path 3809 ] 3810 if model.field_path 3811 else None 3812 ) 3813 field_name = ( 3814 InterpolatedString.create(model.field_name, parameters=kwargs.get("parameters", {})) 3815 if model.field_name 3816 else None 3817 ) 3818 return RequestOption( 3819 field_name=field_name, 3820 field_path=field_path, 3821 inject_into=inject_into, 3822 parameters=kwargs.get("parameters", {}), 3823 ) 3824 3825 def create_record_selector( 3826 self, 3827 model: RecordSelectorModel, 3828 config: Config, 3829 *, 3830 name: str, 3831 transformations: List[RecordTransformation] | None = None, 3832 decoder: Decoder | None = None, 3833 client_side_incremental_sync_cursor: Optional[Cursor] = None, 3834 is_client_side_incremental_sync: bool = False, 3835 file_uploader: Optional[DefaultFileUploader] = None, 3836 **kwargs: Any, 3837 ) -> RecordSelector: 3838 extractor = self._create_component_from_model( 3839 model=model.extractor, decoder=decoder, config=config 3840 ) 3841 self._count_dropped_empty_records(extractor) 3842 record_filter = ( 3843 self._create_component_from_model(model.record_filter, config=config) 3844 if model.record_filter 3845 else None 3846 ) 3847 3848 # A client-side incremental stream transforms before filtering by default. That default belongs to the flag, 3849 # not to the component that ends up doing the cursor comparison: a data feed does it in the retriever and 3850 # receives no cursor here, but its `record_filter` condition must keep running after the transformations. 3851 default_transform_before_filtering = bool( 3852 client_side_incremental_sync_cursor or is_client_side_incremental_sync 3853 ) 3854 transform_before_filtering = ( 3855 default_transform_before_filtering 3856 if model.transform_before_filtering is None 3857 else model.transform_before_filtering 3858 ) 3859 if client_side_incremental_sync_cursor: 3860 record_filter = ClientSideIncrementalRecordFilterDecorator( 3861 config=config, 3862 parameters=model.parameters, 3863 condition=model.record_filter.condition 3864 if (model.record_filter and hasattr(model.record_filter, "condition")) 3865 else None, 3866 cursor=client_side_incremental_sync_cursor, 3867 ) 3868 3869 if model.schema_normalization is None: 3870 # default to no schema normalization if not set 3871 model.schema_normalization = SchemaNormalizationModel.None_ 3872 3873 schema_normalization = ( 3874 TypeTransformer(SCHEMA_TRANSFORMER_TYPE_MAPPING[model.schema_normalization]) 3875 if isinstance(model.schema_normalization, SchemaNormalizationModel) 3876 else self._create_component_from_model(model.schema_normalization, config=config) # type: ignore[arg-type] # custom normalization model expected here 3877 ) 3878 3879 return RecordSelector( 3880 extractor=extractor, 3881 name=name, 3882 config=config, 3883 record_filter=record_filter, 3884 transformations=transformations or [], 3885 file_uploader=file_uploader, 3886 schema_normalization=schema_normalization, 3887 parameters=model.parameters or {}, 3888 transform_before_filtering=transform_before_filtering, 3889 ) 3890 3891 @staticmethod 3892 def create_remove_fields( 3893 model: RemoveFieldsModel, config: Config, **kwargs: Any 3894 ) -> RemoveFields: 3895 return RemoveFields( 3896 field_pointers=model.field_pointers, condition=model.condition or "", parameters={} 3897 ) 3898 3899 def create_selective_authenticator( 3900 self, 3901 model: SelectiveAuthenticatorModel, 3902 config: Config, 3903 *, 3904 name: Optional[str] = None, 3905 url_base: Optional[str] = None, 3906 **kwargs: Any, 3907 ) -> DeclarativeAuthenticator: 3908 # Keyword-only so that _create_nested_component also fills them under a custom component, 3909 # from the parent's kwargs or $parameters. None is not forwarded, so a missing required one 3910 # keeps its hint. 3911 nested_kwargs = { 3912 key: value 3913 for key, value in {"name": name, "url_base": url_base}.items() 3914 if value is not None 3915 } 3916 authenticators = { 3917 key: self._create_component_from_model( 3918 model=auth, config=config, **nested_kwargs, **kwargs 3919 ) 3920 for key, auth in model.authenticators.items() 3921 } 3922 # SelectiveAuthenticator will return instance of DeclarativeAuthenticator or raise ValueError error 3923 return SelectiveAuthenticator( # type: ignore[abstract] 3924 config=config, 3925 authenticators=authenticators, 3926 authenticator_selection_path=model.authenticator_selection_path, 3927 **kwargs, 3928 ) 3929 3930 @staticmethod 3931 def create_legacy_session_token_authenticator( 3932 model: LegacySessionTokenAuthenticatorModel, config: Config, *, url_base: str, **kwargs: Any 3933 ) -> LegacySessionTokenAuthenticator: 3934 return LegacySessionTokenAuthenticator( 3935 api_url=url_base, 3936 header=model.header, 3937 login_url=model.login_url, 3938 password=model.password or "", 3939 session_token=model.session_token or "", 3940 session_token_response_key=model.session_token_response_key or "", 3941 username=model.username or "", 3942 validate_session_url=model.validate_session_url, 3943 config=config, 3944 parameters=model.parameters or {}, 3945 ) 3946 3947 def create_simple_retriever( 3948 self, 3949 model: SimpleRetrieverModel, 3950 config: Config, 3951 *, 3952 name: str, 3953 primary_key: Optional[Union[str, List[str], List[List[str]]]], 3954 request_options_provider: Optional[RequestOptionsProvider] = None, 3955 cursor: Optional[Cursor] = None, 3956 has_stop_condition_cursor: bool = False, 3957 is_client_side_incremental_sync: bool = False, 3958 transformations: List[RecordTransformation], 3959 file_uploader: Optional[DefaultFileUploader] = None, 3960 incremental_sync: Optional[ 3961 Union[ 3962 IncrementingCountCursorModel, 3963 DatetimeBasedCursorModel, 3964 SyncTokenCursorModel, 3965 ] 3966 ] = None, 3967 use_cache: Optional[bool] = None, 3968 log_formatter: Optional[Callable[[Response], Any]] = None, 3969 partition_router: Optional[PartitionRouter] = None, 3970 **kwargs: Any, 3971 ) -> SimpleRetriever: 3972 def _get_url(req: Requester) -> str: 3973 """ 3974 Closure to get the URL from the requester. This is used to get the URL in the case of a lazy retriever. 3975 This is needed because the URL is not set until the requester is created. 3976 """ 3977 3978 _url: str = ( 3979 model.requester.url 3980 if hasattr(model.requester, "url") and model.requester.url is not None 3981 else req.get_url(stream_state=None, stream_slice=None, next_page_token=None) 3982 ) 3983 _url_base: str = ( 3984 model.requester.url_base 3985 if hasattr(model.requester, "url_base") and model.requester.url_base is not None 3986 else req.get_url_base(stream_state=None, stream_slice=None, next_page_token=None) 3987 ) 3988 3989 return _url or _url_base 3990 3991 if cursor is None: 3992 cursor = FinalStateCursor(name, None, self._message_repository) 3993 3994 # A data feed drops the records the cursor considers already synced in the retriever, which sits downstream of 3995 # the paginator. Letting the record selector drop them as well would be redundant and would hide them from the 3996 # pagination stop condition, so a data feed never delegates that filtering to the record selector, whether 3997 # `is_client_side_incremental` is set or not. The `condition` from `record_filter` is intentionally left out of 3998 # the post-pagination filter and stays in the record selector, which preserves the existing behaviour: the 3999 # selector runs inside the page loop, so the records the condition rejects never reach the paginator's 4000 # accounting. Moving it downstream would start counting them. 4001 post_pagination_filter = ( 4002 ClientSideIncrementalRecordFilterDecorator( 4003 config=config, 4004 parameters=model.parameters or {}, 4005 condition=None, 4006 cursor=cursor, 4007 ) 4008 if has_stop_condition_cursor 4009 else None 4010 ) 4011 client_side_incremental_cursor = ( 4012 cursor if is_client_side_incremental_sync and not post_pagination_filter else None 4013 ) 4014 if post_pagination_filter and is_client_side_incremental_sync: 4015 LOGGER.warning( 4016 f"Stream {name}: `is_client_side_incremental` adds no record filtering when `is_data_feed` is set, " 4017 "as a data feed already filters on the cursor value. It still makes the record selector apply the " 4018 "transformations before the `record_filter` condition." 4019 ) 4020 4021 decoder = ( 4022 cursor.decoder 4023 if isinstance(cursor, SyncTokenCursor) 4024 else ( 4025 self._create_component_from_model(model=model.decoder, config=config) 4026 if model.decoder 4027 else JsonDecoder(parameters={}) 4028 ) 4029 ) 4030 record_selector = self._create_component_from_model( 4031 model=model.record_selector, 4032 name=name, 4033 config=config, 4034 decoder=decoder, 4035 transformations=transformations, 4036 client_side_incremental_sync_cursor=client_side_incremental_cursor, 4037 is_client_side_incremental_sync=is_client_side_incremental_sync, 4038 file_uploader=file_uploader, 4039 ) 4040 4041 query_properties: Optional[QueryProperties] = None 4042 query_properties_key: Optional[str] = None 4043 self._ensure_query_properties_to_model(model.requester) 4044 if self._has_query_properties_in_request_parameters(model.requester): 4045 # It is better to be explicit about an error if PropertiesFromEndpoint is defined in multiple 4046 # places instead of default to request_parameters which isn't clearly documented 4047 if ( 4048 hasattr(model.requester, "fetch_properties_from_endpoint") 4049 and model.requester.fetch_properties_from_endpoint 4050 ): 4051 raise ValueError( 4052 f"PropertiesFromEndpoint should only be specified once per stream, but found in {model.requester.type}.fetch_properties_from_endpoint and {model.requester.type}.request_parameters" 4053 ) 4054 4055 query_properties_definitions = [] 4056 for key, request_parameter in model.requester.request_parameters.items(): # type: ignore # request_parameters is already validated to be a Mapping using _has_query_properties_in_request_parameters() 4057 if isinstance(request_parameter, QueryPropertiesModel): 4058 query_properties_key = key 4059 query_properties_definitions.append(request_parameter) 4060 4061 if len(query_properties_definitions) > 1: 4062 raise ValueError( 4063 f"request_parameters only supports defining one QueryProperties field, but found {len(query_properties_definitions)} usages" 4064 ) 4065 4066 if len(query_properties_definitions) == 1: 4067 query_properties = self._create_component_from_model( 4068 model=query_properties_definitions[0], stream_name=name, config=config 4069 ) 4070 4071 # Removes QueryProperties components from the interpolated mappings because it has been designed 4072 # to be used by the SimpleRetriever and will be resolved from the provider from the slice directly 4073 # instead of through jinja interpolation 4074 if hasattr(model.requester, "request_parameters") and isinstance( 4075 model.requester.request_parameters, Mapping 4076 ): 4077 model.requester.request_parameters = self._remove_query_properties( 4078 model.requester.request_parameters 4079 ) 4080 elif ( 4081 hasattr(model.requester, "fetch_properties_from_endpoint") 4082 and model.requester.fetch_properties_from_endpoint 4083 ): 4084 # todo: Deprecate this condition once dependent connectors migrate to query_properties 4085 query_properties_definition = QueryPropertiesModel( 4086 type="QueryProperties", 4087 property_list=model.requester.fetch_properties_from_endpoint, 4088 always_include_properties=None, 4089 property_chunking=None, 4090 ) # type: ignore # $parameters has a default value 4091 4092 query_properties = self.create_query_properties( 4093 model=query_properties_definition, 4094 stream_name=name, 4095 config=config, 4096 ) 4097 elif hasattr(model.requester, "query_properties") and model.requester.query_properties: 4098 query_properties = self.create_query_properties( 4099 model=model.requester.query_properties, 4100 stream_name=name, 4101 config=config, 4102 ) 4103 4104 requester = self._create_component_from_model( 4105 model=model.requester, 4106 decoder=decoder, 4107 name=name, 4108 query_properties_key=query_properties_key, 4109 use_cache=use_cache, 4110 config=config, 4111 ) 4112 4113 if not request_options_provider: 4114 request_options_provider = DefaultRequestOptionsProvider(parameters={}) 4115 if isinstance(request_options_provider, DefaultRequestOptionsProvider) and isinstance( 4116 partition_router, PartitionRouter 4117 ): 4118 request_options_provider = partition_router 4119 4120 paginator = ( 4121 self._create_component_from_model( 4122 model=model.paginator, 4123 config=config, 4124 url_base=_get_url(requester), 4125 extractor_model=model.record_selector.extractor, 4126 decoder=decoder, 4127 cursor_used_for_stop_condition=cursor if has_stop_condition_cursor else None, 4128 ) 4129 if model.paginator 4130 else NoPagination(parameters={}) 4131 ) 4132 4133 ignore_stream_slicer_parameters_on_paginated_requests = ( 4134 model.ignore_stream_slicer_parameters_on_paginated_requests or False 4135 ) 4136 4137 reads_parent_stream_lazily = bool( 4138 model.partition_router 4139 and isinstance(model.partition_router, SubstreamPartitionRouterModel) 4140 and any( 4141 parent_stream_config.lazy_read_pointer 4142 for parent_stream_config in model.partition_router.parent_stream_configs 4143 ) 4144 ) 4145 if reads_parent_stream_lazily and ( 4146 model.page_size_reduction 4147 or self._uses_reduce_page_size_action(getattr(model.requester, "error_handler", None)) 4148 ): 4149 # Checked outside of the LazySimpleRetriever branch below, which only applies on the first 4150 # sync of a stream: gating it on the absence of state would accept the same manifest from the 4151 # second sync onwards. LazySimpleRetriever paginates the parent's embedded pages, so there is 4152 # no page of its own to re-issue with a smaller page size. 4153 raise ValueError( 4154 f"`page_size_reduction` and the REDUCE_PAGE_SIZE response action are not supported when " 4155 f"reading a parent stream lazily. Remove either the page size reduction or the parent " 4156 f"stream's `lazy_read_pointer` for stream {name}." 4157 ) 4158 4159 if reads_parent_stream_lazily and ( 4160 model.request_window_splitting 4161 or self._uses_split_request_window_action( 4162 getattr(model.requester, "error_handler", None) 4163 ) 4164 ): 4165 # Same reasoning as the page_size_reduction check above: LazySimpleRetriever reads records 4166 # embedded in the parent's own pages rather than requesting a window of its own, so there is 4167 # nothing here for a split child window to be read from. 4168 raise ValueError( 4169 f"`request_window_splitting` and the SPLIT_REQUEST_WINDOW response action are not " 4170 f"supported when reading a parent stream lazily. Remove either the request window " 4171 f"splitting or the parent stream's `lazy_read_pointer` for stream {name}." 4172 ) 4173 4174 if reads_parent_stream_lazily and not bool( 4175 self._connector_state_manager.get_stream_state(name, None) 4176 ): 4177 if incremental_sync: 4178 if incremental_sync.type != "DatetimeBasedCursor": 4179 raise ValueError( 4180 f"LazySimpleRetriever only supports DatetimeBasedCursor. Found: {incremental_sync.type}." 4181 ) 4182 4183 elif incremental_sync.step or incremental_sync.cursor_granularity: 4184 raise ValueError( 4185 f"Found more that one slice per parent. LazySimpleRetriever only supports single slice read for stream - {name}." 4186 ) 4187 4188 if model.decoder and model.decoder.type != "JsonDecoder": 4189 raise ValueError( 4190 f"LazySimpleRetriever only supports JsonDecoder. Found: {model.decoder.type}." 4191 ) 4192 4193 return LazySimpleRetriever( 4194 name=name, 4195 paginator=paginator, 4196 primary_key=primary_key, 4197 requester=requester, 4198 record_selector=record_selector, 4199 stream_slicer=_NO_STREAM_SLICING, 4200 request_option_provider=request_options_provider, 4201 config=config, 4202 ignore_stream_slicer_parameters_on_paginated_requests=ignore_stream_slicer_parameters_on_paginated_requests, 4203 post_pagination_filter=post_pagination_filter, 4204 parameters=model.parameters or {}, 4205 ) 4206 4207 if ( 4208 model.record_selector.record_filter 4209 and model.pagination_reset 4210 and model.pagination_reset.limits 4211 ): 4212 raise ValueError("PaginationResetLimits are not supported while having record filter.") 4213 4214 request_window_splitting = self._create_request_window_splitting( 4215 model, name, cursor, incremental_sync, query_properties, file_uploader 4216 ) 4217 return SimpleRetriever( 4218 name=name, 4219 paginator=paginator, 4220 primary_key=primary_key, 4221 requester=requester, 4222 record_selector=record_selector, 4223 stream_slicer=_NO_STREAM_SLICING, 4224 request_option_provider=request_options_provider, 4225 config=config, 4226 ignore_stream_slicer_parameters_on_paginated_requests=ignore_stream_slicer_parameters_on_paginated_requests, 4227 additional_query_properties=query_properties, 4228 log_formatter=self._get_log_formatter(log_formatter, name), 4229 pagination_tracker_factory=self._create_pagination_tracker_factory( 4230 model.pagination_reset, cursor 4231 ), 4232 page_size_reduction=self._create_page_size_reduction( 4233 model, name, query_properties, file_uploader 4234 ), 4235 request_window_splitting=request_window_splitting, 4236 request_window_splitter=( 4237 cursor.split_request_window 4238 if request_window_splitting and hasattr(cursor, "split_request_window") 4239 else None 4240 ), 4241 post_pagination_filter=post_pagination_filter, 4242 page_observer=( 4243 cursor.on_page_completed if isinstance(cursor, SyncTokenCursor) else None 4244 ), 4245 parameters=model.parameters or {}, 4246 ) 4247 4248 def _create_page_size_reduction( 4249 self, 4250 model: SimpleRetrieverModel, 4251 name: str, 4252 query_properties: Optional[QueryProperties], 4253 file_uploader: Optional[DefaultFileUploader] = None, 4254 ) -> Optional[PageSizeReduction]: 4255 # A CustomRequester does not necessarily define an error handler. A CustomRequester that does define 4256 # one keeps it as a raw dict rather than a typed model, so this returns False for it as well. 4257 error_handler = getattr(model.requester, "error_handler", None) 4258 uses_action = self._uses_reduce_page_size_action(error_handler) 4259 if uses_action and not model.page_size_reduction: 4260 raise ValueError( 4261 f"Stream {name} has a response filter with the REDUCE_PAGE_SIZE action but the retriever does not " 4262 f"define `page_size_reduction`. Add a `page_size_reduction` block to the retriever." 4263 ) 4264 4265 if not model.page_size_reduction: 4266 return None 4267 4268 if not uses_action: 4269 # Not raised: a CustomErrorHandler can resolve to REDUCE_PAGE_SIZE without us being able to see 4270 # it, so the only safe reaction to a block we cannot tie to an action is a warning. Without it a 4271 # misspelled action leaves the feature silently dead on a stream that only exists because it 4272 # would otherwise fail. 4273 LOGGER.warning( 4274 f"Stream {name} defines `page_size_reduction` but no response filter with the " 4275 f"REDUCE_PAGE_SIZE action was found on its requester. The page size will never be reduced " 4276 f"unless a custom error handler resolves to that action." 4277 ) 4278 4279 self._validate_page_size_reduction_is_supported( 4280 model, name, query_properties, file_uploader 4281 ) 4282 4283 reset_policy = model.page_size_reduction.reset_policy 4284 return PageSizeReduction( 4285 reduction_factor=model.page_size_reduction.reduction_factor, # type: ignore[arg-type] # the schema defines a default 4286 minimum_page_size=model.page_size_reduction.minimum_page_size, # type: ignore[arg-type] # the schema defines a default 4287 max_attempts=model.page_size_reduction.max_attempts, # type: ignore[arg-type] # the schema defines a default 4288 backoff_seconds=model.page_size_reduction.backoff_seconds, # type: ignore[arg-type] # the schema defines a default 4289 retries_at_minimum_page_size=model.page_size_reduction.retries_at_minimum_page_size, # type: ignore[arg-type] # the schema defines a default 4290 failure_message=model.page_size_reduction.failure_message, 4291 reset_policy=PageSizeResetPolicy(reset_policy.value) 4292 if reset_policy is not None 4293 else PageSizeResetPolicy.NEVER, 4294 ) 4295 4296 def _validate_page_size_reduction_is_supported( 4297 self, 4298 model: SimpleRetrieverModel, 4299 name: str, 4300 query_properties: Optional[QueryProperties], 4301 file_uploader: Optional[DefaultFileUploader] = None, 4302 ) -> None: 4303 """ 4304 Page size reduction re-issues the same page with a smaller page size. That is only correct when the next 4305 page does not depend on the page size, and it only has an effect when the paginator injects the page size 4306 in the request. A custom pagination strategy is accepted when it can receive the reduced page size, which 4307 is checked by inspecting its signature rather than by recognizing its type. 4308 """ 4309 if query_properties: 4310 raise ValueError( 4311 f"`page_size_reduction` cannot be used together with query properties on stream {name}. Records " 4312 f"from the earlier property chunks have already been emitted when a chunk asks for a smaller page, " 4313 f"so retrying the page would emit them twice." 4314 ) 4315 4316 if file_uploader: 4317 raise ValueError( 4318 f"`page_size_reduction` cannot be used together with a `file_uploader` on stream {name}. The " 4319 f"file uploader sends one request per record from inside the page's record generator, so a " 4320 f"reduction asked for halfway through a page would re-emit the records already yielded by it." 4321 ) 4322 4323 if not isinstance(model.paginator, DefaultPaginatorModel): 4324 raise ValueError( 4325 f"`page_size_reduction` requires a DefaultPaginator on stream {name} so that the connector can " 4326 f"send a smaller page size." 4327 ) 4328 4329 if not model.paginator.page_size_option: 4330 raise ValueError( 4331 f"`page_size_reduction` requires `page_size_option` on the paginator of stream {name}: without it " 4332 f"the connector cannot tell the API to send a smaller page." 4333 ) 4334 4335 if isinstance(model.paginator.page_token_option, RequestPathModel): 4336 # A RequestPath page token is a full URL built by the API, and it already carries the page size the 4337 # API echoed back. The reduced page size is injected as a request option on top of that URL, so the 4338 # request goes out with the page size twice - the original one from the URL and the reduced one - 4339 # and which of the two the API honors is up to the API. Every page after the first would then keep 4340 # asking for the page size that just failed. 4341 raise ValueError( 4342 f"`page_size_reduction` does not support a `page_token_option` of type RequestPath on stream " 4343 f"{name}. The next page is then requested through a URL returned by the API, which already " 4344 f"carries the page size, so the reduced page size would be sent alongside the original one. Use " 4345 f"a CursorPagination strategy with a `page_token_option` of type RequestOption instead." 4346 ) 4347 4348 strategy = model.paginator.pagination_strategy 4349 if isinstance(strategy, PageIncrementModel): 4350 raise ValueError( 4351 f"`page_size_reduction` does not support the PageIncrement pagination strategy used by stream " 4352 f"{name}. Pages are addressed as page number * page size, so a smaller page size shifts every " 4353 f"following page boundary and would skip records. Use OffsetIncrement or CursorPagination." 4354 ) 4355 if isinstance(strategy, CustomPaginationStrategyModel): 4356 # A custom strategy is written by the same person enabling the reduction, so the 4357 # question is not whether we recognize it but whether it can be told the reduced 4358 # page size. Checking the signature keeps a strategy that would raise TypeError 4359 # mid-sync from being accepted at config time. 4360 custom_class = self._get_class_from_fully_qualified_class_name(strategy.class_name) 4361 if isinstance(custom_class, type) and issubclass(custom_class, PageIncrement): 4362 # `PageIncrement.next_page_token` declares `page_size_override` only to reject it, so a 4363 # subclass that does not override the method would pass the signature check below and only 4364 # fail once the first reduction is requested, mid-sync. 4365 raise ValueError( 4366 f"`page_size_reduction` does not support the PageIncrement pagination strategy that the " 4367 f"custom pagination strategy {strategy.class_name} used by stream {name} inherits from. " 4368 f"Pages are addressed as page number * page size, so a smaller page size shifts every " 4369 f"following page boundary and would skip records. Use OffsetIncrement or CursorPagination." 4370 ) 4371 try: 4372 parameters = inspect.signature(custom_class.next_page_token).parameters 4373 except (AttributeError, TypeError, ValueError) as exception: 4374 raise ValueError( 4375 f"`page_size_reduction` could not check the signature of `next_page_token` on the custom " 4376 f"pagination strategy {strategy.class_name} used by stream {name}: {exception}. Make sure " 4377 f"`class_name` points at a PaginationStrategy subclass whose `next_page_token` accepts a " 4378 f"`page_size_override` keyword argument." 4379 ) 4380 accepts_override = "page_size_override" in parameters or any( 4381 parameter.kind == inspect.Parameter.VAR_KEYWORD for parameter in parameters.values() 4382 ) 4383 if not accepts_override: 4384 raise ValueError( 4385 f"`page_size_reduction` requires the custom pagination strategy " 4386 f"{strategy.class_name} used by stream {name} to accept a `page_size_override` keyword " 4387 f"argument in `next_page_token`, so that it can honor the reduced page size. Add " 4388 f"`page_size_override: Optional[int] = None` to its signature; a strategy that does not " 4389 f"use its page size as a stop condition can ignore the value." 4390 ) 4391 elif not isinstance(strategy, (CursorPaginationModel, OffsetIncrementModel)): 4392 raise ValueError( 4393 f"`page_size_reduction` only supports the CursorPagination, OffsetIncrement and " 4394 f"CustomPaginationStrategy pagination strategies. Stream {name} uses " 4395 f"{type(strategy).__name__}." 4396 ) 4397 else: 4398 # A CustomPaginationStrategy carries its page size in its own code, so this is only checkable 4399 # for the strategies the CDK defines. `page_size` is optional on both of them, and without it 4400 # `get_page_size` returns None, the paginator injects nothing, and the reduction is a dead end 4401 # that only surfaces on the first failing response - after records have been emitted. 4402 self._validate_page_size_is_reducible(strategy, model.page_size_reduction, name) 4403 if isinstance(strategy, CursorPaginationModel): 4404 self._validate_stop_condition_is_reduction_aware( 4405 strategy, model.page_size_reduction, name 4406 ) 4407 4408 @staticmethod 4409 def _validate_stop_condition_is_reduction_aware( 4410 strategy: CursorPaginationModel, 4411 page_size_reduction: Optional[PageSizeReductionModel], 4412 name: str, 4413 ) -> None: 4414 """ 4415 A `stop_condition` comparing `last_page_size` to a hardcoded page size reads a full reduced page as a 4416 short page and ends the pagination early, dropping the rest of the partition without failing. The 4417 strategy exposes the page size that was actually requested as `page_size`, so the condition can be 4418 written correctly - but only if it is, which is what this checks. 4419 4420 The condition is parsed as a Jinja expression rather than matched as a string: only the AST tells 4421 `page_size`, which follows the reduction, apart from `config['page_size']`, which does not, and only the 4422 AST tells an inequality, which a reduction can invalidate, apart from `last_page_size == 0`, which it 4423 cannot. A condition that never names `last_page_size` is not waved through: a count read from the 4424 response body - `{{ response['data'] | length < 100 }}` - truncates in exactly the same way, and which 4425 response field counts the records of a page is not knowable here. A shape the analysis does not 4426 understand is warned about rather than rejected - this runs at stream construction, so a false 4427 rejection takes `check`, `discover` and `read` down with it. 4428 """ 4429 stop_condition = strategy.stop_condition 4430 if not stop_condition: 4431 return 4432 4433 minimum_page_size = ( 4434 page_size_reduction.minimum_page_size if page_size_reduction else None 4435 ) or 1 4436 verdict, reason = classify_stop_condition(stop_condition, minimum_page_size) 4437 if verdict is StopConditionSafety.TRUNCATES: 4438 raise ValueError( 4439 f"`page_size_reduction` on stream {name} cannot be used with the `stop_condition` " 4440 f"{stop_condition!r}: {reason}. The pagination would then end early, silently dropping the " 4441 f"rest of the partition. Compare against the `page_size` interpolation variable instead, " 4442 f"which holds the page size that was actually requested (for example " 4443 f"`{{{{ last_page_size < page_size }}}}`), or test the page for emptiness with " 4444 f"`{{{{ last_page_size == 0 }}}}`." 4445 ) 4446 if verdict is StopConditionSafety.UNKNOWN: 4447 LOGGER.warning( 4448 f"Stream {name} uses `page_size_reduction` with the `stop_condition` {stop_condition!r}, " 4449 f"which could not be checked against the reduction because {reason}. Make sure a page that is " 4450 f"full at a reduced page size does not satisfy it, otherwise the pagination ends early and the " 4451 f"rest of the partition is silently dropped. Comparing against the `page_size` interpolation " 4452 f"variable, which holds the page size that was actually requested, is always safe." 4453 ) 4454 4455 @staticmethod 4456 def _validate_page_size_is_reducible( 4457 strategy: Union[CursorPaginationModel, OffsetIncrementModel], 4458 page_size_reduction: Optional[PageSizeReductionModel], 4459 name: str, 4460 ) -> None: 4461 page_size = strategy.page_size 4462 if page_size is None: 4463 raise ValueError( 4464 f"`page_size_reduction` requires `page_size` on the pagination strategy of stream {name}: " 4465 f"without it the paginator does not send a page size, so there is nothing to reduce." 4466 ) 4467 4468 # The schema allows a string so that the page size can be interpolated. `page_size` is 4469 # `Optional[Union[int, str]]` and pydantic v1 tries `int` first, so both `100` and `"100"` arrive as 4470 # an int and only a genuinely non-numeric template stays a str. Such a template is only known once 4471 # the config is available, so it is left to the runtime check in `PageSizeReducer.reduce`. 4472 try: 4473 configured_page_size: Optional[int] = int(page_size) 4474 except ValueError: 4475 configured_page_size = None 4476 4477 minimum_page_size = ( 4478 page_size_reduction.minimum_page_size if page_size_reduction else None 4479 ) or 1 4480 if configured_page_size is not None and configured_page_size <= minimum_page_size: 4481 raise ValueError( 4482 f"`page_size_reduction` on stream {name} can never reduce its page size: the pagination " 4483 f"strategy's `page_size` is {configured_page_size} and `minimum_page_size` is " 4484 f"{minimum_page_size}. Lower `minimum_page_size` or raise `page_size`." 4485 ) 4486 4487 # Each reduction divides the page size by `reduction_factor` and spends one attempt, so an unbroken 4488 # run of failures bottoms out at `page_size / reduction_factor ** max_attempts` whatever 4489 # `minimum_page_size` says. Only warned about, and not raised: pages that succeed in between restart 4490 # the budget while `NEVER` keeps the page size, so the floor is reachable over a partition even when 4491 # it is out of reach of a single run - and a manifest that deliberately gives up earlier than its 4492 # floor is not wrong, only worth pointing out. 4493 reduction_factor = ( 4494 page_size_reduction.reduction_factor if page_size_reduction else None 4495 ) or 2.0 4496 max_attempts = (page_size_reduction.max_attempts if page_size_reduction else None) or 5 4497 # Only when there is a floor worth reaching. The default of 1 is out of reach of the default budget on 4498 # any page size above 32, so this would otherwise fire on nearly every stream that opts in - and 4499 # `__fields_set__` would not help, since it records that a value was supplied and not that it differs 4500 # from the default, so spelling `minimum_page_size: 1` out longhand would earn the warning. 4501 if ( 4502 configured_page_size is not None 4503 and minimum_page_size > 1 4504 and reduction_factor**max_attempts < configured_page_size / minimum_page_size 4505 ): 4506 reachable_page_size = max( 4507 minimum_page_size, int(configured_page_size // reduction_factor**max_attempts) 4508 ) 4509 LOGGER.warning( 4510 f"Stream {name} sets `minimum_page_size` to {minimum_page_size}, which `page_size_reduction` " 4511 f"cannot reach in one run of failing pages: `max_attempts` is {max_attempts} and " 4512 f"`reduction_factor` is {reduction_factor}, so {max_attempts} reductions of a page size of " 4513 f"{configured_page_size} stop at {reachable_page_size} records per page and the sync then " 4514 f"fails with a transient error. Whichever of the two bounds is tighter wins; reaching " 4515 f"{minimum_page_size} in one run needs `max_attempts` of at least " 4516 f"{math.ceil(math.log(configured_page_size / minimum_page_size, reduction_factor))}." 4517 ) 4518 4519 def _reject_reduce_page_size_action(self, requester: Any, description: str) -> None: 4520 """ 4521 `error_handler` is defined on `HttpRequester`, which is referenced by requesters that have no page of 4522 their own, so REDUCE_PAGE_SIZE is schema-legal in places where nothing can honor it. Only 4523 `SimpleRetriever._read_pages` re-issues a page, so every other requester is rejected here rather than 4524 surfacing the exception mid-sync as a generic failure. 4525 """ 4526 if requester is None: 4527 return 4528 if self._uses_reduce_page_size_action(getattr(requester, "error_handler", None)): 4529 raise ValueError( 4530 f"The REDUCE_PAGE_SIZE response action is not supported on the {description}: only the main " 4531 f"requester of a SimpleRetriever can re-issue its page with a smaller page size. Use a " 4532 f"different action on that error handler." 4533 ) 4534 4535 def _uses_reduce_page_size_action(self, error_handler: Any) -> bool: 4536 if isinstance(error_handler, CompositeErrorHandlerModel): 4537 return any( 4538 self._uses_reduce_page_size_action(nested) 4539 for nested in error_handler.error_handlers 4540 ) 4541 if isinstance(error_handler, DefaultErrorHandlerModel): 4542 return any( 4543 response_filter.action == HttpResponseFilterActionModel.REDUCE_PAGE_SIZE 4544 for response_filter in error_handler.response_filters or [] 4545 ) 4546 # A CustomErrorHandler can return any action and we cannot inspect it, so we do not validate it. 4547 return False 4548 4549 def _create_request_window_splitting( 4550 self, 4551 model: SimpleRetrieverModel, 4552 name: str, 4553 cursor: Optional[Cursor], 4554 incremental_sync: Optional[ 4555 Union[IncrementingCountCursorModel, DatetimeBasedCursorModel, SyncTokenCursorModel] 4556 ] = None, 4557 query_properties: Optional[QueryProperties] = None, 4558 file_uploader: Optional[DefaultFileUploader] = None, 4559 ) -> Optional[RequestWindowSplitting]: 4560 # A CustomRequester does not necessarily define an error handler. A CustomRequester that does define 4561 # one keeps it as a raw dict rather than a typed model, so this returns False for it as well. 4562 error_handler = getattr(model.requester, "error_handler", None) 4563 uses_action = self._uses_split_request_window_action(error_handler) 4564 if uses_action and not model.request_window_splitting: 4565 raise ValueError( 4566 f"Stream {name} has a response filter with the SPLIT_REQUEST_WINDOW action but the " 4567 f"retriever does not define `request_window_splitting`. Add a `request_window_splitting` " 4568 f"block to the retriever." 4569 ) 4570 4571 if not model.request_window_splitting: 4572 return None 4573 4574 if not uses_action: 4575 # Not raised: a CustomErrorHandler can resolve to SPLIT_REQUEST_WINDOW without us being able to 4576 # see it, and custom code can raise RequestWindowSplitRequiredException directly without 4577 # going through any error handler at all - so the only safe reaction to a block we cannot tie to 4578 # a known trigger is a warning. 4579 LOGGER.warning( 4580 f"Stream {name} defines `request_window_splitting` but no response filter with the " 4581 f"SPLIT_REQUEST_WINDOW action was found on its requester. The request window will never " 4582 f"be split unless a custom error handler or custom code resolves to that action." 4583 ) 4584 4585 self._validate_request_window_splitting_is_supported( 4586 name, 4587 cursor, 4588 incremental_sync, 4589 model.request_window_splitting.min_split_window, 4590 query_properties, 4591 file_uploader, 4592 ) 4593 4594 return RequestWindowSplitting( 4595 failure_message=model.request_window_splitting.failure_message, 4596 min_split_window=parse_duration(model.request_window_splitting.min_split_window) 4597 if model.request_window_splitting.min_split_window 4598 else None, 4599 ) 4600 4601 def _validate_request_window_splitting_is_supported( 4602 self, 4603 name: str, 4604 cursor: Optional[Cursor], 4605 incremental_sync: Optional[ 4606 Union[IncrementingCountCursorModel, DatetimeBasedCursorModel, SyncTokenCursorModel] 4607 ] = None, 4608 min_split_window: Optional[str] = None, 4609 query_properties: Optional[QueryProperties] = None, 4610 file_uploader: Optional[DefaultFileUploader] = None, 4611 ) -> None: 4612 """ 4613 Request window splitting replaces a failing slice with smaller children derived from the stream's own 4614 cursor, so it only makes sense on a stream whose cursor exposes a `split_request_window` method and it 4615 is rejected for the same structural reasons `page_size_reduction` is: `additional_query_properties` and 4616 a `file_uploader` both mean records of the failing window may already have been emitted by the time the 4617 split is requested, from inside the record generator rather than from a page boundary this retriever 4618 controls. 4619 """ 4620 if not hasattr(cursor, "split_request_window"): 4621 raise ValueError( 4622 f"`request_window_splitting` requires an incremental cursor that supports window splitting on " 4623 f"stream {name}. Found {type(cursor).__name__ if cursor is not None else 'no cursor'}: this is " 4624 f"only supported today for a `DatetimeBasedCursor` with `cursor_granularity` set." 4625 ) 4626 4627 if not ( 4628 isinstance(incremental_sync, DatetimeBasedCursorModel) 4629 and incremental_sync.cursor_granularity 4630 ): 4631 # `cursor_granularity` is both the smallest window the connector will ever request and what keeps 4632 # two child windows from overlapping at their shared edge, so `split_request_window` cannot split 4633 # at all without it - checked here, on the manifest model, rather than by reaching into the 4634 # cursor's private state from outside the class it belongs to. 4635 raise ValueError( 4636 f"`request_window_splitting` requires `cursor_granularity` on the `DatetimeBasedCursor` of " 4637 f"stream {name}: without it there is no smallest window to stop splitting at, and no way to " 4638 f"keep two child windows from overlapping at their shared edge." 4639 ) 4640 4641 if incremental_sync.is_client_side_incremental: 4642 # A client-side-incremental cursor filters records after they are read rather than sending the 4643 # window as request parameters, so every child window sends the exact same request as its parent: 4644 # splitting changes nothing, and the sync would keep splitting all the way to `min_split_window` 4645 # or the safety-net depth before failing. 4646 raise ValueError( 4647 f"`request_window_splitting` cannot be used together with `is_client_side_incremental` on " 4648 f"stream {name}: the window is filtered client-side rather than sent to the API, so splitting " 4649 f"it would not change the request and could not resolve a SPLIT_REQUEST_WINDOW response." 4650 ) 4651 4652 parsed_cursor_granularity = parse_duration(incremental_sync.cursor_granularity) 4653 if isinstance( 4654 parsed_cursor_granularity, datetime.timedelta 4655 ) and parsed_cursor_granularity <= datetime.timedelta(0): 4656 # Passes the truthy check above (the string is non-empty), but a zero-or-negative granularity can 4657 # never produce a child strictly smaller than the parent. 4658 raise ValueError( 4659 f"`request_window_splitting` requires a `cursor_granularity` greater than zero on stream " 4660 f"{name}: `{incremental_sync.cursor_granularity}` parses to a zero-length duration, which can " 4661 f"never produce a smaller child window." 4662 ) 4663 4664 smallest_format_unit = self._smallest_datetime_format_unit(incremental_sync.datetime_format) 4665 if ( 4666 smallest_format_unit is not None 4667 and isinstance(parsed_cursor_granularity, datetime.timedelta) 4668 and parsed_cursor_granularity < smallest_format_unit 4669 ): 4670 # Two children split closer together than `datetime_format` can render would format to the same 4671 # boundary value instead of splitting. 4672 raise ValueError( 4673 f"`request_window_splitting` requires `cursor_granularity` to be no finer than what " 4674 f"`datetime_format` can represent on stream {name}: `datetime_format` " 4675 f"{incremental_sync.datetime_format!r} cannot represent a value finer than " 4676 f"{smallest_format_unit}, but `cursor_granularity` {incremental_sync.cursor_granularity!r} is " 4677 f"finer than that. Use a `datetime_format` precise enough to represent `cursor_granularity`, " 4678 f"or a coarser `cursor_granularity`." 4679 ) 4680 4681 if min_split_window: 4682 parsed_min_split_window = parse_duration(min_split_window) 4683 if not isinstance( 4684 parsed_min_split_window, datetime.timedelta 4685 ) or parsed_min_split_window <= datetime.timedelta(0): 4686 # A duration with years or months (e.g. `P1M`) parses to an `isodate.Duration`, which has no 4687 # fixed length to compare a window's span against - `split_request_window` would raise 4688 # mid-sync trying to. A zero-or-negative duration would never stop splitting. 4689 raise ValueError( 4690 f"`min_split_window` must be a positive duration expressible as a fixed number of " 4691 f"days/hours/minutes/seconds on stream {name}: `{min_split_window}` is not." 4692 ) 4693 4694 if query_properties: 4695 raise ValueError( 4696 f"`request_window_splitting` cannot be used together with query properties on stream {name}. " 4697 f"Records from the earlier property chunks may have already been emitted when a chunk asks to " 4698 f"split the window, so splitting and re-reading it could duplicate them." 4699 ) 4700 4701 if file_uploader: 4702 raise ValueError( 4703 f"`request_window_splitting` cannot be used together with a `file_uploader` on stream {name}. " 4704 f"The file uploader sends one request per record from inside the window's record generator, so " 4705 f"a split requested partway through could re-emit records already yielded by it." 4706 ) 4707 4708 @staticmethod 4709 def _smallest_datetime_format_unit(datetime_format: str) -> Optional[datetime.timedelta]: 4710 """ 4711 :return: the finest duration `datetime_format` can distinguish, per 4712 `_DATETIME_FORMAT_DIRECTIVE_GRANULARITY`, or `None` if it contains none of those directives. 4713 """ 4714 matches = [ 4715 granularity 4716 for directive, granularity in _DATETIME_FORMAT_DIRECTIVE_GRANULARITY 4717 if directive in datetime_format 4718 ] 4719 return min(matches) if matches else None 4720 4721 def _reject_split_request_window_action(self, requester: Any, description: str) -> None: 4722 """ 4723 `error_handler` is defined on `HttpRequester`, which is referenced by requesters that have no window of 4724 their own to split, so SPLIT_REQUEST_WINDOW is schema-legal in places where nothing can honor it. Only 4725 the main requester of a `SimpleRetriever` reads a cursor-sliced window, so every other requester is 4726 rejected here rather than surfacing the exception mid-sync as a generic failure. 4727 """ 4728 if requester is None: 4729 return 4730 if self._uses_split_request_window_action(getattr(requester, "error_handler", None)): 4731 raise ValueError( 4732 f"The SPLIT_REQUEST_WINDOW response action is not supported on the {description}: only the " 4733 f"main requester of a SimpleRetriever reads a window that can be split. Use a different " 4734 f"action on that error handler." 4735 ) 4736 4737 def _uses_split_request_window_action(self, error_handler: Any) -> bool: 4738 if isinstance(error_handler, CompositeErrorHandlerModel): 4739 return any( 4740 self._uses_split_request_window_action(nested) 4741 for nested in error_handler.error_handlers 4742 ) 4743 if isinstance(error_handler, DefaultErrorHandlerModel): 4744 return any( 4745 response_filter.action == HttpResponseFilterActionModel.SPLIT_REQUEST_WINDOW 4746 for response_filter in error_handler.response_filters or [] 4747 ) 4748 # A CustomErrorHandler can return any action and we cannot inspect it, so we do not validate it. 4749 return False 4750 4751 def _create_pagination_tracker_factory( 4752 self, model: Optional[PaginationResetModel], cursor: Cursor 4753 ) -> Callable[[], PaginationTracker]: 4754 if model is None: 4755 return lambda: PaginationTracker() 4756 4757 # Until we figure out a way to use any cursor for PaginationTracker, we will have to have this cursor selector logic 4758 cursor_factory: Callable[[], Optional[ConcurrentCursor]] = lambda: None 4759 if model.action == PaginationResetActionModel.RESET: 4760 # in that case, we will let cursor_factory to return None even if the stream has a cursor 4761 pass 4762 elif model.action == PaginationResetActionModel.SPLIT_USING_CURSOR: 4763 if isinstance(cursor, ConcurrentCursor): 4764 cursor_factory = lambda: cursor.copy_without_state() # type: ignore # the if condition validates that it is a ConcurrentCursor 4765 elif isinstance(cursor, ConcurrentPerPartitionCursor): 4766 cursor_factory = lambda: cursor._cursor_factory.create( # type: ignore # if this becomes a problem, we would need to extract the cursor_factory instantiation logic and make it accessible here 4767 {}, datetime.timedelta(0) 4768 ) 4769 elif not isinstance(cursor, FinalStateCursor): 4770 LOGGER.warning( 4771 "Unknown cursor for PaginationTracker. Pagination resets might not work properly" 4772 ) 4773 else: 4774 raise ValueError(f"Unknown PaginationReset action: {model.action}") 4775 4776 limit = model.limits.number_of_records if model and model.limits else None 4777 return lambda: PaginationTracker(cursor_factory(), limit) 4778 4779 def _get_log_formatter( 4780 self, log_formatter: Callable[[Response], Any] | None, name: str 4781 ) -> Callable[[Response], Any] | None: 4782 if self._should_limit_slices_fetched(): 4783 return ( 4784 ( 4785 lambda response: format_http_message( 4786 response, 4787 f"Stream '{name}' request", 4788 f"Request performed in order to extract records for stream '{name}'", 4789 name, 4790 ) 4791 ) 4792 if not log_formatter 4793 else log_formatter 4794 ) 4795 return None 4796 4797 def _should_limit_slices_fetched(self) -> bool: 4798 """ 4799 Returns True if the number of slices fetched should be limited, False otherwise. 4800 This is used to limit the number of slices fetched during tests. 4801 """ 4802 return bool(self._limit_slices_fetched or self._emit_connector_builder_messages) 4803 4804 @staticmethod 4805 def _has_query_properties_in_request_parameters( 4806 requester: Union[HttpRequesterModel, CustomRequesterModel], 4807 ) -> bool: 4808 if not hasattr(requester, "request_parameters"): 4809 return False 4810 request_parameters = requester.request_parameters 4811 if request_parameters and isinstance(request_parameters, Mapping): 4812 for request_parameter in request_parameters.values(): 4813 if isinstance(request_parameter, QueryPropertiesModel): 4814 return True 4815 return False 4816 4817 @staticmethod 4818 def _remove_query_properties( 4819 request_parameters: Mapping[str, Union[str, QueryPropertiesModel]], 4820 ) -> Mapping[str, str]: 4821 return { 4822 parameter_field: request_parameter 4823 for parameter_field, request_parameter in request_parameters.items() 4824 if not isinstance(request_parameter, QueryPropertiesModel) 4825 } 4826 4827 def create_state_delegating_stream( 4828 self, 4829 model: StateDelegatingStreamModel, 4830 config: Config, 4831 **kwargs: Any, 4832 ) -> DefaultStream: 4833 if ( 4834 model.full_refresh_stream.name != model.name 4835 or model.name != model.incremental_stream.name 4836 ): 4837 raise ValueError( 4838 f"state_delegating_stream, full_refresh_stream name and incremental_stream must have equal names. Instead has {model.name}, {model.full_refresh_stream.name} and {model.incremental_stream.name}." 4839 ) 4840 4841 # Resolve api_retention_period with config context (supports Jinja2 interpolation) 4842 resolved_retention_period: Optional[str] = None 4843 if model.api_retention_period: 4844 interpolated_retention = InterpolatedString.create( 4845 model.api_retention_period, parameters=model.parameters or {} 4846 ) 4847 resolved_value = interpolated_retention.eval(config=config) 4848 if resolved_value: 4849 resolved_retention_period = str(resolved_value) 4850 4851 if resolved_retention_period: 4852 for stream_model in (model.full_refresh_stream, model.incremental_stream): 4853 if isinstance(stream_model.incremental_sync, IncrementingCountCursorModel): 4854 raise ValueError( 4855 f"Stream '{model.name}' uses IncrementingCountCursor which is not supported " 4856 f"with api_retention_period. IncrementingCountCursor does not use datetime-based " 4857 f"cursors, so cursor age validation cannot be performed." 4858 ) 4859 if isinstance(stream_model.incremental_sync, SyncTokenCursorModel): 4860 raise ValueError( 4861 f"Stream '{model.name}' uses SyncTokenCursor which is not supported " 4862 f"with api_retention_period. A sync token has no datetime cursor, so " 4863 f"cursor age validation cannot be performed." 4864 ) 4865 4866 stream_state = self._connector_state_manager.get_stream_state(model.name, None) 4867 4868 if not stream_state: 4869 return self._create_component_from_model( # type: ignore[no-any-return] 4870 model.full_refresh_stream, config=config, **kwargs 4871 ) 4872 4873 incremental_stream: DefaultStream = self._create_component_from_model( 4874 model.incremental_stream, config=config, **kwargs 4875 ) # type: ignore[assignment] 4876 4877 # Only run cursor age validation for streams that are in the configured 4878 # catalog (or when no catalog was provided, e.g. during discover / connector 4879 # builder). Streams not selected by the user but instantiated as parent-stream 4880 # dependencies must not go through this path because it emits state messages 4881 # that the destination does not know about, causing "Stream not found" crashes. 4882 stream_is_in_catalog = ( 4883 not self._stream_name_to_configured_stream # no catalog → validate by default 4884 or model.name in self._stream_name_to_configured_stream 4885 ) 4886 if resolved_retention_period and stream_is_in_catalog: 4887 full_refresh_stream: DefaultStream = self._create_component_from_model( 4888 model.full_refresh_stream, config=config, **kwargs 4889 ) # type: ignore[assignment] 4890 if self._is_cursor_older_than_retention_period( 4891 stream_state, 4892 full_refresh_stream.cursor, 4893 incremental_stream.cursor, 4894 resolved_retention_period, 4895 model.name, 4896 ): 4897 # Clear state BEFORE constructing the full_refresh_stream so that 4898 # its cursor starts from start_date instead of the stale cursor. 4899 self._connector_state_manager.update_state_for_stream(model.name, None, {}) 4900 state_message = self._connector_state_manager.create_state_message(model.name, None) 4901 self._message_repository.emit_message(state_message) 4902 return self._create_component_from_model( # type: ignore[no-any-return] 4903 model.full_refresh_stream, config=config, **kwargs 4904 ) 4905 4906 return incremental_stream 4907 4908 @staticmethod 4909 def _is_cursor_older_than_retention_period( 4910 stream_state: Mapping[str, Any], 4911 full_refresh_cursor: Cursor, 4912 incremental_cursor: Cursor, 4913 api_retention_period: str, 4914 stream_name: str, 4915 ) -> bool: 4916 """Check if the cursor value in the state is older than the API's retention period. 4917 4918 Checks cursors in sequence: full refresh cursor first, then incremental cursor. 4919 FinalStateCursor returns now() for completed full refresh state (NO_CURSOR_STATE_KEY), 4920 which is always within retention, so we use incremental. For other states, it returns 4921 None and we fall back to checking the incremental cursor. 4922 4923 Returns True if the cursor is older than the retention period (should use full refresh). 4924 Returns False if the cursor is within the retention period (safe to use incremental). 4925 """ 4926 retention_duration = parse_duration(api_retention_period) 4927 retention_cutoff = datetime.datetime.now(datetime.timezone.utc) - retention_duration 4928 4929 # Check full refresh cursor first 4930 cursor_datetime = full_refresh_cursor.get_cursor_datetime_from_state(stream_state) 4931 4932 # If full refresh cursor returns None, check incremental cursor 4933 if cursor_datetime is None: 4934 cursor_datetime = incremental_cursor.get_cursor_datetime_from_state(stream_state) 4935 4936 if cursor_datetime is None: 4937 # Neither cursor could parse the state - fall back to full refresh to be safe 4938 return True 4939 4940 if cursor_datetime < retention_cutoff: 4941 logging.warning( 4942 f"Stream '{stream_name}' has a cursor value older than " 4943 f"the API's retention period of {api_retention_period} " 4944 f"(cutoff: {retention_cutoff.isoformat()}). " 4945 f"Falling back to full refresh to avoid data loss." 4946 ) 4947 return True 4948 4949 return False 4950 4951 def _get_state_delegating_stream_model( 4952 self, 4953 model: StateDelegatingStreamModel, 4954 parent_state: Optional[Mapping[str, Any]] = None, 4955 ) -> DeclarativeStreamModel: 4956 """Return the appropriate underlying stream model based on state.""" 4957 return ( 4958 model.incremental_stream 4959 if self._connector_state_manager.get_stream_state(model.name, None) or parent_state 4960 else model.full_refresh_stream 4961 ) 4962 4963 _OPTIONAL_ASYNC_STATUS_FIELDS = {"skipped"} 4964 4965 def _create_async_job_status_mapping( 4966 self, model: AsyncJobStatusMapModel, config: Config, **kwargs: Any 4967 ) -> Mapping[str, AsyncJobStatus]: 4968 api_status_to_cdk_status = {} 4969 for cdk_status, api_statuses in model.dict().items(): 4970 if cdk_status == "type": 4971 # This is an element of the dict because of the typing of the CDK but it is not a CDK status 4972 continue 4973 4974 if api_statuses is None: 4975 if cdk_status in self._OPTIONAL_ASYNC_STATUS_FIELDS: 4976 continue 4977 raise ValueError( 4978 f"Required CDK status '{cdk_status}' has no API statuses mapped. " 4979 f"Please provide at least an empty list for required status fields." 4980 ) 4981 4982 for status in api_statuses: 4983 if status in api_status_to_cdk_status: 4984 raise ValueError( 4985 f"API status {status} is already set for CDK status {cdk_status}. Please ensure API statuses are only provided once" 4986 ) 4987 api_status_to_cdk_status[status] = self._get_async_job_status(cdk_status) 4988 return api_status_to_cdk_status 4989 4990 def _get_async_job_status(self, status: str) -> AsyncJobStatus: 4991 match status: 4992 case "running": 4993 return AsyncJobStatus.RUNNING 4994 case "completed": 4995 return AsyncJobStatus.COMPLETED 4996 case "failed": 4997 return AsyncJobStatus.FAILED 4998 case "timeout": 4999 return AsyncJobStatus.TIMED_OUT 5000 case "skipped": 5001 return AsyncJobStatus.SKIPPED 5002 case _: 5003 raise ValueError(f"Unsupported CDK status {status}") 5004 5005 def create_async_retriever( 5006 self, 5007 model: AsyncRetrieverModel, 5008 config: Config, 5009 *, 5010 name: str, 5011 primary_key: Optional[ 5012 Union[str, List[str], List[List[str]]] 5013 ], # this seems to be needed to match create_simple_retriever 5014 stream_slicer: Optional[StreamSlicer], 5015 client_side_incremental_sync: Optional[Dict[str, Any]] = None, 5016 transformations: List[RecordTransformation], 5017 **kwargs: Any, 5018 ) -> AsyncRetriever: 5019 if model.download_target_requester and not model.download_target_extractor: 5020 raise ValueError( 5021 f"`download_target_extractor` required if using a `download_target_requester`" 5022 ) 5023 5024 for requester_field in ( 5025 "creation_requester", 5026 "polling_requester", 5027 "download_requester", 5028 "download_target_requester", 5029 "abort_requester", 5030 "delete_requester", 5031 ): 5032 self._reject_reduce_page_size_action( 5033 getattr(model, requester_field, None), 5034 f"`{requester_field}` of the AsyncRetriever of stream {name}", 5035 ) 5036 self._reject_split_request_window_action( 5037 getattr(model, requester_field, None), 5038 f"`{requester_field}` of the AsyncRetriever of stream {name}", 5039 ) 5040 5041 def _get_download_retriever( 5042 requester: Requester, extractor: RecordExtractor, _decoder: Decoder 5043 ) -> SimpleRetriever: 5044 self._count_dropped_empty_records(extractor) 5045 # We create a record selector for the download retriever 5046 # with no schema normalization and no transformations, neither record filter 5047 # as all this occurs in the record_selector of the AsyncRetriever 5048 record_selector = RecordSelector( 5049 extractor=extractor, 5050 name=name, 5051 record_filter=None, 5052 transformations=[], 5053 schema_normalization=TypeTransformer(TransformConfig.NoTransform), 5054 config=config, 5055 parameters={}, 5056 ) 5057 paginator = ( 5058 self._create_component_from_model( 5059 model=model.download_paginator, 5060 decoder=_decoder, 5061 config=config, 5062 url_base="", 5063 # Only read by the `union` rejection of `OffsetIncrement`. 5064 extractor_model=( 5065 model.download_extractor 5066 if isinstance(model.download_extractor, CombinedExtractorModel) 5067 else None 5068 ), 5069 ) 5070 if model.download_paginator 5071 else NoPagination(parameters={}) 5072 ) 5073 5074 return SimpleRetriever( 5075 requester=requester, 5076 record_selector=record_selector, 5077 primary_key=None, 5078 name=name, 5079 paginator=paginator, 5080 config=config, 5081 parameters={}, 5082 log_formatter=self._get_log_formatter(None, name), 5083 ) 5084 5085 def _get_job_timeout() -> datetime.timedelta: 5086 user_defined_timeout: Optional[int] = ( 5087 int( 5088 InterpolatedString.create( 5089 str(model.polling_job_timeout), 5090 parameters={}, 5091 ).eval(config) 5092 ) 5093 if model.polling_job_timeout 5094 else None 5095 ) 5096 5097 # check for user defined timeout during the test read or 15 minutes 5098 test_read_timeout = datetime.timedelta(minutes=user_defined_timeout or 15) 5099 # default value for non-connector builder is 60 minutes. 5100 default_sync_timeout = datetime.timedelta(minutes=user_defined_timeout or 60) 5101 5102 return ( 5103 test_read_timeout if self._emit_connector_builder_messages else default_sync_timeout 5104 ) 5105 5106 decoder = ( 5107 self._create_component_from_model(model=model.decoder, config=config) 5108 if model.decoder 5109 else JsonDecoder(parameters={}) 5110 ) 5111 record_selector = self._create_component_from_model( 5112 model=model.record_selector, 5113 config=config, 5114 decoder=decoder, 5115 name=name, 5116 transformations=transformations, 5117 client_side_incremental_sync=client_side_incremental_sync, 5118 ) 5119 5120 stream_slicer = stream_slicer or SinglePartitionRouter(parameters={}) 5121 if self._should_limit_slices_fetched(): 5122 stream_slicer = cast( 5123 StreamSlicer, 5124 StreamSlicerTestReadDecorator( 5125 wrapped_slicer=stream_slicer, 5126 maximum_number_of_slices=self._limit_slices_fetched or 5, 5127 ), 5128 ) 5129 5130 creation_requester = self._create_component_from_model( 5131 model=model.creation_requester, 5132 decoder=decoder, 5133 config=config, 5134 name=f"job creation - {name}", 5135 ) 5136 polling_requester = self._create_component_from_model( 5137 model=model.polling_requester, 5138 decoder=decoder, 5139 config=config, 5140 name=f"job polling - {name}", 5141 ) 5142 job_download_components_name = f"job download - {name}" 5143 download_decoder = ( 5144 self._create_component_from_model(model=model.download_decoder, config=config) 5145 if model.download_decoder 5146 else JsonDecoder(parameters={}) 5147 ) 5148 download_extractor = ( 5149 self._create_component_from_model( 5150 model=model.download_extractor, 5151 config=config, 5152 decoder=download_decoder, 5153 parameters=model.parameters, 5154 ) 5155 if model.download_extractor 5156 else DpathExtractor( 5157 [], 5158 config=config, 5159 decoder=download_decoder, 5160 parameters=model.parameters or {}, 5161 ) 5162 ) 5163 download_requester = self._create_component_from_model( 5164 model=model.download_requester, 5165 decoder=download_decoder, 5166 config=config, 5167 name=job_download_components_name, 5168 ) 5169 download_retriever = _get_download_retriever( 5170 download_requester, download_extractor, download_decoder 5171 ) 5172 abort_requester = ( 5173 self._create_component_from_model( 5174 model=model.abort_requester, 5175 decoder=decoder, 5176 config=config, 5177 name=f"job abort - {name}", 5178 ) 5179 if model.abort_requester 5180 else None 5181 ) 5182 delete_requester = ( 5183 self._create_component_from_model( 5184 model=model.delete_requester, 5185 decoder=decoder, 5186 config=config, 5187 name=f"job delete - {name}", 5188 ) 5189 if model.delete_requester 5190 else None 5191 ) 5192 download_target_requester = ( 5193 self._create_component_from_model( 5194 model=model.download_target_requester, 5195 decoder=decoder, 5196 config=config, 5197 name=f"job extract_url - {name}", 5198 ) 5199 if model.download_target_requester 5200 else None 5201 ) 5202 status_extractor = self._create_component_from_model( 5203 model=model.status_extractor, decoder=decoder, config=config, name=name 5204 ) 5205 download_target_extractor = ( 5206 self._create_component_from_model( 5207 model=model.download_target_extractor, 5208 decoder=decoder, 5209 config=config, 5210 name=name, 5211 ) 5212 if model.download_target_extractor 5213 else None 5214 ) 5215 5216 job_repository: AsyncJobRepository = AsyncHttpJobRepository( 5217 creation_requester=creation_requester, 5218 polling_requester=polling_requester, 5219 download_retriever=download_retriever, 5220 download_target_requester=download_target_requester, 5221 abort_requester=abort_requester, 5222 delete_requester=delete_requester, 5223 status_extractor=status_extractor, 5224 status_mapping=self._create_async_job_status_mapping(model.status_mapping, config), 5225 download_target_extractor=download_target_extractor, 5226 job_timeout=_get_job_timeout(), 5227 ) 5228 5229 failed_retry_wait_time_in_seconds: Optional[int] = ( 5230 int( 5231 InterpolatedString.create( 5232 str(model.failed_retry_wait_time_in_seconds), 5233 parameters={}, 5234 ).eval(config) 5235 ) 5236 if model.failed_retry_wait_time_in_seconds 5237 else None 5238 ) 5239 5240 async_job_partition_router = AsyncJobPartitionRouter( 5241 job_orchestrator_factory=lambda stream_slices: AsyncJobOrchestrator( 5242 job_repository, 5243 stream_slices, 5244 self._job_tracker, 5245 self._message_repository, 5246 # FIXME work would need to be done here in order to detect if a stream as a parent stream that is bulk 5247 has_bulk_parent=False, 5248 # set the `job_max_retry` to 1 for the `Connector Builder`` use-case. 5249 # `None` == default retry is set to 3 attempts, under the hood. 5250 job_max_retry=1 if self._emit_connector_builder_messages else None, 5251 failed_retry_wait_time_in_seconds=failed_retry_wait_time_in_seconds, 5252 ), 5253 stream_slicer=stream_slicer, 5254 config=config, 5255 parameters=model.parameters or {}, 5256 ) 5257 5258 return AsyncRetriever( 5259 record_selector=record_selector, 5260 stream_slicer=async_job_partition_router, 5261 config=config, 5262 parameters=model.parameters or {}, 5263 ) 5264 5265 def create_spec(self, model: SpecModel, config: Config, **kwargs: Any) -> Spec: 5266 config_migrations = [ 5267 self._create_component_from_model(migration, config) 5268 for migration in ( 5269 model.config_normalization_rules.config_migrations 5270 if ( 5271 model.config_normalization_rules 5272 and model.config_normalization_rules.config_migrations 5273 ) 5274 else [] 5275 ) 5276 ] 5277 config_transformations = [ 5278 self._create_component_from_model(transformation, config) 5279 for transformation in ( 5280 model.config_normalization_rules.transformations 5281 if ( 5282 model.config_normalization_rules 5283 and model.config_normalization_rules.transformations 5284 ) 5285 else [] 5286 ) 5287 ] 5288 config_validations = [ 5289 self._create_component_from_model(validation, config) 5290 for validation in ( 5291 model.config_normalization_rules.validations 5292 if ( 5293 model.config_normalization_rules 5294 and model.config_normalization_rules.validations 5295 ) 5296 else [] 5297 ) 5298 ] 5299 5300 return Spec( 5301 connection_specification=model.connection_specification, 5302 documentation_url=model.documentation_url, 5303 advanced_auth=model.advanced_auth, 5304 parameters={}, 5305 config_migrations=config_migrations, 5306 config_transformations=config_transformations, 5307 config_validations=config_validations, 5308 ) 5309 5310 def create_substream_partition_router( 5311 self, 5312 model: SubstreamPartitionRouterModel, 5313 config: Config, 5314 *, 5315 stream_name: str, 5316 **kwargs: Any, 5317 ) -> SubstreamPartitionRouter: 5318 parent_stream_configs = [] 5319 if model.parent_stream_configs: 5320 parent_stream_configs.extend( 5321 [ 5322 self.create_parent_stream_config_with_substream_wrapper( 5323 model=parent_stream_config, config=config, stream_name=stream_name, **kwargs 5324 ) 5325 for parent_stream_config in model.parent_stream_configs 5326 ] 5327 ) 5328 5329 return SubstreamPartitionRouter( 5330 parent_stream_configs=parent_stream_configs, 5331 parameters=model.parameters or {}, 5332 config=config, 5333 ) 5334 5335 def create_parent_stream_config_with_substream_wrapper( 5336 self, model: ParentStreamConfigModel, config: Config, *, stream_name: str, **kwargs: Any 5337 ) -> Any: 5338 child_state = self._connector_state_manager.get_stream_state(stream_name, None) 5339 if NO_CURSOR_STATE_KEY in child_state: 5340 # Full refresh streams checkpoint a `{NO_CURSOR_STATE_KEY: true}` sentinel. When such a 5341 # stream is later converted to incremental with an incremental_dependency parent, 5342 # `_instantiate_parent_stream_state_manager` would treat the sentinel's boolean as a legacy 5343 # cursor value and re-key it under the parent's cursor field, crashing cursor initialization. 5344 child_state = { 5345 key: value for key, value in child_state.items() if key != NO_CURSOR_STATE_KEY 5346 } 5347 5348 parent_state: Optional[Mapping[str, Any]] = ( 5349 child_state if model.incremental_dependency and child_state else None 5350 ) 5351 connector_state_manager = self._instantiate_parent_stream_state_manager( 5352 child_state, config, model, parent_state 5353 ) 5354 5355 substream_factory = ModelToComponentFactory( 5356 custom_components_trusted=self._custom_components_trusted, 5357 connector_state_manager=connector_state_manager, 5358 limit_pages_fetched_per_slice=self._limit_pages_fetched_per_slice, 5359 limit_slices_fetched=self._limit_slices_fetched, 5360 emit_connector_builder_messages=self._emit_connector_builder_messages, 5361 disable_retries=self._disable_retries, 5362 disable_cache=self._disable_cache, 5363 message_repository=StateFilteringMessageRepository( 5364 LogAppenderMessageRepositoryDecorator( 5365 { 5366 "airbyte_cdk": {"stream": {"is_substream": True}}, 5367 "http": {"is_auxiliary": True}, 5368 }, 5369 self._message_repository, 5370 self._evaluate_log_level(self._emit_connector_builder_messages), 5371 ), 5372 ), 5373 api_budget=self._api_budget, 5374 # Share the authenticator registry so parent and child streams draw from the 5375 # same token quota counters 5376 rate_limited_authenticators=self._rate_limited_authenticators, 5377 ) 5378 5379 return substream_factory.create_parent_stream_config( 5380 model=model, config=config, stream_name=stream_name, **kwargs 5381 ) 5382 5383 def _instantiate_parent_stream_state_manager( 5384 self, 5385 child_state: MutableMapping[str, Any], 5386 config: Config, 5387 model: ParentStreamConfigModel, 5388 parent_state: Optional[Mapping[str, Any]] = None, 5389 ) -> ConnectorStateManager: 5390 """ 5391 With DefaultStream, the state needs to be provided during __init__ of the cursor as opposed to the 5392 `set_initial_state` flow that existed for the declarative cursors. This state is taken from 5393 self._connector_state_manager.get_stream_state (`self` being a newly created ModelToComponentFactory to account 5394 for the MessageRepository being different). So we need to pass a ConnectorStateManager to the 5395 ModelToComponentFactory that has the parent states. This method populates this if there is a child state and if 5396 incremental_dependency is set. 5397 """ 5398 if model.incremental_dependency and child_state: 5399 parent_stream_name = model.stream.name or "" 5400 extracted_parent_state = ConcurrentPerPartitionCursor.get_parent_state( 5401 child_state, parent_stream_name 5402 ) 5403 5404 if not extracted_parent_state: 5405 extracted_parent_state = ConcurrentPerPartitionCursor.get_global_state( 5406 child_state, parent_stream_name 5407 ) 5408 5409 if not extracted_parent_state and not isinstance(extracted_parent_state, dict): 5410 cursor_values = list(child_state.values()) 5411 # Only a scalar legacy cursor value can seed the parent. Sentinels such as 5412 # `{"__ab_full_refresh_sync_complete": true}` or legacy per-partition `states` 5413 # would crash the parent cursor initialization. 5414 if ( 5415 len(cursor_values) == 1 5416 and isinstance(cursor_values[0], (str, int, float)) 5417 and not isinstance(cursor_values[0], bool) 5418 ): 5419 incremental_sync_model: Union[ 5420 DatetimeBasedCursorModel, 5421 IncrementingCountCursorModel, 5422 ] = ( 5423 model.stream.incremental_sync # type: ignore # if we are there, it is because there is incremental_dependency and therefore there is an incremental_sync on the parent stream 5424 if isinstance(model.stream, DeclarativeStreamModel) 5425 else self._get_state_delegating_stream_model( 5426 model.stream, parent_state=parent_state 5427 ).incremental_sync 5428 ) 5429 cursor_field = InterpolatedString.create( 5430 incremental_sync_model.cursor_field, 5431 parameters=incremental_sync_model.parameters or {}, 5432 ).eval(config) 5433 extracted_parent_state = AirbyteStateMessage( 5434 type=AirbyteStateType.STREAM, 5435 stream=AirbyteStreamState( 5436 stream_descriptor=StreamDescriptor( 5437 name=parent_stream_name, namespace=None 5438 ), 5439 stream_state=AirbyteStateBlob({cursor_field: cursor_values[0]}), 5440 ), 5441 ) 5442 return ConnectorStateManager([extracted_parent_state] if extracted_parent_state else []) 5443 5444 return ConnectorStateManager([]) 5445 5446 @staticmethod 5447 def create_wait_time_from_header( 5448 model: WaitTimeFromHeaderModel, config: Config, **kwargs: Any 5449 ) -> WaitTimeFromHeaderBackoffStrategy: 5450 return WaitTimeFromHeaderBackoffStrategy( 5451 header=model.header, 5452 parameters=model.parameters or {}, 5453 config=config, 5454 regex=model.regex, 5455 max_waiting_time_in_seconds=model.max_waiting_time_in_seconds, 5456 ) 5457 5458 @staticmethod 5459 def create_wait_time_from_response( 5460 model: WaitTimeFromResponseModel, config: Config, **kwargs: Any 5461 ) -> WaitTimeFromResponseBackoffStrategy: 5462 return WaitTimeFromResponseBackoffStrategy( 5463 wait_time_in_seconds=model.wait_time_in_seconds, 5464 parameters=model.parameters or {}, 5465 config=config, 5466 max_waiting_time_in_seconds=model.max_waiting_time_in_seconds, 5467 ) 5468 5469 @staticmethod 5470 def create_wait_until_time_from_header( 5471 model: WaitUntilTimeFromHeaderModel, config: Config, **kwargs: Any 5472 ) -> WaitUntilTimeFromHeaderBackoffStrategy: 5473 return WaitUntilTimeFromHeaderBackoffStrategy( 5474 header=model.header, 5475 parameters=model.parameters or {}, 5476 config=config, 5477 min_wait=model.min_wait, 5478 regex=model.regex, 5479 max_waiting_time_in_seconds=model.max_waiting_time_in_seconds, 5480 ) 5481 5482 def get_message_repository(self) -> MessageRepository: 5483 return self._message_repository 5484 5485 def _evaluate_log_level(self, emit_connector_builder_messages: bool) -> Level: 5486 return Level.DEBUG if emit_connector_builder_messages else Level.INFO 5487 5488 @staticmethod 5489 def create_components_mapping_definition( 5490 model: ComponentMappingDefinitionModel, config: Config, **kwargs: Any 5491 ) -> ComponentMappingDefinition: 5492 interpolated_value = InterpolatedString.create( 5493 model.value, parameters=model.parameters or {} 5494 ) 5495 field_path = [ 5496 InterpolatedString.create(path, parameters=model.parameters or {}) 5497 for path in model.field_path 5498 ] 5499 return ComponentMappingDefinition( 5500 field_path=field_path, # type: ignore[arg-type] # field_path can be str and InterpolatedString 5501 value=interpolated_value, 5502 value_type=ModelToComponentFactory._json_schema_type_name_to_type(model.value_type), 5503 create_or_update=model.create_or_update, 5504 condition=model.condition, 5505 parameters=model.parameters or {}, 5506 ) 5507 5508 def create_http_components_resolver( 5509 self, model: HttpComponentsResolverModel, config: Config, stream_name: Optional[str] = None 5510 ) -> Any: 5511 partition_router = self._build_stream_slicer_from_partition_router(model.retriever, config) 5512 retriever = self._create_component_from_model( 5513 model=model.retriever, 5514 config=config, 5515 name=f"{stream_name if stream_name else '__http_components_resolver'}", 5516 primary_key=None, 5517 stream_slicer=partition_router, 5518 partition_router=partition_router, 5519 transformations=[], 5520 ) 5521 5522 components_mapping = [] 5523 for component_mapping_definition_model in model.components_mapping: 5524 if component_mapping_definition_model.condition: 5525 raise ValueError("`condition` is only supported for `ConfigComponentsResolver`") 5526 components_mapping.append( 5527 self._create_component_from_model( 5528 model=component_mapping_definition_model, 5529 value_type=ModelToComponentFactory._json_schema_type_name_to_type( 5530 component_mapping_definition_model.value_type 5531 ), 5532 config=config, 5533 ) 5534 ) 5535 5536 return HttpComponentsResolver( 5537 retriever=retriever, 5538 # AsyncRetriever reads records only from the job slices its own slicer yields 5539 stream_slicer=retriever.stream_slicer 5540 if isinstance(retriever, AsyncRetriever) 5541 else partition_router, 5542 config=config, 5543 components_mapping=components_mapping, 5544 parameters=model.parameters or {}, 5545 ) 5546 5547 @staticmethod 5548 def create_stream_config( 5549 model: StreamConfigModel, config: Config, **kwargs: Any 5550 ) -> StreamConfig: 5551 model_configs_pointer: List[Union[InterpolatedString, str]] = ( 5552 [x for x in model.configs_pointer] if model.configs_pointer else [] 5553 ) 5554 5555 return StreamConfig( 5556 configs_pointer=model_configs_pointer, 5557 default_values=model.default_values, 5558 parameters=model.parameters or {}, 5559 ) 5560 5561 def create_config_components_resolver( 5562 self, 5563 model: ConfigComponentsResolverModel, 5564 config: Config, 5565 ) -> Any: 5566 model_stream_configs = ( 5567 model.stream_config if isinstance(model.stream_config, list) else [model.stream_config] 5568 ) 5569 5570 stream_configs = [ 5571 self._create_component_from_model( 5572 stream_config, config=config, parameters=model.parameters or {} 5573 ) 5574 for stream_config in model_stream_configs 5575 ] 5576 5577 components_mapping = [ 5578 self._create_component_from_model( 5579 model=components_mapping_definition_model, 5580 value_type=ModelToComponentFactory._json_schema_type_name_to_type( 5581 components_mapping_definition_model.value_type 5582 ), 5583 config=config, 5584 parameters=model.parameters, 5585 ) 5586 for components_mapping_definition_model in model.components_mapping 5587 ] 5588 5589 return ConfigComponentsResolver( 5590 stream_configs=stream_configs, 5591 config=config, 5592 components_mapping=components_mapping, 5593 parameters=model.parameters or {}, 5594 ) 5595 5596 def create_parametrized_components_resolver( 5597 self, 5598 model: ParametrizedComponentsResolverModel, 5599 config: Config, 5600 ) -> ParametrizedComponentsResolver: 5601 stream_parameters = StreamParametersDefinition( 5602 list_of_parameters_for_stream=model.stream_parameters.list_of_parameters_for_stream 5603 ) 5604 5605 components_mapping = [] 5606 for components_mapping_definition_model in model.components_mapping: 5607 if components_mapping_definition_model.condition: 5608 raise ValueError("`condition` is only supported for `ConfigComponentsResolver`") 5609 components_mapping.append( 5610 self._create_component_from_model( 5611 model=components_mapping_definition_model, 5612 value_type=ModelToComponentFactory._json_schema_type_name_to_type( 5613 components_mapping_definition_model.value_type 5614 ), 5615 config=config, 5616 ) 5617 ) 5618 return ParametrizedComponentsResolver( 5619 stream_parameters=stream_parameters, 5620 config=config, 5621 components_mapping=components_mapping, 5622 parameters=model.parameters or {}, 5623 ) 5624 5625 _UNSUPPORTED_DECODER_ERROR = ( 5626 "Specified decoder of {decoder_type} is not supported for pagination." 5627 "Please set as `JsonDecoder`, `XmlDecoder`, or a `CompositeRawDecoder` with an inner_parser of `JsonParser` or `GzipParser` instead." 5628 "If using `GzipParser`, please ensure that the lowest level inner_parser is a `JsonParser`." 5629 ) 5630 5631 def _is_supported_decoder_for_pagination(self, decoder: Decoder) -> bool: 5632 if isinstance(decoder, (JsonDecoder, XmlDecoder)): 5633 return True 5634 elif isinstance(decoder, CompositeRawDecoder): 5635 return self._is_supported_parser_for_pagination(decoder.parser) 5636 else: 5637 return False 5638 5639 def _is_supported_parser_for_pagination(self, parser: Parser) -> bool: 5640 if isinstance(parser, JsonParser): 5641 return True 5642 elif isinstance(parser, GzipParser): 5643 return isinstance(parser.inner_parser, JsonParser) 5644 else: 5645 return False 5646 5647 def create_http_api_budget( 5648 self, model: HTTPAPIBudgetModel, config: Config, **kwargs: Any 5649 ) -> HttpAPIBudget: 5650 policies = [ 5651 self._create_component_from_model(model=policy, config=config) 5652 for policy in model.policies 5653 ] 5654 5655 return HttpAPIBudget( 5656 policies=policies, 5657 ratelimit_reset_header=model.ratelimit_reset_header or "ratelimit-reset", 5658 ratelimit_remaining_header=model.ratelimit_remaining_header or "ratelimit-remaining", 5659 status_codes_for_ratelimit_hit=model.status_codes_for_ratelimit_hit or [429], 5660 ) 5661 5662 def create_fixed_window_call_rate_policy( 5663 self, model: FixedWindowCallRatePolicyModel, config: Config, **kwargs: Any 5664 ) -> FixedWindowCallRatePolicy: 5665 matchers = [ 5666 self._create_component_from_model(model=matcher, config=config) 5667 for matcher in model.matchers 5668 ] 5669 5670 # Set the initial reset timestamp to 10 days from now. 5671 # This value will be updated by the first request. 5672 return FixedWindowCallRatePolicy( 5673 next_reset_ts=datetime.datetime.now() + datetime.timedelta(days=10), 5674 period=parse_duration(model.period), 5675 call_limit=model.call_limit, 5676 matchers=matchers, 5677 ) 5678 5679 def create_file_uploader( 5680 self, model: FileUploaderModel, config: Config, **kwargs: Any 5681 ) -> FileUploader: 5682 self._reject_reduce_page_size_action(model.requester, "requester of a `file_uploader`") 5683 self._reject_split_request_window_action(model.requester, "requester of a `file_uploader`") 5684 name = "File Uploader" 5685 requester = self._create_component_from_model( 5686 model=model.requester, 5687 config=config, 5688 name=name, 5689 **kwargs, 5690 ) 5691 download_target_extractor = self._create_component_from_model( 5692 model=model.download_target_extractor, 5693 config=config, 5694 name=name, 5695 **kwargs, 5696 ) 5697 emit_connector_builder_messages = self._emit_connector_builder_messages 5698 file_uploader = DefaultFileUploader( 5699 requester=requester, 5700 download_target_extractor=download_target_extractor, 5701 config=config, 5702 file_writer=NoopFileWriter() 5703 if emit_connector_builder_messages 5704 else LocalFileSystemFileWriter(), 5705 parameters=model.parameters or {}, 5706 filename_extractor=model.filename_extractor if model.filename_extractor else None, 5707 ) 5708 5709 return ( 5710 ConnectorBuilderFileUploader(file_uploader) 5711 if emit_connector_builder_messages 5712 else file_uploader 5713 ) 5714 5715 def create_moving_window_call_rate_policy( 5716 self, model: MovingWindowCallRatePolicyModel, config: Config, **kwargs: Any 5717 ) -> MovingWindowCallRatePolicy: 5718 rates = [ 5719 self._create_component_from_model(model=rate, config=config) for rate in model.rates 5720 ] 5721 matchers = [ 5722 self._create_component_from_model(model=matcher, config=config) 5723 for matcher in model.matchers 5724 ] 5725 return MovingWindowCallRatePolicy( 5726 rates=rates, 5727 matchers=matchers, 5728 ) 5729 5730 def create_unlimited_call_rate_policy( 5731 self, model: UnlimitedCallRatePolicyModel, config: Config, **kwargs: Any 5732 ) -> UnlimitedCallRatePolicy: 5733 matchers = [ 5734 self._create_component_from_model(model=matcher, config=config) 5735 for matcher in model.matchers 5736 ] 5737 5738 return UnlimitedCallRatePolicy( 5739 matchers=matchers, 5740 ) 5741 5742 def create_rate(self, model: RateModel, config: Config, **kwargs: Any) -> Rate: 5743 interpolated_limit = InterpolatedString.create(str(model.limit), parameters={}) 5744 return Rate( 5745 limit=int(interpolated_limit.eval(config=config)), 5746 interval=parse_duration(model.interval), 5747 ) 5748 5749 def create_http_request_matcher( 5750 self, model: HttpRequestRegexMatcherModel, config: Config, **kwargs: Any 5751 ) -> HttpRequestRegexMatcher: 5752 weight = model.weight 5753 if weight is not None: 5754 if isinstance(weight, str): 5755 weight = int(InterpolatedString.create(weight, parameters={}).eval(config)) 5756 else: 5757 weight = int(weight) 5758 if weight < 1: 5759 raise ValueError(f"weight must be >= 1, got {weight}") 5760 return HttpRequestRegexMatcher( 5761 method=model.method, 5762 url_base=model.url_base, 5763 url_path_pattern=model.url_path_pattern, 5764 params=model.params, 5765 headers=model.headers, 5766 weight=weight, 5767 ) 5768 5769 def create_rate_limited_multiple_token_authenticator( 5770 self, 5771 model: RateLimitedMultipleTokenAuthenticatorModel, 5772 config: Config, 5773 **kwargs: Any, 5774 ) -> RateLimitedMultipleTokenAuthenticator: 5775 if isinstance(model.tokens, str): 5776 tokens_value = InterpolatedString.create(model.tokens, parameters={}).eval(config) 5777 delimiter = model.token_delimiter or "," 5778 tokens = [ 5779 token.strip() for token in str(tokens_value).split(delimiter) if token.strip() 5780 ] 5781 else: 5782 tokens = [ 5783 token_value 5784 for token in model.tokens 5785 if ( 5786 token_value := str( 5787 InterpolatedString.create(token, parameters={}).eval(config) 5788 ).strip() 5789 ) 5790 ] 5791 5792 quota_specs = [ 5793 { 5794 "name": quota_model.name, 5795 "remaining_path": quota_model.remaining_path, 5796 "reset_path": quota_model.reset_path, 5797 "limit_path": quota_model.limit_path, 5798 "remaining_header": quota_model.remaining_header, 5799 "reset_header": quota_model.reset_header, 5800 "limit_header": quota_model.limit_header, 5801 # Normalize the same way as the runtime TokenQuota below, so an omitted field 5802 # and an explicit `[]` key identically and keep sharing one set of counters. 5803 "exhaustion_status_codes": quota_model.exhaustion_status_codes or [], 5804 "matchers": [ 5805 { 5806 "method": matcher_model.method, 5807 "url_base": matcher_model.url_base, 5808 "url_path_pattern": matcher_model.url_path_pattern, 5809 "params": matcher_model.params, 5810 "headers": matcher_model.headers, 5811 "weight": matcher_model.weight, 5812 } 5813 for matcher_model in quota_model.matchers or [] 5814 ], 5815 } 5816 for quota_model in model.quotas 5817 ] 5818 5819 quota_status_url = str( 5820 InterpolatedString.create(model.quota_status_source.url, parameters={}).eval(config) 5821 ) 5822 quota_status_http_method = ( 5823 model.quota_status_source.http_method.value 5824 if model.quota_status_source.http_method 5825 else "GET" 5826 ) 5827 quota_status_headers = { 5828 key: str(InterpolatedString.create(value, parameters={}).eval(config)) 5829 for key, value in (model.quota_status_source.request_headers or {}).items() 5830 } 5831 # Normalize the same way as the quota specs above, so an omitted field and an explicit 5832 # `[]` key identically and keep sharing one set of counters. Deduplicated as well as 5833 # sorted, because the runtime turns this into a set: without it `[404]` and `[404, 404]` 5834 # would key differently and stop sharing counters while behaving identically. 5835 quota_status_unavailable_status_codes = sorted( 5836 set(model.quota_status_source.unavailable_status_codes or []) 5837 ) 5838 auth_method = model.auth_method or "Bearer" 5839 header = model.header or "Authorization" 5840 max_wait_time_str = str( 5841 InterpolatedString.create(model.max_wait_time or "PT2H", parameters={}).eval(config) 5842 ) 5843 max_wait_time = parse_duration(max_wait_time_str) 5844 if not isinstance(max_wait_time, datetime.timedelta): 5845 raise ValueError( 5846 f"max_wait_time must be a fixed-length ISO 8601 duration (e.g. 'PT2H'); " 5847 f"calendar-unit durations like '{max_wait_time_str}' are not supported" 5848 ) 5849 budget_reserve_fraction = ( 5850 model.budget_reserve_fraction if model.budget_reserve_fraction is not None else 0.1 5851 ) 5852 budget_min_reserve = ( 5853 model.budget_min_reserve if model.budget_min_reserve is not None else 50 5854 ) 5855 5856 # Reuse the same instance for identical definitions so that all streams share the 5857 # same token quota counters (similar to how api_budget is shared). The key is built 5858 # from the resolved constructor arguments rather than the raw model so that 5859 # stream-specific `$parameters` propagated onto the model (and its nested components) 5860 # cannot break instance sharing. 5861 cache_key = json.dumps( 5862 { 5863 "tokens": tokens, 5864 "quotas": quota_specs, 5865 "quota_status_url": quota_status_url, 5866 "quota_status_http_method": quota_status_http_method, 5867 "quota_status_headers": quota_status_headers, 5868 "quota_status_unavailable_status_codes": quota_status_unavailable_status_codes, 5869 "auth_method": auth_method, 5870 "header": header, 5871 "max_wait_time": max_wait_time.total_seconds(), 5872 "budget_reserve_fraction": budget_reserve_fraction, 5873 "budget_min_reserve": budget_min_reserve, 5874 }, 5875 sort_keys=True, 5876 ) 5877 if cache_key in self._rate_limited_authenticators: 5878 return self._rate_limited_authenticators[cache_key] 5879 5880 quotas = [ 5881 TokenQuota( 5882 name=quota_model.name, 5883 remaining_path=quota_model.remaining_path, 5884 reset_path=quota_model.reset_path, 5885 limit_path=quota_model.limit_path, 5886 remaining_header=quota_model.remaining_header, 5887 reset_header=quota_model.reset_header, 5888 limit_header=quota_model.limit_header, 5889 exhaustion_status_codes=quota_model.exhaustion_status_codes or [], 5890 matchers=[ 5891 self.create_http_request_matcher(matcher_model, config) 5892 for matcher_model in quota_model.matchers or [] 5893 ], 5894 ) 5895 for quota_model in model.quotas 5896 ] 5897 5898 authenticator = RateLimitedMultipleTokenAuthenticator( 5899 tokens=tokens, 5900 quotas=quotas, 5901 quota_status_url=quota_status_url, 5902 quota_status_http_method=quota_status_http_method, 5903 quota_status_headers=quota_status_headers, 5904 quota_status_unavailable_status_codes=quota_status_unavailable_status_codes, 5905 auth_method=auth_method, 5906 header=header, 5907 max_wait_time=max_wait_time, 5908 budget_reserve_fraction=budget_reserve_fraction, 5909 budget_min_reserve=budget_min_reserve, 5910 ) 5911 self._rate_limited_authenticators[cache_key] = authenticator 5912 return authenticator 5913 5914 def set_api_budget(self, component_definition: ComponentDefinition, config: Config) -> None: 5915 self._api_budget = self.create_component( 5916 model_type=HTTPAPIBudgetModel, component_definition=component_definition, config=config 5917 ) 5918 5919 def create_grouping_partition_router( 5920 self, 5921 model: GroupingPartitionRouterModel, 5922 config: Config, 5923 *, 5924 stream_name: str, 5925 **kwargs: Any, 5926 ) -> GroupingPartitionRouter: 5927 underlying_router = self._create_component_from_model( 5928 model=model.underlying_partition_router, 5929 config=config, 5930 stream_name=stream_name, 5931 **kwargs, 5932 ) 5933 if model.group_size < 1: 5934 raise ValueError(f"Group size must be greater than 0, got {model.group_size}") 5935 5936 # Request options in underlying partition routers are not supported for GroupingPartitionRouter 5937 # because they are specific to individual partitions and cannot be aggregated or handled 5938 # when grouping, potentially leading to incorrect API calls. Any request customization 5939 # should be managed at the stream level through the requester's configuration. 5940 if isinstance(underlying_router, SubstreamPartitionRouter): 5941 if any( 5942 parent_config.request_option 5943 for parent_config in underlying_router.parent_stream_configs 5944 ): 5945 raise ValueError("Request options are not supported for GroupingPartitionRouter.") 5946 5947 if isinstance(underlying_router, ListPartitionRouter): 5948 if underlying_router.request_option: 5949 raise ValueError("Request options are not supported for GroupingPartitionRouter.") 5950 5951 return GroupingPartitionRouter( 5952 group_size=model.group_size, 5953 underlying_partition_router=underlying_router, 5954 deduplicate=model.deduplicate if model.deduplicate is not None else True, 5955 config=config, 5956 ) 5957 5958 def create_union_partition_router( 5959 self, 5960 model: UnionPartitionRouterModel, 5961 config: Config, 5962 *, 5963 stream_name: str, 5964 **kwargs: Any, 5965 ) -> UnionPartitionRouter: 5966 # The schema enforces minItems: 2 for manifests; this guard covers construction paths 5967 # that bypass JSON-schema validation (the generated model carries no min_items constraint). 5968 if len(model.partition_routers) < 2: 5969 raise ValueError( 5970 f"UnionPartitionRouter for stream {stream_name} needs at least 2 child partition routers" 5971 ) 5972 5973 partition_routers = [ 5974 self._create_component_from_model( 5975 model=child, 5976 config=config, 5977 stream_name=stream_name, 5978 **kwargs, 5979 ) 5980 for child in model.partition_routers 5981 ] 5982 5983 # partition_field depends only on config/parameters, so it is evaluated once at build 5984 # time; the runtime component always receives a plain string. 5985 partition_field = InterpolatedString.create( 5986 model.partition_field, parameters=model.parameters or {} 5987 ).eval(config) 5988 5989 # Fail fast at build time when a built-in child router is statically known to emit a 5990 # partition field different from the union's. CustomPartitionRouter children are opaque 5991 # and can only be validated at runtime. 5992 for child_model in model.partition_routers: 5993 child_partition_fields: List[str] = [] 5994 if isinstance(child_model, ListPartitionRouterModel): 5995 child_partition_fields.append( 5996 InterpolatedString.create( 5997 child_model.cursor_field, parameters=child_model.parameters or {} 5998 ).eval(config) 5999 ) 6000 elif isinstance(child_model, SubstreamPartitionRouterModel): 6001 for parent_stream_config in child_model.parent_stream_configs: 6002 child_partition_fields.append( 6003 InterpolatedString.create( 6004 parent_stream_config.partition_field, 6005 parameters=parent_stream_config.parameters 6006 or child_model.parameters 6007 or {}, 6008 ).eval(config) 6009 ) 6010 elif isinstance(child_model, UnionPartitionRouterModel): 6011 child_partition_fields.append( 6012 InterpolatedString.create( 6013 child_model.partition_field, parameters=child_model.parameters or {} 6014 ).eval(config) 6015 ) 6016 for child_partition_field in child_partition_fields: 6017 if child_partition_field != partition_field: 6018 raise ValueError( 6019 f"UnionPartitionRouter expects all child partition routers to emit the " 6020 f"partition field '{partition_field}', but a " 6021 f"{child_model.type} child emits '{child_partition_field}'." 6022 ) 6023 6024 # A union slice comes from exactly one child partition router, so request options 6025 # declared on children cannot be applied consistently to requests built from the 6026 # normalized union slices. Partition values should be consumed via interpolation 6027 # (e.g. stream_partition) instead. Note that this validation only covers built-in 6028 # router types; CustomPartitionRouter children are opaque, so any request options 6029 # they implement internally cannot be detected or rejected here. 6030 for router in partition_routers: 6031 if isinstance(router, SubstreamPartitionRouter): 6032 if any( 6033 parent_config.request_option for parent_config in router.parent_stream_configs 6034 ): 6035 raise ValueError("Request options are not supported for UnionPartitionRouter.") 6036 if isinstance(router, ListPartitionRouter) and router.request_option: 6037 raise ValueError("Request options are not supported for UnionPartitionRouter.") 6038 6039 return UnionPartitionRouter( 6040 partition_routers=partition_routers, 6041 partition_field=partition_field, 6042 parameters=model.parameters or {}, 6043 ) 6044 6045 def _ensure_query_properties_to_model( 6046 self, requester: Union[HttpRequesterModel, CustomRequesterModel] 6047 ) -> None: 6048 """ 6049 For some reason, it seems like CustomRequesterModel request_parameters stays as dictionaries which means that 6050 the other conditions relying on it being QueryPropertiesModel instead of a dict fail. Here, we migrate them to 6051 proper model. 6052 """ 6053 if not hasattr(requester, "request_parameters"): 6054 return 6055 6056 request_parameters = requester.request_parameters 6057 if request_parameters and isinstance(request_parameters, Dict): 6058 for request_parameter_key in request_parameters.keys(): 6059 request_parameter = request_parameters[request_parameter_key] 6060 if ( 6061 isinstance(request_parameter, Dict) 6062 and request_parameter.get("type") == "QueryProperties" 6063 ): 6064 request_parameters[request_parameter_key] = QueryPropertiesModel.parse_obj( 6065 request_parameter 6066 ) 6067 6068 def _get_catalog_defined_cursor_field( 6069 self, stream_name: str, allow_catalog_defined_cursor_field: bool 6070 ) -> Optional[CursorField]: 6071 if not allow_catalog_defined_cursor_field: 6072 return None 6073 6074 configured_stream = self._stream_name_to_configured_stream.get(stream_name) 6075 6076 # Depending on the operation is being performed, there may not be a configured stream yet. In this 6077 # case we return None which will then use the default cursor field defined on the cursor model. 6078 # We also treat cursor_field: [""] (list with empty string) as no cursor field, since this can 6079 # occur when the platform serializes "no cursor configured" streams incorrectly. 6080 if ( 6081 not configured_stream 6082 or not configured_stream.cursor_field 6083 or not configured_stream.cursor_field[0] 6084 ): 6085 return None 6086 elif len(configured_stream.cursor_field) > 1: 6087 raise ValueError( 6088 f"The `{stream_name}` stream does not support nested cursor_field. Please specify only a single cursor_field for the stream in the configured catalog." 6089 ) 6090 else: 6091 return CursorField( 6092 cursor_field_key=configured_stream.cursor_field[0], 6093 supports_catalog_defined_cursor_field=allow_catalog_defined_cursor_field, 6094 )
ComponentDefinition =
typing.Mapping[str, typing.Any]
SCHEMA_TRANSFORMER_TYPE_MAPPING =
{<SchemaNormalization.None_: 'None'>: <TransformConfig.NoTransform: 1>, <SchemaNormalization.Default: 'Default'>: <TransformConfig.DefaultSchemaNormalization: 2>}
MAX_SLICES =
5
LOGGER =
<Logger airbyte.model_to_component_factory (INFO)>
class
ModelToComponentFactory:
761class ModelToComponentFactory: 762 EPOCH_DATETIME_FORMAT = "%s" 763 764 def __init__( 765 self, 766 limit_pages_fetched_per_slice: Optional[int] = None, 767 limit_slices_fetched: Optional[int] = None, 768 emit_connector_builder_messages: bool = False, 769 disable_retries: bool = False, 770 disable_cache: bool = False, 771 message_repository: Optional[MessageRepository] = None, 772 connector_state_manager: Optional[ConnectorStateManager] = None, 773 max_concurrent_async_job_count: Optional[int] = None, 774 configured_catalog: Optional[ConfiguredAirbyteCatalog] = None, 775 api_budget: Optional[APIBudget] = None, 776 rate_limited_authenticators: Optional[ 777 Dict[str, RateLimitedMultipleTokenAuthenticator] 778 ] = None, 779 custom_components_trusted: bool = True, 780 ): 781 self._init_mappings() 782 self._custom_components_trusted = custom_components_trusted 783 self._limit_pages_fetched_per_slice = limit_pages_fetched_per_slice 784 self._limit_slices_fetched = limit_slices_fetched 785 self._emit_connector_builder_messages = emit_connector_builder_messages 786 self._disable_retries = disable_retries 787 self._disable_cache = disable_cache 788 self._message_repository = message_repository or InMemoryMessageRepository( 789 self._evaluate_log_level(emit_connector_builder_messages) 790 ) 791 self._stream_name_to_configured_stream = self._create_stream_name_to_configured_stream( 792 configured_catalog 793 ) 794 self._connector_state_manager = connector_state_manager or ConnectorStateManager() 795 self._api_budget: Optional[Union[APIBudget]] = api_budget 796 # Shared instances so all streams see the same token quota counters (like api_budget) 797 self._rate_limited_authenticators: Dict[str, RateLimitedMultipleTokenAuthenticator] = ( 798 rate_limited_authenticators if rate_limited_authenticators is not None else {} 799 ) 800 self._job_tracker: JobTracker = JobTracker(max_concurrent_async_job_count or 1) 801 # placeholder for deprecation warnings 802 self._collected_deprecation_logs: List[ConnectorBuilderLogMessage] = [] 803 804 def _init_mappings(self) -> None: 805 self.PYDANTIC_MODEL_TO_CONSTRUCTOR: Mapping[Type[BaseModel], Callable[..., Any]] = { 806 AddedFieldDefinitionModel: self.create_added_field_definition, 807 AddFieldsModel: self.create_add_fields, 808 ApiKeyAuthenticatorModel: self.create_api_key_authenticator, 809 BasicHttpAuthenticatorModel: self.create_basic_http_authenticator, 810 BearerAuthenticatorModel: self.create_bearer_authenticator, 811 CheckStreamModel: self.create_check_stream, 812 DynamicStreamCheckConfigModel: self.create_dynamic_stream_check_config, 813 CheckDynamicStreamModel: self.create_check_dynamic_stream, 814 CompositeErrorHandlerModel: self.create_composite_error_handler, 815 ConcurrencyLevelModel: self.create_concurrency_level, 816 ConfigMigrationModel: self.create_config_migration, 817 ConfigAddFieldsModel: self.create_config_add_fields, 818 ConfigRemapFieldModel: self.create_config_remap_field, 819 ConfigRemoveFieldsModel: self.create_config_remove_fields, 820 ConstantBackoffStrategyModel: self.create_constant_backoff_strategy, 821 CsvDecoderModel: self.create_csv_decoder, 822 CursorPaginationModel: self.create_cursor_pagination, 823 CustomAuthenticatorModel: self.create_custom_component, 824 CustomBackoffStrategyModel: self.create_custom_component, 825 CustomDecoderModel: self.create_custom_component, 826 CustomErrorHandlerModel: self.create_custom_component, 827 CustomRecordExtractorModel: self.create_custom_component, 828 CustomRecordFilterModel: self.create_custom_component, 829 CustomRequesterModel: self.create_custom_component, 830 CustomRetrieverModel: self.create_custom_component, 831 CustomSchemaLoader: self.create_custom_component, 832 CustomSchemaNormalizationModel: self.create_custom_component, 833 CustomStateMigration: self.create_custom_component, 834 CustomPaginationStrategyModel: self.create_custom_component, 835 CustomPartitionRouterModel: self.create_custom_component, 836 CustomTransformationModel: self.create_custom_component, 837 CustomValidationStrategyModel: self.create_custom_component, 838 CustomConfigTransformationModel: self.create_custom_component, 839 DeclarativeStreamModel: self.create_default_stream, 840 DefaultErrorHandlerModel: self.create_default_error_handler, 841 DefaultPaginatorModel: self.create_default_paginator, 842 CombinedExtractorModel: self.create_combined_extractor, 843 DpathExtractorModel: self.create_dpath_extractor, 844 DpathValidatorModel: self.create_dpath_validator, 845 ResponseToFileExtractorModel: self.create_response_to_file_extractor, 846 ExponentialBackoffStrategyModel: self.create_exponential_backoff_strategy, 847 SessionTokenAuthenticatorModel: self.create_session_token_authenticator, 848 GroupByKeyMergeStrategyModel: self.create_group_by_key, 849 HttpRequesterModel: self.create_http_requester, 850 HttpResponseFilterModel: self.create_http_response_filter, 851 InlineSchemaLoaderModel: self.create_inline_schema_loader, 852 JsonDecoderModel: self.create_json_decoder, 853 JsonItemsDecoderModel: self.create_json_items_decoder, 854 JsonlDecoderModel: self.create_jsonl_decoder, 855 JsonSchemaPropertySelectorModel: self.create_json_schema_property_selector, 856 GzipDecoderModel: self.create_gzip_decoder, 857 KeysToLowerModel: self.create_keys_to_lower_transformation, 858 KeysToSnakeCaseModel: self.create_keys_to_snake_transformation, 859 KeysReplaceModel: self.create_keys_replace_transformation, 860 FlattenFieldsModel: self.create_flatten_fields, 861 DpathFlattenFieldsModel: self.create_dpath_flatten_fields, 862 IterableDecoderModel: self.create_iterable_decoder, 863 XmlDecoderModel: self.create_xml_decoder, 864 JsonFileSchemaLoaderModel: self.create_json_file_schema_loader, 865 DynamicSchemaLoaderModel: self.create_dynamic_schema_loader, 866 SchemaTypeIdentifierModel: self.create_schema_type_identifier, 867 TypesMapModel: self.create_types_map, 868 ComplexFieldTypeModel: self.create_complex_field_type, 869 JwtAuthenticatorModel: self.create_jwt_authenticator, 870 LegacyToPerPartitionStateMigrationModel: self.create_legacy_to_per_partition_state_migration, 871 ListPartitionRouterModel: self.create_list_partition_router, 872 MinMaxDatetimeModel: self.create_min_max_datetime, 873 NoAuthModel: self.create_no_auth, 874 NoPaginationModel: self.create_no_pagination, 875 OAuthAuthenticatorModel: self.create_oauth_authenticator, 876 OffsetIncrementModel: self.create_offset_increment, 877 PageIncrementModel: self.create_page_increment, 878 ParentStreamConfigModel: self.create_parent_stream_config_with_substream_wrapper, 879 PredicateValidatorModel: self.create_predicate_validator, 880 PropertiesFromEndpointModel: self.create_properties_from_endpoint, 881 PropertyChunkingModel: self.create_property_chunking, 882 QueryPropertiesModel: self.create_query_properties, 883 ParentFieldPathModel: self.create_parent_field_path, 884 RecordExpanderModel: self.create_record_expander, 885 RecordFilterModel: self.create_record_filter, 886 RecordSelectorModel: self.create_record_selector, 887 RemoveFieldsModel: self.create_remove_fields, 888 RequestPathModel: self.create_request_path, 889 RequestOptionModel: self.create_request_option, 890 LegacySessionTokenAuthenticatorModel: self.create_legacy_session_token_authenticator, 891 SelectiveAuthenticatorModel: self.create_selective_authenticator, 892 SimpleRetrieverModel: self.create_simple_retriever, 893 StateDelegatingStreamModel: self.create_state_delegating_stream, 894 SpecModel: self.create_spec, 895 SubstreamPartitionRouterModel: self.create_substream_partition_router, 896 ValidateAdheresToSchemaModel: self.create_validate_adheres_to_schema, 897 WaitTimeFromHeaderModel: self.create_wait_time_from_header, 898 WaitTimeFromResponseModel: self.create_wait_time_from_response, 899 WaitUntilTimeFromHeaderModel: self.create_wait_until_time_from_header, 900 AsyncRetrieverModel: self.create_async_retriever, 901 HttpComponentsResolverModel: self.create_http_components_resolver, 902 ConfigComponentsResolverModel: self.create_config_components_resolver, 903 ParametrizedComponentsResolverModel: self.create_parametrized_components_resolver, 904 StreamConfigModel: self.create_stream_config, 905 ComponentMappingDefinitionModel: self.create_components_mapping_definition, 906 ZipfileDecoderModel: self.create_zipfile_decoder, 907 HTTPAPIBudgetModel: self.create_http_api_budget, 908 FileUploaderModel: self.create_file_uploader, 909 FixedWindowCallRatePolicyModel: self.create_fixed_window_call_rate_policy, 910 MovingWindowCallRatePolicyModel: self.create_moving_window_call_rate_policy, 911 UnlimitedCallRatePolicyModel: self.create_unlimited_call_rate_policy, 912 RateModel: self.create_rate, 913 HttpRequestRegexMatcherModel: self.create_http_request_matcher, 914 RateLimitedMultipleTokenAuthenticatorModel: self.create_rate_limited_multiple_token_authenticator, 915 GroupingPartitionRouterModel: self.create_grouping_partition_router, 916 UnionPartitionRouterModel: self.create_union_partition_router, 917 } 918 919 # Needed for the case where we need to perform a second parse on the fields of a custom component 920 self.TYPE_NAME_TO_MODEL = {cls.__name__: cls for cls in self.PYDANTIC_MODEL_TO_CONSTRUCTOR} 921 922 @staticmethod 923 def _create_stream_name_to_configured_stream( 924 configured_catalog: Optional[ConfiguredAirbyteCatalog], 925 ) -> Mapping[str, ConfiguredAirbyteStream]: 926 return ( 927 {stream.stream.name: stream for stream in configured_catalog.streams} 928 if configured_catalog 929 else {} 930 ) 931 932 def create_component( 933 self, 934 model_type: Type[BaseModel], 935 component_definition: ComponentDefinition, 936 config: Config, 937 **kwargs: Any, 938 ) -> Any: 939 """ 940 Takes a given Pydantic model type and Mapping representing a component definition and creates a declarative component and 941 subcomponents which will be used at runtime. This is done by first parsing the mapping into a Pydantic model and then creating 942 creating declarative components from that model. 943 944 :param model_type: The type of declarative component that is being initialized 945 :param component_definition: The mapping that represents a declarative component 946 :param config: The connector config that is provided by the customer 947 :return: The declarative component to be used at runtime 948 """ 949 950 component_type = component_definition.get("type") 951 if component_definition.get("type") != model_type.__name__: 952 raise ValueError( 953 f"Expected manifest component of type {model_type.__name__}, but received {component_type} instead" 954 ) 955 956 declarative_component_model = model_type.parse_obj(component_definition) 957 958 if not isinstance(declarative_component_model, model_type): 959 raise ValueError( 960 f"Expected {model_type.__name__} component, but received {declarative_component_model.__class__.__name__}" 961 ) 962 963 return self._create_component_from_model( 964 model=declarative_component_model, config=config, **kwargs 965 ) 966 967 def _create_component_from_model(self, model: BaseModel, config: Config, **kwargs: Any) -> Any: 968 if model.__class__ not in self.PYDANTIC_MODEL_TO_CONSTRUCTOR: 969 raise ValueError( 970 f"{model.__class__} with attributes {model} is not a valid component type" 971 ) 972 component_constructor = self.PYDANTIC_MODEL_TO_CONSTRUCTOR.get(model.__class__) 973 if not component_constructor: 974 raise ValueError(f"Could not find constructor for {model.__class__}") 975 976 # collect deprecation warnings for supported models. 977 if isinstance(model, BaseModelWithDeprecations): 978 self._collect_model_deprecations(model) 979 980 return component_constructor(model=model, config=config, **kwargs) 981 982 def get_model_deprecations(self) -> List[ConnectorBuilderLogMessage]: 983 """ 984 Returns the deprecation warnings that were collected during the creation of components. 985 """ 986 return self._collected_deprecation_logs 987 988 def _collect_model_deprecations(self, model: BaseModelWithDeprecations) -> None: 989 """ 990 Collects deprecation logs from the given model and appends any new logs to the internal collection. 991 992 This method checks if the provided model has deprecation logs (identified by the presence of the DEPRECATION_LOGS_TAG attribute and a non-None `_deprecation_logs` property). It iterates through each deprecation log in the model and appends it to the `_collected_deprecation_logs` list if it has not already been collected, ensuring that duplicate logs are avoided. 993 994 Args: 995 model (BaseModelWithDeprecations): The model instance from which to collect deprecation logs. 996 """ 997 if hasattr(model, DEPRECATION_LOGS_TAG) and model._deprecation_logs is not None: 998 for log in model._deprecation_logs: 999 # avoid duplicates for deprecation logs observed. 1000 if log not in self._collected_deprecation_logs: 1001 self._collected_deprecation_logs.append(log) 1002 1003 def create_config_migration( 1004 self, model: ConfigMigrationModel, config: Config 1005 ) -> ConfigMigration: 1006 transformations: List[ConfigTransformation] = [ 1007 self._create_component_from_model(transformation, config) 1008 for transformation in model.transformations 1009 ] 1010 1011 return ConfigMigration( 1012 description=model.description, 1013 transformations=transformations, 1014 ) 1015 1016 def create_config_add_fields( 1017 self, model: ConfigAddFieldsModel, config: Config, **kwargs: Any 1018 ) -> ConfigAddFields: 1019 fields = [self._create_component_from_model(field, config) for field in model.fields] 1020 return ConfigAddFields( 1021 fields=fields, 1022 condition=model.condition or "", 1023 ) 1024 1025 @staticmethod 1026 def create_config_remove_fields( 1027 model: ConfigRemoveFieldsModel, config: Config, **kwargs: Any 1028 ) -> ConfigRemoveFields: 1029 return ConfigRemoveFields( 1030 field_pointers=model.field_pointers, 1031 condition=model.condition or "", 1032 ) 1033 1034 @staticmethod 1035 def create_config_remap_field( 1036 model: ConfigRemapFieldModel, config: Config, **kwargs: Any 1037 ) -> ConfigRemapField: 1038 mapping = cast(Mapping[str, Any], model.map) 1039 return ConfigRemapField( 1040 map=mapping, 1041 field_path=model.field_path, 1042 config=config, 1043 ) 1044 1045 def create_dpath_validator(self, model: DpathValidatorModel, config: Config) -> DpathValidator: 1046 strategy = self._create_component_from_model(model.validation_strategy, config) 1047 1048 return DpathValidator( 1049 field_path=model.field_path, 1050 strategy=strategy, 1051 ) 1052 1053 def create_predicate_validator( 1054 self, model: PredicateValidatorModel, config: Config 1055 ) -> PredicateValidator: 1056 strategy = self._create_component_from_model(model.validation_strategy, config) 1057 1058 return PredicateValidator( 1059 value=model.value, 1060 strategy=strategy, 1061 ) 1062 1063 @staticmethod 1064 def create_validate_adheres_to_schema( 1065 model: ValidateAdheresToSchemaModel, config: Config, **kwargs: Any 1066 ) -> ValidateAdheresToSchema: 1067 base_schema = cast(Mapping[str, Any], model.base_schema) 1068 return ValidateAdheresToSchema( 1069 schema=base_schema, 1070 ) 1071 1072 @staticmethod 1073 def create_added_field_definition( 1074 model: AddedFieldDefinitionModel, config: Config, **kwargs: Any 1075 ) -> AddedFieldDefinition: 1076 interpolated_value = InterpolatedString.create( 1077 model.value, parameters=model.parameters or {} 1078 ) 1079 return AddedFieldDefinition( 1080 path=model.path, 1081 value=interpolated_value, 1082 value_type=ModelToComponentFactory._json_schema_type_name_to_type(model.value_type), 1083 parameters=model.parameters or {}, 1084 ) 1085 1086 def create_add_fields(self, model: AddFieldsModel, config: Config, **kwargs: Any) -> AddFields: 1087 added_field_definitions = [ 1088 self._create_component_from_model( 1089 model=added_field_definition_model, 1090 value_type=ModelToComponentFactory._json_schema_type_name_to_type( 1091 added_field_definition_model.value_type 1092 ), 1093 config=config, 1094 ) 1095 for added_field_definition_model in model.fields 1096 ] 1097 return AddFields( 1098 fields=added_field_definitions, 1099 condition=model.condition or "", 1100 parameters=model.parameters or {}, 1101 ) 1102 1103 def create_keys_to_lower_transformation( 1104 self, model: KeysToLowerModel, config: Config, **kwargs: Any 1105 ) -> KeysToLowerTransformation: 1106 return KeysToLowerTransformation() 1107 1108 def create_keys_to_snake_transformation( 1109 self, model: KeysToSnakeCaseModel, config: Config, **kwargs: Any 1110 ) -> KeysToSnakeCaseTransformation: 1111 return KeysToSnakeCaseTransformation() 1112 1113 def create_keys_replace_transformation( 1114 self, model: KeysReplaceModel, config: Config, **kwargs: Any 1115 ) -> KeysReplaceTransformation: 1116 return KeysReplaceTransformation( 1117 old=model.old, new=model.new, parameters=model.parameters or {} 1118 ) 1119 1120 def create_flatten_fields( 1121 self, model: FlattenFieldsModel, config: Config, **kwargs: Any 1122 ) -> FlattenFields: 1123 return FlattenFields( 1124 flatten_lists=model.flatten_lists if model.flatten_lists is not None else True 1125 ) 1126 1127 def create_dpath_flatten_fields( 1128 self, model: DpathFlattenFieldsModel, config: Config, **kwargs: Any 1129 ) -> DpathFlattenFields: 1130 model_field_path: List[Union[InterpolatedString, str]] = [x for x in model.field_path] 1131 key_transformation = ( 1132 KeyTransformation( 1133 config=config, 1134 prefix=model.key_transformation.prefix, 1135 suffix=model.key_transformation.suffix, 1136 parameters=model.parameters or {}, 1137 ) 1138 if model.key_transformation is not None 1139 else None 1140 ) 1141 return DpathFlattenFields( 1142 config=config, 1143 field_path=model_field_path, 1144 delete_origin_value=model.delete_origin_value 1145 if model.delete_origin_value is not None 1146 else False, 1147 replace_record=model.replace_record if model.replace_record is not None else False, 1148 key_transformation=key_transformation, 1149 parameters=model.parameters or {}, 1150 ) 1151 1152 @staticmethod 1153 def _json_schema_type_name_to_type(value_type: Optional[ValueType]) -> Optional[Type[Any]]: 1154 if not value_type: 1155 return None 1156 names_to_types = { 1157 ValueType.string: str, 1158 ValueType.number: float, 1159 ValueType.integer: int, 1160 ValueType.boolean: bool, 1161 } 1162 return names_to_types[value_type] 1163 1164 def create_api_key_authenticator( 1165 self, 1166 model: ApiKeyAuthenticatorModel, 1167 config: Config, 1168 token_provider: Optional[TokenProvider] = None, 1169 **kwargs: Any, 1170 ) -> ApiKeyAuthenticator: 1171 if model.inject_into is None and model.header is None: 1172 raise ValueError( 1173 "Expected either inject_into or header to be set for ApiKeyAuthenticator" 1174 ) 1175 1176 if model.inject_into is not None and model.header is not None: 1177 raise ValueError( 1178 "inject_into and header cannot be set both for ApiKeyAuthenticator - remove the deprecated header option" 1179 ) 1180 1181 if token_provider is not None and model.api_token != "": 1182 raise ValueError( 1183 "If token_provider is set, api_token is ignored and has to be set to empty string." 1184 ) 1185 1186 request_option = ( 1187 self._create_component_from_model( 1188 model.inject_into, config, parameters=model.parameters or {} 1189 ) 1190 if model.inject_into 1191 else RequestOption( 1192 inject_into=RequestOptionType.header, 1193 field_name=model.header or "", 1194 parameters=model.parameters or {}, 1195 ) 1196 ) 1197 1198 return ApiKeyAuthenticator( 1199 token_provider=( 1200 token_provider 1201 if token_provider is not None 1202 else InterpolatedStringTokenProvider( 1203 api_token=model.api_token or "", 1204 config=config, 1205 parameters=model.parameters or {}, 1206 ) 1207 ), 1208 request_option=request_option, 1209 config=config, 1210 parameters=model.parameters or {}, 1211 ) 1212 1213 def create_legacy_to_per_partition_state_migration( 1214 self, 1215 model: LegacyToPerPartitionStateMigrationModel, 1216 config: Mapping[str, Any], 1217 declarative_stream: DeclarativeStreamModel, 1218 ) -> LegacyToPerPartitionStateMigration: 1219 retriever = declarative_stream.retriever 1220 if not isinstance(retriever, (SimpleRetrieverModel, AsyncRetrieverModel)): 1221 raise ValueError( 1222 f"LegacyToPerPartitionStateMigrations can only be applied on a DeclarativeStream with a SimpleRetriever or AsyncRetriever. Got {type(retriever)}" 1223 ) 1224 partition_router = retriever.partition_router 1225 if not isinstance( 1226 partition_router, 1227 ( 1228 SubstreamPartitionRouterModel, 1229 CustomPartitionRouterModel, 1230 UnionPartitionRouterModel, 1231 ), 1232 ): 1233 raise ValueError( 1234 f"LegacyToPerPartitionStateMigrations can only be applied on a SimpleRetriever with a SubstreamPartitionRouter, UnionPartitionRouter or CustomPartitionRouter. Got {type(partition_router)}" 1235 ) 1236 if not isinstance(partition_router, UnionPartitionRouterModel) and not hasattr( 1237 partition_router, "parent_stream_configs" 1238 ): 1239 raise ValueError( 1240 "LegacyToPerPartitionStateMigrations can only be applied with a parent stream configuration." 1241 ) 1242 1243 if not hasattr(declarative_stream, "incremental_sync"): 1244 raise ValueError( 1245 "LegacyToPerPartitionStateMigrations can only be applied with an incremental_sync configuration." 1246 ) 1247 1248 return LegacyToPerPartitionStateMigration( 1249 partition_router, # type: ignore # was already checked above 1250 declarative_stream.incremental_sync, # type: ignore # was already checked. Migration can be applied only to incremental streams. 1251 config, 1252 declarative_stream.parameters, # type: ignore # different type is expected here Mapping[str, Any], got Dict[str, Any] 1253 ) 1254 1255 def create_session_token_authenticator( 1256 self, model: SessionTokenAuthenticatorModel, config: Config, *, name: str, **kwargs: Any 1257 ) -> Union[ApiKeyAuthenticator, BearerAuthenticator]: 1258 self._reject_reduce_page_size_action( 1259 model.login_requester, f"`login_requester` of the SessionTokenAuthenticator of {name}" 1260 ) 1261 self._reject_split_request_window_action( 1262 model.login_requester, f"`login_requester` of the SessionTokenAuthenticator of {name}" 1263 ) 1264 decoder = ( 1265 self._create_component_from_model(model=model.decoder, config=config) 1266 if model.decoder 1267 else JsonDecoder(parameters={}) 1268 ) 1269 login_requester = self._create_component_from_model( 1270 model=model.login_requester, 1271 config=config, 1272 name=f"{name}_login_requester", 1273 decoder=decoder, 1274 ) 1275 token_provider = SessionTokenProvider( 1276 login_requester=login_requester, 1277 session_token_path=model.session_token_path, 1278 expiration_duration=parse_duration(model.expiration_duration) 1279 if model.expiration_duration 1280 else None, 1281 parameters=model.parameters or {}, 1282 message_repository=self._message_repository, 1283 decoder=decoder, 1284 ) 1285 if model.request_authentication.type == "Bearer": 1286 return ModelToComponentFactory.create_bearer_authenticator( 1287 BearerAuthenticatorModel(type="BearerAuthenticator", api_token=""), # type: ignore # $parameters has a default value 1288 config, 1289 token_provider=token_provider, 1290 ) 1291 else: 1292 # Get the api_token template if specified, default to just the session token 1293 api_token_template = ( 1294 getattr(model.request_authentication, "api_token", None) or "{{ session_token }}" 1295 ) 1296 final_token_provider: TokenProvider = InterpolatedSessionTokenProvider( 1297 config=config, 1298 api_token=api_token_template, 1299 session_token_provider=token_provider, 1300 parameters=model.parameters or {}, 1301 ) 1302 return self.create_api_key_authenticator( 1303 ApiKeyAuthenticatorModel( 1304 type="ApiKeyAuthenticator", 1305 api_token="", 1306 inject_into=model.request_authentication.inject_into, 1307 ), # type: ignore # $parameters and headers default to None 1308 config=config, 1309 token_provider=final_token_provider, 1310 ) 1311 1312 @staticmethod 1313 def create_basic_http_authenticator( 1314 model: BasicHttpAuthenticatorModel, config: Config, **kwargs: Any 1315 ) -> BasicHttpAuthenticator: 1316 return BasicHttpAuthenticator( 1317 password=model.password or "", 1318 username=model.username, 1319 config=config, 1320 parameters=model.parameters or {}, 1321 ) 1322 1323 @staticmethod 1324 def create_bearer_authenticator( 1325 model: BearerAuthenticatorModel, 1326 config: Config, 1327 token_provider: Optional[TokenProvider] = None, 1328 **kwargs: Any, 1329 ) -> BearerAuthenticator: 1330 if token_provider is not None and model.api_token != "": 1331 raise ValueError( 1332 "If token_provider is set, api_token is ignored and has to be set to empty string." 1333 ) 1334 return BearerAuthenticator( 1335 token_provider=( 1336 token_provider 1337 if token_provider is not None 1338 else InterpolatedStringTokenProvider( 1339 api_token=model.api_token or "", 1340 config=config, 1341 parameters=model.parameters or {}, 1342 ) 1343 ), 1344 config=config, 1345 parameters=model.parameters or {}, 1346 ) 1347 1348 @staticmethod 1349 def create_dynamic_stream_check_config( 1350 model: DynamicStreamCheckConfigModel, config: Config, **kwargs: Any 1351 ) -> DynamicStreamCheckConfig: 1352 return DynamicStreamCheckConfig( 1353 dynamic_stream_name=model.dynamic_stream_name, 1354 stream_count=model.stream_count, 1355 ) 1356 1357 def create_check_stream( 1358 self, model: CheckStreamModel, config: Config, **kwargs: Any 1359 ) -> CheckStream: 1360 if model.dynamic_streams_check_configs is None and model.stream_names is None: 1361 raise ValueError( 1362 "Expected either stream_names or dynamic_streams_check_configs to be set for CheckStream" 1363 ) 1364 1365 dynamic_streams_check_configs = ( 1366 [ 1367 self._create_component_from_model(model=dynamic_stream_check_config, config=config) 1368 for dynamic_stream_check_config in model.dynamic_streams_check_configs 1369 ] 1370 if model.dynamic_streams_check_configs 1371 else [] 1372 ) 1373 1374 # `model.config_overrides` is deliberately not read here. The source applies it around the whole 1375 # check operation (`ConcurrentDeclarativeSource._config_overridden_for_check`), which is what makes 1376 # it work for every checker type rather than only this one. Do not wire it in a second time. 1377 return CheckStream( 1378 stream_names=model.stream_names or [], 1379 dynamic_streams_check_configs=dynamic_streams_check_configs, 1380 parameters={}, 1381 ) 1382 1383 @staticmethod 1384 def create_check_dynamic_stream( 1385 model: CheckDynamicStreamModel, config: Config, **kwargs: Any 1386 ) -> CheckDynamicStream: 1387 assert model.use_check_availability is not None # for mypy 1388 1389 use_check_availability = model.use_check_availability 1390 1391 # See `create_check_stream`: `model.config_overrides` is applied by the source, not here. 1392 return CheckDynamicStream( 1393 stream_count=model.stream_count, 1394 use_check_availability=use_check_availability, 1395 parameters={}, 1396 ) 1397 1398 def create_composite_error_handler( 1399 self, model: CompositeErrorHandlerModel, config: Config, **kwargs: Any 1400 ) -> CompositeErrorHandler: 1401 error_handlers = [ 1402 self._create_component_from_model(model=error_handler_model, config=config) 1403 for error_handler_model in model.error_handlers 1404 ] 1405 return CompositeErrorHandler( 1406 error_handlers=error_handlers, parameters=model.parameters or {} 1407 ) 1408 1409 @staticmethod 1410 def create_concurrency_level( 1411 model: ConcurrencyLevelModel, config: Config, **kwargs: Any 1412 ) -> ConcurrencyLevel: 1413 return ConcurrencyLevel( 1414 default_concurrency=model.default_concurrency, 1415 max_concurrency=model.max_concurrency, 1416 config=config, 1417 parameters={}, 1418 ) 1419 1420 @staticmethod 1421 def apply_stream_state_migrations( 1422 stream_state_migrations: List[Any] | None, stream_state: MutableMapping[str, Any] 1423 ) -> MutableMapping[str, Any]: 1424 if stream_state_migrations: 1425 for state_migration in stream_state_migrations: 1426 if state_migration.should_migrate(stream_state): 1427 # The state variable is expected to be mutable but the migrate method returns an immutable mapping. 1428 stream_state = dict(state_migration.migrate(stream_state)) 1429 return stream_state 1430 1431 def create_concurrent_cursor_from_datetime_based_cursor( 1432 self, 1433 model_type: Type[BaseModel], 1434 component_definition: ComponentDefinition, 1435 stream_name: str, 1436 stream_namespace: Optional[str], 1437 stream_state: MutableMapping[str, Any], 1438 config: Config, 1439 message_repository: Optional[MessageRepository] = None, 1440 runtime_lookback_window: Optional[datetime.timedelta] = None, 1441 **kwargs: Any, 1442 ) -> ConcurrentCursor: 1443 component_type = component_definition.get("type") 1444 if component_definition.get("type") != model_type.__name__: 1445 raise ValueError( 1446 f"Expected manifest component of type {model_type.__name__}, but received {component_type} instead" 1447 ) 1448 1449 # FIXME the interfaces of the concurrent cursor are kind of annoying as they take a `ComponentDefinition` instead of the actual model. This was done because the ConcurrentDeclarativeSource didn't have access to the models [here for example](https://github.com/airbytehq/airbyte-python-cdk/blob/f525803b3fec9329e4cc8478996a92bf884bfde9/airbyte_cdk/sources/declarative/concurrent_declarative_source.py#L354C54-L354C91). So now we have two cases: 1450 # * The ComponentDefinition comes from model.__dict__ in which case we have `parameters` 1451 # * The ComponentDefinition comes from the manifest as a dict in which case we have `$parameters` 1452 # We should change those interfaces to use the model once we clean up the code in CDS at which point the parameter propagation should happen as part of the ModelToComponentFactory. 1453 if "$parameters" not in component_definition and "parameters" in component_definition: 1454 component_definition["$parameters"] = component_definition.get("parameters") # type: ignore # This is a dict 1455 datetime_based_cursor_model = model_type.parse_obj(component_definition) 1456 1457 if not isinstance(datetime_based_cursor_model, DatetimeBasedCursorModel): 1458 raise ValueError( 1459 f"Expected {model_type.__name__} component, but received {datetime_based_cursor_model.__class__.__name__}" 1460 ) 1461 1462 model_parameters = datetime_based_cursor_model.parameters or {} 1463 1464 cursor_field = self._get_catalog_defined_cursor_field( 1465 stream_name=stream_name, 1466 allow_catalog_defined_cursor_field=datetime_based_cursor_model.allow_catalog_defined_cursor_field 1467 or False, 1468 ) 1469 1470 if not cursor_field: 1471 interpolated_cursor_field = InterpolatedString.create( 1472 datetime_based_cursor_model.cursor_field, 1473 parameters=model_parameters, 1474 ) 1475 cursor_field = CursorField( 1476 cursor_field_key=interpolated_cursor_field.eval(config=config), 1477 supports_catalog_defined_cursor_field=datetime_based_cursor_model.allow_catalog_defined_cursor_field 1478 or False, 1479 ) 1480 1481 interpolated_partition_field_start = InterpolatedString.create( 1482 datetime_based_cursor_model.partition_field_start or "start_time", 1483 parameters=model_parameters, 1484 ) 1485 interpolated_partition_field_end = InterpolatedString.create( 1486 datetime_based_cursor_model.partition_field_end or "end_time", 1487 parameters=model_parameters, 1488 ) 1489 1490 slice_boundary_fields = ( 1491 interpolated_partition_field_start.eval(config=config), 1492 interpolated_partition_field_end.eval(config=config), 1493 ) 1494 1495 datetime_format = datetime_based_cursor_model.datetime_format 1496 1497 cursor_granularity = ( 1498 parse_duration(datetime_based_cursor_model.cursor_granularity) 1499 if datetime_based_cursor_model.cursor_granularity 1500 else None 1501 ) 1502 1503 lookback_window = None 1504 interpolated_lookback_window = ( 1505 InterpolatedString.create( 1506 datetime_based_cursor_model.lookback_window, 1507 parameters=model_parameters, 1508 ) 1509 if datetime_based_cursor_model.lookback_window 1510 else None 1511 ) 1512 if interpolated_lookback_window: 1513 evaluated_lookback_window = interpolated_lookback_window.eval(config=config) 1514 if evaluated_lookback_window: 1515 lookback_window = parse_duration(evaluated_lookback_window) 1516 1517 connector_state_converter: DateTimeStreamStateConverter 1518 connector_state_converter = CustomFormatConcurrentStreamStateConverter( 1519 datetime_format=datetime_format, 1520 input_datetime_formats=datetime_based_cursor_model.cursor_datetime_formats, 1521 is_sequential_state=True, # ConcurrentPerPartitionCursor only works with sequential state 1522 cursor_granularity=cursor_granularity, 1523 ) 1524 1525 # Adjusts the stream state by applying the runtime lookback window. 1526 # This is used to ensure correct state handling in case of failed partitions. 1527 stream_state_value = stream_state.get(cursor_field.cursor_field_key) 1528 if runtime_lookback_window and stream_state_value: 1529 new_stream_state = ( 1530 connector_state_converter.parse_timestamp(stream_state_value) 1531 - runtime_lookback_window 1532 ) 1533 stream_state[cursor_field.cursor_field_key] = connector_state_converter.output_format( 1534 new_stream_state 1535 ) 1536 1537 start_date_runtime_value: Union[InterpolatedString, str, MinMaxDatetime] 1538 if isinstance(datetime_based_cursor_model.start_datetime, MinMaxDatetimeModel): 1539 start_date_runtime_value = self.create_min_max_datetime( 1540 model=datetime_based_cursor_model.start_datetime, config=config 1541 ) 1542 else: 1543 start_date_runtime_value = datetime_based_cursor_model.start_datetime 1544 1545 end_date_runtime_value: Optional[Union[InterpolatedString, str, MinMaxDatetime]] 1546 if isinstance(datetime_based_cursor_model.end_datetime, MinMaxDatetimeModel): 1547 end_date_runtime_value = self.create_min_max_datetime( 1548 model=datetime_based_cursor_model.end_datetime, config=config 1549 ) 1550 else: 1551 end_date_runtime_value = datetime_based_cursor_model.end_datetime 1552 1553 interpolated_start_date = MinMaxDatetime.create( 1554 interpolated_string_or_min_max_datetime=start_date_runtime_value, 1555 parameters=datetime_based_cursor_model.parameters, 1556 ) 1557 interpolated_end_date = ( 1558 None 1559 if not end_date_runtime_value 1560 else MinMaxDatetime.create( 1561 end_date_runtime_value, datetime_based_cursor_model.parameters 1562 ) 1563 ) 1564 1565 # If datetime format is not specified then start/end datetime should inherit it from the stream slicer 1566 if not interpolated_start_date.datetime_format: 1567 interpolated_start_date.datetime_format = datetime_format 1568 if interpolated_end_date and not interpolated_end_date.datetime_format: 1569 interpolated_end_date.datetime_format = datetime_format 1570 1571 start_date = interpolated_start_date.get_datetime(config=config) 1572 end_date_provider = ( 1573 partial(interpolated_end_date.get_datetime, config) 1574 if interpolated_end_date 1575 else connector_state_converter.get_end_provider() 1576 ) 1577 1578 if ( 1579 datetime_based_cursor_model.step and not datetime_based_cursor_model.cursor_granularity 1580 ) or ( 1581 not datetime_based_cursor_model.step and datetime_based_cursor_model.cursor_granularity 1582 ): 1583 raise ValueError( 1584 f"If step is defined, cursor_granularity should be as well and vice-versa. " 1585 f"Right now, step is `{datetime_based_cursor_model.step}` and cursor_granularity is `{datetime_based_cursor_model.cursor_granularity}`" 1586 ) 1587 1588 # When step is not defined, default to a step size from the starting date to the present moment 1589 step_length = datetime.timedelta.max 1590 interpolated_step = ( 1591 InterpolatedString.create( 1592 datetime_based_cursor_model.step, 1593 parameters=model_parameters, 1594 ) 1595 if datetime_based_cursor_model.step 1596 else None 1597 ) 1598 if interpolated_step: 1599 evaluated_step = interpolated_step.eval(config) 1600 if evaluated_step: 1601 step_length = parse_duration(evaluated_step) 1602 1603 clamping_strategy: ClampingStrategy = NoClamping() 1604 if datetime_based_cursor_model.clamping: 1605 # While it is undesirable to interpolate within the model factory (as opposed to at runtime), 1606 # it is still better than shifting interpolation low-code concept into the ConcurrentCursor runtime 1607 # object which we want to keep agnostic of being low-code 1608 target = InterpolatedString( 1609 string=datetime_based_cursor_model.clamping.target, 1610 parameters=model_parameters, 1611 ) 1612 evaluated_target = target.eval(config=config) 1613 match evaluated_target: 1614 case "DAY": 1615 clamping_strategy = DayClampingStrategy() 1616 end_date_provider = ClampingEndProvider( 1617 DayClampingStrategy(is_ceiling=False), 1618 end_date_provider, # type: ignore # Having issues w/ inspection for GapType and CursorValueType as shown in existing tests. Confirmed functionality is working in practice 1619 granularity=cursor_granularity or datetime.timedelta(seconds=1), 1620 ) 1621 case "WEEK": 1622 if ( 1623 not datetime_based_cursor_model.clamping.target_details 1624 or "weekday" not in datetime_based_cursor_model.clamping.target_details 1625 ): 1626 raise ValueError( 1627 "Given WEEK clamping, weekday needs to be provided as target_details" 1628 ) 1629 weekday = self._assemble_weekday( 1630 datetime_based_cursor_model.clamping.target_details["weekday"] 1631 ) 1632 clamping_strategy = WeekClampingStrategy(weekday) 1633 end_date_provider = ClampingEndProvider( 1634 WeekClampingStrategy(weekday, is_ceiling=False), 1635 end_date_provider, # type: ignore # Having issues w/ inspection for GapType and CursorValueType as shown in existing tests. Confirmed functionality is working in practice 1636 granularity=cursor_granularity or datetime.timedelta(days=1), 1637 ) 1638 case "MONTH": 1639 clamping_strategy = MonthClampingStrategy() 1640 end_date_provider = ClampingEndProvider( 1641 MonthClampingStrategy(is_ceiling=False), 1642 end_date_provider, # type: ignore # Having issues w/ inspection for GapType and CursorValueType as shown in existing tests. Confirmed functionality is working in practice 1643 granularity=cursor_granularity or datetime.timedelta(days=1), 1644 ) 1645 case _: 1646 raise ValueError( 1647 f"Invalid clamping target {evaluated_target}, expected DAY, WEEK, MONTH" 1648 ) 1649 1650 return ConcurrentCursor( 1651 stream_name=stream_name, 1652 stream_namespace=stream_namespace, 1653 stream_state=stream_state, 1654 message_repository=message_repository or self._message_repository, 1655 connector_state_manager=self._connector_state_manager, 1656 connector_state_converter=connector_state_converter, 1657 cursor_field=cursor_field, 1658 slice_boundary_fields=slice_boundary_fields, 1659 start=start_date, # type: ignore # Having issues w/ inspection for GapType and CursorValueType as shown in existing tests. Confirmed functionality is working in practice 1660 end_provider=end_date_provider, # type: ignore # Having issues w/ inspection for GapType and CursorValueType as shown in existing tests. Confirmed functionality is working in practice 1661 lookback_window=lookback_window, 1662 slice_range=step_length, 1663 cursor_granularity=cursor_granularity, 1664 clamping_strategy=clamping_strategy, 1665 is_compare_strictly=datetime_based_cursor_model.is_compare_strictly or False, 1666 ) 1667 1668 def create_concurrent_cursor_from_incrementing_count_cursor( 1669 self, 1670 model_type: Type[BaseModel], 1671 component_definition: ComponentDefinition, 1672 stream_name: str, 1673 stream_namespace: Optional[str], 1674 stream_state: MutableMapping[str, Any], 1675 config: Config, 1676 message_repository: Optional[MessageRepository] = None, 1677 **kwargs: Any, 1678 ) -> ConcurrentCursor: 1679 component_type = component_definition.get("type") 1680 if component_definition.get("type") != model_type.__name__: 1681 raise ValueError( 1682 f"Expected manifest component of type {model_type.__name__}, but received {component_type} instead" 1683 ) 1684 1685 incrementing_count_cursor_model = model_type.parse_obj(component_definition) 1686 1687 if not isinstance(incrementing_count_cursor_model, IncrementingCountCursorModel): 1688 raise ValueError( 1689 f"Expected {model_type.__name__} component, but received {incrementing_count_cursor_model.__class__.__name__}" 1690 ) 1691 1692 start_value: Union[int, str, None] = incrementing_count_cursor_model.start_value 1693 # Pydantic Union type coercion can convert int 0 to string '0' depending on Union order. 1694 # We need to handle both int and str representations of numeric values. 1695 # Evaluate the InterpolatedString and convert to int for the ConcurrentCursor. 1696 if start_value is not None: 1697 interpolated_start_value = InterpolatedString.create( 1698 str(start_value), # Ensure we pass a string to InterpolatedString.create 1699 parameters=incrementing_count_cursor_model.parameters or {}, 1700 ) 1701 evaluated_start_value: int = int(interpolated_start_value.eval(config=config)) 1702 else: 1703 evaluated_start_value = 0 1704 1705 cursor_field = self._get_catalog_defined_cursor_field( 1706 stream_name=stream_name, 1707 allow_catalog_defined_cursor_field=incrementing_count_cursor_model.allow_catalog_defined_cursor_field 1708 or False, 1709 ) 1710 1711 if not cursor_field: 1712 interpolated_cursor_field = InterpolatedString.create( 1713 incrementing_count_cursor_model.cursor_field, 1714 parameters=incrementing_count_cursor_model.parameters or {}, 1715 ) 1716 cursor_field = CursorField( 1717 cursor_field_key=interpolated_cursor_field.eval(config=config), 1718 supports_catalog_defined_cursor_field=incrementing_count_cursor_model.allow_catalog_defined_cursor_field 1719 or False, 1720 ) 1721 1722 connector_state_converter = IncrementingCountStreamStateConverter( 1723 is_sequential_state=True, # ConcurrentPerPartitionCursor only works with sequential state 1724 ) 1725 1726 return ConcurrentCursor( 1727 stream_name=stream_name, 1728 stream_namespace=stream_namespace, 1729 stream_state=stream_state, 1730 message_repository=message_repository or self._message_repository, 1731 connector_state_manager=self._connector_state_manager, 1732 connector_state_converter=connector_state_converter, 1733 cursor_field=cursor_field, 1734 slice_boundary_fields=None, 1735 start=evaluated_start_value, # type: ignore # Having issues w/ inspection for GapType and CursorValueType as shown in existing tests. Confirmed functionality is working in practice 1736 end_provider=connector_state_converter.get_end_provider(), # type: ignore # Having issues w/ inspection for GapType and CursorValueType as shown in existing tests. Confirmed functionality is working in practice 1737 ) 1738 1739 def _assemble_weekday(self, weekday: str) -> Weekday: 1740 match weekday: 1741 case "MONDAY": 1742 return Weekday.MONDAY 1743 case "TUESDAY": 1744 return Weekday.TUESDAY 1745 case "WEDNESDAY": 1746 return Weekday.WEDNESDAY 1747 case "THURSDAY": 1748 return Weekday.THURSDAY 1749 case "FRIDAY": 1750 return Weekday.FRIDAY 1751 case "SATURDAY": 1752 return Weekday.SATURDAY 1753 case "SUNDAY": 1754 return Weekday.SUNDAY 1755 case _: 1756 raise ValueError(f"Unknown weekday {weekday}") 1757 1758 def create_concurrent_cursor_from_perpartition_cursor( 1759 self, 1760 state_manager: ConnectorStateManager, 1761 model_type: Type[BaseModel], 1762 component_definition: ComponentDefinition, 1763 stream_name: str, 1764 stream_namespace: Optional[str], 1765 config: Config, 1766 stream_state: MutableMapping[str, Any], 1767 partition_router: PartitionRouter, 1768 attempt_to_create_cursor_if_not_provided: bool = False, 1769 **kwargs: Any, 1770 ) -> ConcurrentPerPartitionCursor: 1771 component_type = component_definition.get("type") 1772 if component_definition.get("type") != model_type.__name__: 1773 raise ValueError( 1774 f"Expected manifest component of type {model_type.__name__}, but received {component_type} instead" 1775 ) 1776 1777 # FIXME the interfaces of the concurrent cursor are kind of annoying as they take a `ComponentDefinition` instead of the actual model. This was done because the ConcurrentDeclarativeSource didn't have access to the models [here for example](https://github.com/airbytehq/airbyte-python-cdk/blob/f525803b3fec9329e4cc8478996a92bf884bfde9/airbyte_cdk/sources/declarative/concurrent_declarative_source.py#L354C54-L354C91). So now we have two cases: 1778 # * The ComponentDefinition comes from model.__dict__ in which case we have `parameters` 1779 # * The ComponentDefinition comes from the manifest as a dict in which case we have `$parameters` 1780 # We should change those interfaces to use the model once we clean up the code in CDS at which point the parameter propagation should happen as part of the ModelToComponentFactory. 1781 if "$parameters" not in component_definition and "parameters" in component_definition: 1782 component_definition["$parameters"] = component_definition.get("parameters") # type: ignore # This is a dict 1783 datetime_based_cursor_model = model_type.parse_obj(component_definition) 1784 1785 if not isinstance(datetime_based_cursor_model, DatetimeBasedCursorModel): 1786 raise ValueError( 1787 f"Expected {model_type.__name__} component, but received {datetime_based_cursor_model.__class__.__name__}" 1788 ) 1789 1790 cursor_field = self._get_catalog_defined_cursor_field( 1791 stream_name=stream_name, 1792 allow_catalog_defined_cursor_field=datetime_based_cursor_model.allow_catalog_defined_cursor_field 1793 or False, 1794 ) 1795 1796 if not cursor_field: 1797 interpolated_cursor_field = InterpolatedString.create( 1798 datetime_based_cursor_model.cursor_field, 1799 # FIXME the interfaces of the concurrent cursor are kind of annoying as they take a `ComponentDefinition` instead of the actual model. This was done because the ConcurrentDeclarativeSource didn't have access to the models [here for example](https://github.com/airbytehq/airbyte-python-cdk/blob/f525803b3fec9329e4cc8478996a92bf884bfde9/airbyte_cdk/sources/declarative/concurrent_declarative_source.py#L354C54-L354C91). So now we have two cases: 1800 # * The ComponentDefinition comes from model.__dict__ in which case we have `parameters` 1801 # * The ComponentDefinition comes from the manifest as a dict in which case we have `$parameters` 1802 # We should change those interfaces to use the model once we clean up the code in CDS at which point the parameter propagation should happen as part of the ModelToComponentFactory. 1803 parameters=datetime_based_cursor_model.parameters or {}, 1804 ) 1805 cursor_field = CursorField( 1806 cursor_field_key=interpolated_cursor_field.eval(config=config), 1807 supports_catalog_defined_cursor_field=datetime_based_cursor_model.allow_catalog_defined_cursor_field 1808 or False, 1809 ) 1810 1811 datetime_format = datetime_based_cursor_model.datetime_format 1812 1813 cursor_granularity = ( 1814 parse_duration(datetime_based_cursor_model.cursor_granularity) 1815 if datetime_based_cursor_model.cursor_granularity 1816 else None 1817 ) 1818 1819 connector_state_converter: DateTimeStreamStateConverter 1820 connector_state_converter = CustomFormatConcurrentStreamStateConverter( 1821 datetime_format=datetime_format, 1822 input_datetime_formats=datetime_based_cursor_model.cursor_datetime_formats, 1823 is_sequential_state=True, # ConcurrentPerPartitionCursor only works with sequential state 1824 cursor_granularity=cursor_granularity, 1825 ) 1826 1827 # Create the cursor factory 1828 cursor_factory = ConcurrentCursorFactory( 1829 partial( 1830 self.create_concurrent_cursor_from_datetime_based_cursor, 1831 state_manager=state_manager, 1832 model_type=model_type, 1833 component_definition=component_definition, 1834 stream_name=stream_name, 1835 stream_namespace=stream_namespace, 1836 config=config, 1837 message_repository=NoopMessageRepository(), 1838 ) 1839 ) 1840 1841 # Per-partition state doesn't make sense for GroupingPartitionRouter, so force the global state 1842 use_global_cursor = isinstance( 1843 partition_router, GroupingPartitionRouter 1844 ) or component_definition.get("global_substream_cursor", False) 1845 1846 # Return the concurrent cursor and state converter 1847 return ConcurrentPerPartitionCursor( 1848 cursor_factory=cursor_factory, 1849 partition_router=partition_router, 1850 stream_name=stream_name, 1851 stream_namespace=stream_namespace, 1852 stream_state=stream_state, 1853 message_repository=self._message_repository, # type: ignore 1854 connector_state_manager=state_manager, 1855 connector_state_converter=connector_state_converter, 1856 cursor_field=cursor_field, 1857 use_global_cursor=use_global_cursor, 1858 attempt_to_create_cursor_if_not_provided=attempt_to_create_cursor_if_not_provided, 1859 ) 1860 1861 @staticmethod 1862 def create_constant_backoff_strategy( 1863 model: ConstantBackoffStrategyModel, config: Config, **kwargs: Any 1864 ) -> ConstantBackoffStrategy: 1865 ModelToComponentFactory._validate_jitter_range(model.jitter_range_in_seconds) 1866 return ConstantBackoffStrategy( 1867 backoff_time_in_seconds=model.backoff_time_in_seconds, 1868 jitter_range_in_seconds=model.jitter_range_in_seconds, 1869 config=config, 1870 parameters=model.parameters or {}, 1871 ) 1872 1873 @staticmethod 1874 def _validate_jitter_range(jitter_range_in_seconds: Optional[float]) -> None: 1875 if jitter_range_in_seconds is not None and jitter_range_in_seconds < 0: 1876 raise ValueError("jitter_range_in_seconds must be greater than or equal to 0") 1877 1878 def create_cursor_pagination( 1879 self, 1880 model: CursorPaginationModel, 1881 config: Config, 1882 decoder: Optional[Decoder] = None, 1883 **kwargs: Any, 1884 ) -> CursorPaginationStrategy: 1885 # None when nested under a custom component, which cannot pass a decoder 1886 if decoder is None: 1887 decoder = JsonDecoder(parameters={}) 1888 if isinstance(decoder, PaginationDecoderDecorator): 1889 inner_decoder = decoder.decoder 1890 else: 1891 inner_decoder = decoder 1892 decoder = PaginationDecoderDecorator(decoder=decoder) 1893 1894 if self._is_supported_decoder_for_pagination(inner_decoder): 1895 decoder_to_use = decoder 1896 else: 1897 raise ValueError( 1898 self._UNSUPPORTED_DECODER_ERROR.format(decoder_type=type(inner_decoder)) 1899 ) 1900 1901 # Pydantic v1 Union type coercion can convert int to string depending on Union order. 1902 # If page_size is a string that represents an integer (not an interpolation), convert it back. 1903 page_size = model.page_size 1904 if isinstance(page_size, str) and page_size.isdigit(): 1905 page_size = int(page_size) 1906 1907 return CursorPaginationStrategy( 1908 cursor_value=model.cursor_value, 1909 decoder=decoder_to_use, 1910 page_size=page_size, 1911 stop_condition=model.stop_condition, 1912 config=config, 1913 parameters=model.parameters or {}, 1914 ) 1915 1916 def create_custom_component(self, model: Any, config: Config, **kwargs: Any) -> Any: 1917 """ 1918 Generically creates a custom component based on the model type and a class_name reference to the custom Python class being 1919 instantiated. Only the model's additional properties that match the custom class definition are passed to the constructor 1920 :param model: The Pydantic model of the custom component being created 1921 :param config: The custom defined connector config 1922 :return: The declarative component built from the Pydantic model to be used at runtime 1923 """ 1924 # Instantiating a custom component means importing and executing arbitrary code referenced 1925 # by `class_name`. Manifests supplied by a caller, whether through the config or directly to 1926 # the manifest server, are untrusted input and could point `class_name` at any importable 1927 # callable, so they honor the same `AIRBYTE_ENABLE_UNSAFE_CODE` gate as injected 1928 # `components.py` code. Manifests bundled in a published connector image are trusted and may 1929 # always use their bundled custom components. 1930 manifest_is_untrusted = not self._custom_components_trusted or bool( 1931 config.get(INJECTED_MANIFEST) 1932 ) 1933 if manifest_is_untrusted and not custom_code_execution_permitted(): 1934 raise AirbyteCustomCodeNotPermittedError 1935 1936 custom_component_class = self._get_class_from_fully_qualified_class_name(model.class_name) 1937 component_fields = get_type_hints(custom_component_class) 1938 model_args = model.dict() 1939 model_args["config"] = config 1940 1941 # There are cases where a parent component will pass arguments to a child component via kwargs. When there are field collisions 1942 # we defer to these arguments over the component's definition 1943 for key, arg in kwargs.items(): 1944 model_args[key] = arg 1945 1946 # Pydantic is unable to parse a custom component's fields that are subcomponents into models because their fields and types are not 1947 # defined in the schema. The fields and types are defined within the Python class implementation. Pydantic can only parse down to 1948 # the custom component and this code performs a second parse to convert the sub-fields first into models, then declarative components 1949 for model_field, model_value in model_args.items(): 1950 # If a custom component field doesn't have a type set, we try to use the type hints to infer the type 1951 if ( 1952 isinstance(model_value, dict) 1953 and "type" not in model_value 1954 and model_field in component_fields 1955 ): 1956 derived_type = self._derive_component_type_from_type_hints( 1957 component_fields.get(model_field) 1958 ) 1959 if derived_type: 1960 model_value["type"] = derived_type 1961 1962 if self._is_component(model_value): 1963 model_args[model_field] = self._create_nested_component( 1964 model, 1965 model_field, 1966 model_value, 1967 config, 1968 **kwargs, 1969 ) 1970 elif isinstance(model_value, list): 1971 vals = [] 1972 for v in model_value: 1973 if isinstance(v, dict) and "type" not in v and model_field in component_fields: 1974 derived_type = self._derive_component_type_from_type_hints( 1975 component_fields.get(model_field) 1976 ) 1977 if derived_type: 1978 v["type"] = derived_type 1979 if self._is_component(v): 1980 vals.append( 1981 self._create_nested_component( 1982 model, 1983 model_field, 1984 v, 1985 config, 1986 **kwargs, 1987 ) 1988 ) 1989 else: 1990 vals.append(v) 1991 model_args[model_field] = vals 1992 1993 kwargs = { 1994 class_field: model_args[class_field] 1995 for class_field in component_fields.keys() 1996 if class_field in model_args 1997 } 1998 1999 if "api_budget" in component_fields and kwargs.get("api_budget") is None: 2000 kwargs["api_budget"] = self._api_budget 2001 2002 return custom_component_class(**kwargs) 2003 2004 @staticmethod 2005 def _get_class_from_fully_qualified_class_name( 2006 full_qualified_class_name: str, 2007 ) -> Any: 2008 """Get a class from its fully qualified name. 2009 2010 If a custom components module is needed, we assume it is already registered - probably 2011 as `source_declarative_manifest.components` or `components`. 2012 2013 Args: 2014 full_qualified_class_name (str): The fully qualified name of the class (e.g., "module.ClassName"). 2015 2016 Returns: 2017 Any: The class object. 2018 2019 Raises: 2020 ValueError: If the class cannot be loaded. 2021 """ 2022 split = full_qualified_class_name.split(".") 2023 module_name_full = ".".join(split[:-1]) 2024 class_name = split[-1] 2025 2026 try: 2027 module_ref = importlib.import_module(module_name_full) 2028 except ModuleNotFoundError as e: 2029 if split[0] == "source_declarative_manifest": 2030 # During testing, the modules containing the custom components are not moved to source_declarative_manifest. In order to run the test, add the source folder to your PYTHONPATH or add it runtime using sys.path.append 2031 try: 2032 import os 2033 2034 module_name_with_source_declarative_manifest = ".".join(split[1:-1]) 2035 module_ref = importlib.import_module( 2036 module_name_with_source_declarative_manifest 2037 ) 2038 except ModuleNotFoundError: 2039 raise ValueError(f"Could not load module `{module_name_full}`.") from e 2040 else: 2041 raise ValueError(f"Could not load module `{module_name_full}`.") from e 2042 2043 try: 2044 return getattr(module_ref, class_name) 2045 except AttributeError as e: 2046 raise ValueError( 2047 f"Could not load class `{class_name}` from module `{module_name_full}`.", 2048 ) from e 2049 2050 @staticmethod 2051 def _derive_component_type_from_type_hints(field_type: Any) -> Optional[str]: 2052 interface = field_type 2053 while True: 2054 origin = get_origin(interface) 2055 if origin: 2056 # Unnest types until we reach the raw type 2057 # List[T] -> T 2058 # Optional[List[T]] -> T 2059 args = get_args(interface) 2060 interface = args[0] 2061 else: 2062 break 2063 if isinstance(interface, type) and not ModelToComponentFactory.is_builtin_type(interface): 2064 return interface.__name__ 2065 return None 2066 2067 @staticmethod 2068 def is_builtin_type(cls: Optional[Type[Any]]) -> bool: 2069 if not cls: 2070 return False 2071 return cls.__module__ == "builtins" 2072 2073 @staticmethod 2074 def _extract_missing_parameters(error: TypeError) -> List[str]: 2075 parameter_search = re.search(r"keyword-only.*:\s(.*)", str(error)) 2076 if parameter_search: 2077 return re.findall(r"\'(.+?)\'", parameter_search.group(1)) 2078 else: 2079 return [] 2080 2081 def _create_nested_component( 2082 self, model: Any, model_field: str, model_value: Any, config: Config, **kwargs: Any 2083 ) -> Any: 2084 type_name = model_value.get("type", None) 2085 if not type_name: 2086 # If no type is specified, we can assume this is a dictionary object which can be returned instead of a subcomponent 2087 return model_value 2088 2089 model_type = self.TYPE_NAME_TO_MODEL.get(type_name, None) 2090 if model_type: 2091 parsed_model = model_type.parse_obj(model_value) 2092 try: 2093 # To improve usability of the language, certain fields are shared between components. This can come in the form of 2094 # a parent component passing some of its fields to a child component or the parent extracting fields from other child 2095 # components and passing it to others. One example is the DefaultPaginator referencing the HttpRequester url_base 2096 # while constructing a SimpleRetriever. However, custom components don't support this behavior because they are created 2097 # generically in create_custom_component(). This block allows developers to specify extra arguments in $parameters that 2098 # are needed by a component and could not be shared. 2099 model_constructor = self.PYDANTIC_MODEL_TO_CONSTRUCTOR.get(parsed_model.__class__) 2100 constructor_kwargs = inspect.getfullargspec(model_constructor).kwonlyargs 2101 model_parameters = model_value.get("$parameters", {}) 2102 matching_parameters = { 2103 kwarg: model_parameters[kwarg] 2104 for kwarg in constructor_kwargs 2105 if kwarg in model_parameters 2106 } 2107 matching_kwargs = { 2108 kwarg: kwargs[kwarg] for kwarg in constructor_kwargs if kwarg in kwargs 2109 } 2110 return self._create_component_from_model( 2111 model=parsed_model, config=config, **(matching_parameters | matching_kwargs) 2112 ) 2113 except TypeError as error: 2114 missing_parameters = self._extract_missing_parameters(error) 2115 if missing_parameters: 2116 raise ValueError( 2117 f"Error creating component '{type_name}' with parent custom component {model.class_name}: Please provide " 2118 + ", ".join( 2119 ( 2120 f"{type_name}.$parameters.{parameter}" 2121 for parameter in missing_parameters 2122 ) 2123 ) 2124 ) 2125 raise TypeError( 2126 f"Error creating component '{type_name}' with parent custom component {model.class_name}: {error}" 2127 ) 2128 else: 2129 raise ValueError( 2130 f"Error creating custom component {model.class_name}. Subcomponent creation has not been implemented for '{type_name}'" 2131 ) 2132 2133 @staticmethod 2134 def _is_component(model_value: Any) -> bool: 2135 return isinstance(model_value, dict) and model_value.get("type") is not None 2136 2137 def create_default_stream( 2138 self, model: DeclarativeStreamModel, config: Config, is_parent: bool = False, **kwargs: Any 2139 ) -> AbstractStream: 2140 primary_key = model.primary_key.__root__ if model.primary_key else None 2141 self._validate_sync_token_stream(model) 2142 self._migrate_state(model, config) 2143 self._warn_on_ineffective_incremental_dependency(model) 2144 2145 partition_router = self._build_stream_slicer_from_partition_router( 2146 model.retriever, 2147 config, 2148 stream_name=model.name, 2149 **kwargs, 2150 ) 2151 concurrent_cursor = self._build_concurrent_cursor(model, partition_router, config) 2152 request_options_provider: Optional[RequestOptionsProvider] 2153 if model.incremental_sync and isinstance(model.incremental_sync, DatetimeBasedCursorModel): 2154 cursor_model: DatetimeBasedCursorModel = model.incremental_sync 2155 2156 end_time_option = ( 2157 self._create_component_from_model( 2158 cursor_model.end_time_option, config, parameters=cursor_model.parameters or {} 2159 ) 2160 if cursor_model.end_time_option 2161 else None 2162 ) 2163 start_time_option = ( 2164 self._create_component_from_model( 2165 cursor_model.start_time_option, config, parameters=cursor_model.parameters or {} 2166 ) 2167 if cursor_model.start_time_option 2168 else None 2169 ) 2170 2171 datetime_request_options_provider = DatetimeBasedRequestOptionsProvider( 2172 start_time_option=start_time_option, 2173 end_time_option=end_time_option, 2174 partition_field_start=cursor_model.partition_field_start, 2175 partition_field_end=cursor_model.partition_field_end, 2176 config=config, 2177 parameters=model.parameters or {}, 2178 ) 2179 request_options_provider = ( 2180 datetime_request_options_provider 2181 if not isinstance(concurrent_cursor, ConcurrentPerPartitionCursor) 2182 else PerPartitionRequestOptionsProvider( 2183 partition_router, datetime_request_options_provider 2184 ) 2185 ) 2186 elif isinstance(model.incremental_sync, SyncTokenCursorModel): 2187 sync_token_cursor_model = model.incremental_sync 2188 token_option = self._create_component_from_model( 2189 sync_token_cursor_model.token_option, 2190 config, 2191 parameters=sync_token_cursor_model.parameters or {}, 2192 ) 2193 sync_token_request_options_provider = SyncTokenRequestOptionsProvider( 2194 token_option=token_option, 2195 config=config, 2196 parameters=sync_token_cursor_model.parameters or {}, 2197 initial_token=sync_token_cursor_model.initial_token, 2198 ) 2199 request_options_provider = ( 2200 PerPartitionRequestOptionsProvider( 2201 partition_router, sync_token_request_options_provider 2202 ) 2203 if isinstance(concurrent_cursor, SyncTokenCursor) and concurrent_cursor.partitioned 2204 else sync_token_request_options_provider 2205 ) 2206 elif model.incremental_sync and isinstance( 2207 model.incremental_sync, IncrementingCountCursorModel 2208 ): 2209 if isinstance(concurrent_cursor, ConcurrentPerPartitionCursor): 2210 raise ValueError( 2211 "PerPartition does not support per partition states because switching to global state is time based" 2212 ) 2213 2214 cursor_model: IncrementingCountCursorModel = model.incremental_sync # type: ignore 2215 2216 start_time_option = ( 2217 self._create_component_from_model( 2218 cursor_model.start_value_option, # type: ignore # mypy still thinks cursor_model of type DatetimeBasedCursor 2219 config, 2220 parameters=cursor_model.parameters or {}, 2221 ) 2222 if cursor_model.start_value_option # type: ignore # mypy still thinks cursor_model of type DatetimeBasedCursor 2223 else None 2224 ) 2225 2226 # The concurrent engine defaults the start/end fields on the slice to "start" and "end", but 2227 # the default DatetimeBasedRequestOptionsProvider() sets them to start_time/end_time 2228 partition_field_start = "start" 2229 2230 request_options_provider = DatetimeBasedRequestOptionsProvider( 2231 start_time_option=start_time_option, 2232 partition_field_start=partition_field_start, 2233 config=config, 2234 parameters=model.parameters or {}, 2235 ) 2236 else: 2237 request_options_provider = None 2238 2239 transformations = [] 2240 if model.transformations: 2241 for transformation_model in model.transformations: 2242 transformations.append( 2243 self._create_component_from_model(model=transformation_model, config=config) 2244 ) 2245 file_uploader = None 2246 if model.file_uploader: 2247 file_uploader = self._create_component_from_model( 2248 model=model.file_uploader, config=config 2249 ) 2250 2251 stream_slicer: ConcurrentStreamSlicer = ( 2252 partition_router 2253 if isinstance(concurrent_cursor, FinalStateCursor) 2254 else concurrent_cursor 2255 ) 2256 2257 retriever = self._create_component_from_model( 2258 model=model.retriever, 2259 config=config, 2260 name=model.name, 2261 primary_key=primary_key, 2262 request_options_provider=request_options_provider, 2263 stream_slicer=stream_slicer, 2264 partition_router=partition_router, 2265 has_stop_condition_cursor=self._is_stop_condition_on_cursor(model), 2266 is_client_side_incremental_sync=self._is_client_side_filtering_enabled(model), 2267 cursor=concurrent_cursor, 2268 transformations=transformations, 2269 file_uploader=file_uploader, 2270 incremental_sync=model.incremental_sync, 2271 ) 2272 if isinstance(retriever, AsyncRetriever): 2273 stream_slicer = retriever.stream_slicer 2274 2275 schema_loader: SchemaLoader 2276 if model.schema_loader and isinstance(model.schema_loader, list): 2277 nested_schema_loaders = [ 2278 self._create_component_from_model(model=nested_schema_loader, config=config) 2279 for nested_schema_loader in model.schema_loader 2280 ] 2281 schema_loader = CompositeSchemaLoader( 2282 schema_loaders=nested_schema_loaders, parameters={} 2283 ) 2284 elif model.schema_loader: 2285 schema_loader = self._create_component_from_model( 2286 model=model.schema_loader, # type: ignore # If defined, schema_loader is guaranteed not to be a list and will be one of the existing base models 2287 config=config, 2288 ) 2289 else: 2290 options = model.parameters or {} 2291 if "name" not in options: 2292 options["name"] = model.name 2293 schema_loader = DefaultSchemaLoader(config=config, parameters=options) 2294 schema_loader = CachingSchemaLoaderDecorator(schema_loader) 2295 2296 stream_name = model.name or "" 2297 return DefaultStream( 2298 partition_generator=StreamSlicerPartitionGenerator( 2299 DeclarativePartitionFactory( 2300 stream_name, 2301 schema_loader, 2302 retriever, 2303 self._message_repository, 2304 ), 2305 stream_slicer, 2306 slice_limit=self._limit_slices_fetched, 2307 ), 2308 name=stream_name, 2309 json_schema=schema_loader.get_json_schema, 2310 primary_key=get_primary_key_from_stream(primary_key), 2311 cursor_field=( 2312 concurrent_cursor.cursor_field 2313 if hasattr(concurrent_cursor, "cursor_field") 2314 else None 2315 ), 2316 logger=logging.getLogger(f"airbyte.{stream_name}"), 2317 cursor=concurrent_cursor, 2318 supports_file_transfer=hasattr(model, "file_uploader") and bool(model.file_uploader), 2319 source_defined_incremental=isinstance(model.incremental_sync, SyncTokenCursorModel), 2320 ) 2321 2322 @staticmethod 2323 def _flatten_partition_router_models(router_model: Any) -> list[Any]: 2324 if router_model is None: 2325 return [] 2326 if isinstance(router_model, list): 2327 return [ 2328 nested_router 2329 for router in router_model 2330 for nested_router in ModelToComponentFactory._flatten_partition_router_models( 2331 router 2332 ) 2333 ] 2334 if isinstance(router_model, UnionPartitionRouterModel): 2335 return [ 2336 router_model, 2337 *ModelToComponentFactory._flatten_partition_router_models( 2338 getattr(router_model, "partition_routers", None) 2339 ), 2340 ] 2341 if isinstance(router_model, GroupingPartitionRouterModel): 2342 return [ 2343 router_model, 2344 *ModelToComponentFactory._flatten_partition_router_models( 2345 getattr(router_model, "underlying_partition_router", None) 2346 ), 2347 ] 2348 return [router_model] 2349 2350 @staticmethod 2351 def _validate_sync_token_stream(model: DeclarativeStreamModel) -> None: 2352 cursor_model = model.incremental_sync 2353 stream_name = model.name or "" 2354 retriever_model = model.retriever 2355 router_model = getattr(retriever_model, "partition_router", None) 2356 routers = ModelToComponentFactory._flatten_partition_router_models(router_model) 2357 for router in routers: 2358 if isinstance(router, SubstreamPartitionRouterModel): 2359 for parent in router.parent_stream_configs: 2360 parent_sync = getattr(parent.stream, "incremental_sync", None) 2361 if isinstance(parent.stream, StateDelegatingStreamModel): 2362 parent_sync = parent.stream.incremental_stream.incremental_sync 2363 if parent.incremental_dependency and isinstance( 2364 parent_sync, SyncTokenCursorModel 2365 ): 2366 raise ValueError( 2367 f"Stream {stream_name}: a SyncTokenCursor parent cannot be used with " 2368 "`incremental_dependency`." 2369 ) 2370 if not isinstance(cursor_model, SyncTokenCursorModel): 2371 return 2372 datetime_only_keys = { 2373 "start_datetime", 2374 "end_datetime", 2375 "step", 2376 "cursor_granularity", 2377 "datetime_format", 2378 "cursor_datetime_formats", 2379 "start_time_option", 2380 "end_time_option", 2381 "lookback_window", 2382 "partition_field_start", 2383 "partition_field_end", 2384 "is_data_feed", 2385 "is_client_side_incremental", 2386 "is_compare_strictly", 2387 "global_substream_cursor", 2388 "clamping", 2389 } 2390 unsupported = sorted( 2391 datetime_only_keys.intersection(cursor_model.__dict__).difference( 2392 cursor_model.parameters or {} 2393 ) 2394 ) 2395 if unsupported: 2396 raise ValueError( 2397 f"Stream {stream_name}: SyncTokenCursor does not support datetime-only setting(s): " 2398 f"{', '.join(unsupported)}." 2399 ) 2400 if isinstance(model.retriever, AsyncRetrieverModel): 2401 raise ValueError( 2402 f"Stream {stream_name}: SyncTokenCursor is not supported with AsyncRetriever; " 2403 "use SimpleRetriever so response pages can be observed." 2404 ) 2405 if isinstance(retriever_model, CustomRetrieverModel): 2406 LOGGER.warning( 2407 "Stream %s: CustomRetriever does not expose response pages to SyncTokenCursor; " 2408 "no sync token will be captured.", 2409 stream_name, 2410 ) 2411 2412 for router in routers: 2413 if isinstance(router, GroupingPartitionRouterModel): 2414 raise ValueError( 2415 f"Stream {stream_name}: SyncTokenCursor does not support GroupingPartitionRouter; " 2416 "use an ungrouped partition router." 2417 ) 2418 if isinstance(router, ListPartitionRouterModel) and router.cursor_field == "token": 2419 raise ValueError( 2420 f"Stream {stream_name}: partition router field 'token' conflicts with SyncTokenCursor." 2421 ) 2422 if isinstance(router, SubstreamPartitionRouterModel): 2423 for parent in router.parent_stream_configs: 2424 if parent.partition_field == "token": 2425 raise ValueError( 2426 f"Stream {stream_name}: parent partition field 'token' conflicts with SyncTokenCursor." 2427 ) 2428 if parent.lazy_read_pointer: 2429 raise ValueError( 2430 f"Stream {stream_name}: SyncTokenCursor is not supported with parent " 2431 "`lazy_read_pointer`; remove the lazy pointer." 2432 ) 2433 if parent.incremental_dependency: 2434 raise ValueError( 2435 f"Stream {stream_name}: SyncTokenCursor is not supported with " 2436 "`incremental_dependency` on a SubstreamPartitionRouter." 2437 ) 2438 pagination_reset = getattr(retriever_model, "pagination_reset", None) 2439 if pagination_reset and pagination_reset.limits: 2440 raise ValueError( 2441 f"Stream {stream_name}: pagination reset `limits` are not supported with SyncTokenCursor." 2442 ) 2443 requester_model = getattr(retriever_model, "requester", None) 2444 query_properties = [getattr(requester_model, "query_properties", None)] 2445 request_parameters = getattr(requester_model, "request_parameters", None) 2446 if isinstance(request_parameters, Mapping): 2447 query_properties.extend( 2448 value 2449 for value in request_parameters.values() 2450 if isinstance(value, QueryPropertiesModel) 2451 or (isinstance(value, Mapping) and value.get("type") == "QueryProperties") 2452 ) 2453 for properties in query_properties: 2454 property_chunking = ( 2455 properties.get("property_chunking") 2456 if isinstance(properties, Mapping) 2457 else getattr(properties, "property_chunking", None) 2458 ) 2459 if property_chunking: 2460 raise ValueError( 2461 f"Stream {stream_name}: query property chunking is not supported with " 2462 "SyncTokenCursor." 2463 ) 2464 2465 def _warn_on_ineffective_incremental_dependency(self, model: DeclarativeStreamModel) -> None: 2466 """ 2467 `incremental_dependency: true` only takes effect when the substream defines its own 2468 `incremental_sync`: the parent cursor is persisted under the `parent_state` key of the 2469 substream's state, which is only emitted by incremental substreams. On a stream without 2470 `incremental_sync`, the setting is silently ignored and all parent records are re-read on 2471 every sync, so we warn about the misconfiguration instead. 2472 """ 2473 if model.incremental_sync: 2474 return 2475 2476 partition_router = getattr(model.retriever, "partition_router", None) 2477 if not partition_router: 2478 return 2479 2480 routers = partition_router if isinstance(partition_router, list) else [partition_router] 2481 for router in routers: 2482 if isinstance(router, GroupingPartitionRouterModel): 2483 router = router.underlying_partition_router 2484 if isinstance(router, SubstreamPartitionRouterModel) and any( 2485 parent_stream_config.incremental_dependency 2486 for parent_stream_config in router.parent_stream_configs 2487 ): 2488 LOGGER.warning( 2489 f"Stream `{model.name}` has `incremental_dependency: true` in its parent stream configuration but does not define `incremental_sync`. " 2490 "The parent stream's cursor is only persisted in the state of an incremental substream, so this setting has no effect and all parent records will be re-read on every sync. " 2491 "Define `incremental_sync` on this stream or remove `incremental_dependency`." 2492 ) 2493 return 2494 2495 def _migrate_state(self, model: DeclarativeStreamModel, config: Config) -> None: 2496 stream_name = model.name or "" 2497 stream_state = self._connector_state_manager.get_stream_state( 2498 stream_name=stream_name, namespace=None 2499 ) 2500 if model.state_migrations: 2501 state_transformations = [ 2502 self._create_component_from_model(state_migration, config, declarative_stream=model) 2503 for state_migration in model.state_migrations 2504 ] 2505 else: 2506 state_transformations = [] 2507 stream_state = self.apply_stream_state_migrations(state_transformations, stream_state) 2508 self._connector_state_manager.update_state_for_stream( 2509 stream_name=stream_name, namespace=None, value=stream_state 2510 ) 2511 2512 def _is_stop_condition_on_cursor(self, model: DeclarativeStreamModel) -> bool: 2513 return bool( 2514 model.incremental_sync 2515 and hasattr(model.incremental_sync, "is_data_feed") 2516 and model.incremental_sync.is_data_feed 2517 ) 2518 2519 def _is_client_side_filtering_enabled(self, model: DeclarativeStreamModel) -> bool: 2520 return bool( 2521 model.incremental_sync 2522 and hasattr(model.incremental_sync, "is_client_side_incremental") 2523 and model.incremental_sync.is_client_side_incremental 2524 ) 2525 2526 def _build_stream_slicer_from_partition_router( 2527 self, 2528 model: Union[ 2529 AsyncRetrieverModel, 2530 CustomRetrieverModel, 2531 SimpleRetrieverModel, 2532 ], 2533 config: Config, 2534 stream_name: Optional[str] = None, 2535 **kwargs: Any, 2536 ) -> PartitionRouter: 2537 if ( 2538 hasattr(model, "partition_router") 2539 and isinstance(model, (SimpleRetrieverModel, AsyncRetrieverModel, CustomRetrieverModel)) 2540 and model.partition_router 2541 ): 2542 stream_slicer_model = model.partition_router 2543 if isinstance(stream_slicer_model, list): 2544 return CartesianProductStreamSlicer( 2545 [ 2546 self._create_component_from_model( 2547 model=slicer, config=config, stream_name=stream_name or "" 2548 ) 2549 for slicer in stream_slicer_model 2550 ], 2551 parameters={}, 2552 ) 2553 elif isinstance(stream_slicer_model, dict): 2554 # partition router comes from CustomRetrieverModel therefore has not been parsed as a model 2555 params = stream_slicer_model.get("$parameters") 2556 if not isinstance(params, dict): 2557 params = {} 2558 stream_slicer_model["$parameters"] = params 2559 2560 if stream_name is not None: 2561 params["stream_name"] = stream_name 2562 2563 return self._create_nested_component( # type: ignore[no-any-return] # There is no guarantee that this will return a stream slicer. If not, we expect an AttributeError during the call to `stream_slices` 2564 model, 2565 "partition_router", 2566 stream_slicer_model, 2567 config, 2568 **kwargs, 2569 ) 2570 else: 2571 return self._create_component_from_model( # type: ignore[no-any-return] # Will be created PartitionRouter as stream_slicer_model is model.partition_router 2572 model=stream_slicer_model, config=config, stream_name=stream_name or "" 2573 ) 2574 return SinglePartitionRouter(parameters={}) 2575 2576 def _build_concurrent_cursor( 2577 self, 2578 model: DeclarativeStreamModel, 2579 stream_slicer: Optional[PartitionRouter], 2580 config: Config, 2581 ) -> Cursor: 2582 stream_name = model.name or "" 2583 stream_state = self._connector_state_manager.get_stream_state(stream_name, None) 2584 2585 if isinstance(model.incremental_sync, SyncTokenCursorModel): 2586 return self.create_sync_token_cursor( 2587 model.incremental_sync, 2588 config=config, 2589 stream_name=stream_name, 2590 stream_state=stream_state, 2591 partition_router=stream_slicer or SinglePartitionRouter(parameters={}), 2592 retriever_model=model.retriever, 2593 ) 2594 2595 if ( 2596 model.incremental_sync 2597 and stream_slicer 2598 and not isinstance(stream_slicer, SinglePartitionRouter) 2599 ): 2600 if isinstance(model.incremental_sync, IncrementingCountCursorModel): 2601 # We don't currently support usage of partition routing and IncrementingCountCursor at the 2602 # same time because we didn't solve for design questions like what the lookback window would 2603 # be as well as global cursor fall backs. We have not seen customers that have needed both 2604 # at the same time yet and are currently punting on this until we need to solve it. 2605 raise ValueError( 2606 f"The low-code framework does not currently support usage of a PartitionRouter and an IncrementingCountCursor at the same time. Please specify only one of these options for stream {stream_name}." 2607 ) 2608 return self.create_concurrent_cursor_from_perpartition_cursor( # type: ignore # This is a known issue that we are creating and returning a ConcurrentCursor which does not technically implement the (low-code) StreamSlicer. However, (low-code) StreamSlicer and ConcurrentCursor both implement StreamSlicer.stream_slices() which is the primary method needed for checkpointing 2609 state_manager=self._connector_state_manager, 2610 model_type=DatetimeBasedCursorModel, 2611 component_definition=model.incremental_sync.__dict__, 2612 stream_name=stream_name, 2613 stream_state=stream_state, 2614 stream_namespace=None, 2615 config=config or {}, 2616 partition_router=stream_slicer, 2617 attempt_to_create_cursor_if_not_provided=True, # FIXME can we remove that now? 2618 ) 2619 elif model.incremental_sync: 2620 if type(model.incremental_sync) == IncrementingCountCursorModel: 2621 return self.create_concurrent_cursor_from_incrementing_count_cursor( # type: ignore # This is a known issue that we are creating and returning a ConcurrentCursor which does not technically implement the (low-code) StreamSlicer. However, (low-code) StreamSlicer and ConcurrentCursor both implement StreamSlicer.stream_slices() which is the primary method needed for checkpointing 2622 model_type=IncrementingCountCursorModel, 2623 component_definition=model.incremental_sync.__dict__, 2624 stream_name=stream_name, 2625 stream_namespace=None, 2626 stream_state=stream_state, 2627 config=config or {}, 2628 ) 2629 elif type(model.incremental_sync) == DatetimeBasedCursorModel: 2630 return self.create_concurrent_cursor_from_datetime_based_cursor( # type: ignore # This is a known issue that we are creating and returning a ConcurrentCursor which does not technically implement the (low-code) StreamSlicer. However, (low-code) StreamSlicer and ConcurrentCursor both implement StreamSlicer.stream_slices() which is the primary method needed for checkpointing 2631 model_type=type(model.incremental_sync), 2632 component_definition=model.incremental_sync.__dict__, 2633 stream_name=stream_name, 2634 stream_namespace=None, 2635 stream_state=stream_state, 2636 config=config or {}, 2637 attempt_to_create_cursor_if_not_provided=True, 2638 ) 2639 else: 2640 raise ValueError( 2641 f"Incremental sync of type {type(model.incremental_sync)} is not supported" 2642 ) 2643 return FinalStateCursor(stream_name, None, self._message_repository) 2644 2645 def create_sync_token_cursor( 2646 self, 2647 model: SyncTokenCursorModel, 2648 *, 2649 config: Config, 2650 stream_name: str, 2651 stream_state: Optional[Mapping[str, Any]], 2652 partition_router: PartitionRouter, 2653 retriever_model: Any, 2654 ) -> SyncTokenCursor: 2655 decoder_model = getattr(retriever_model, "decoder", None) 2656 decoder = ( 2657 self._create_component_from_model(model=decoder_model, config=config) 2658 if isinstance(decoder_model, BaseModel) 2659 else JsonDecoder(parameters={}) 2660 ) 2661 if self._decoder_streams_in_production(decoder): 2662 raise AirbyteTracedException( 2663 message=f"Stream {stream_name}: SyncTokenCursor is not supported with a streaming decoder.", 2664 internal_message=( 2665 f"Stream {stream_name} uses {type(decoder).__name__}, which streams the response " 2666 f"and can only read it once. SyncTokenCursor reads the token from the response " 2667 f"after the record selector has consumed it. Use a non-streaming decoder such " 2668 f"as JsonDecoder or XmlDecoder." 2669 ), 2670 failure_type=FailureType.config_error, 2671 ) 2672 return SyncTokenCursor( 2673 stream_name=stream_name, 2674 stream_namespace=None, 2675 stream_state=stream_state, 2676 message_repository=self._message_repository, 2677 connector_state_manager=self._connector_state_manager, 2678 partition_router=partition_router, 2679 token_path=model.token_path, 2680 decoder=decoder, 2681 config=config or {}, 2682 parameters=model.parameters or {}, 2683 completion_condition=model.completion_condition, 2684 cursor_field=( 2685 InterpolatedString.create( 2686 model.cursor_field, parameters=model.parameters or {} 2687 ).eval(config=config) 2688 if model.cursor_field 2689 else None 2690 ), 2691 ) 2692 2693 def create_default_error_handler( 2694 self, model: DefaultErrorHandlerModel, config: Config, **kwargs: Any 2695 ) -> DefaultErrorHandler: 2696 backoff_strategies = [] 2697 if model.backoff_strategies: 2698 for backoff_strategy_model in model.backoff_strategies: 2699 backoff_strategies.append( 2700 self._create_component_from_model(model=backoff_strategy_model, config=config) 2701 ) 2702 2703 response_filters = [] 2704 if model.response_filters: 2705 for response_filter_model in model.response_filters: 2706 response_filters.append( 2707 self._create_component_from_model(model=response_filter_model, config=config) 2708 ) 2709 response_filters.append( 2710 HttpResponseFilter(config=config, parameters=model.parameters or {}) 2711 ) 2712 2713 return DefaultErrorHandler( 2714 backoff_strategies=backoff_strategies, 2715 max_retries=model.max_retries, 2716 response_filters=response_filters, 2717 config=config, 2718 parameters=model.parameters or {}, 2719 ) 2720 2721 def create_default_paginator( 2722 self, 2723 model: DefaultPaginatorModel, 2724 config: Config, 2725 *, 2726 url_base: str, 2727 extractor_model: Optional[ 2728 Union[CustomRecordExtractorModel, DpathExtractorModel, CombinedExtractorModel] 2729 ] = None, 2730 decoder: Optional[Decoder] = None, 2731 cursor_used_for_stop_condition: Optional[Cursor] = None, 2732 ) -> Union[DefaultPaginator, PaginatorTestReadDecorator]: 2733 if decoder: 2734 if self._is_supported_decoder_for_pagination(decoder): 2735 decoder_to_use = PaginationDecoderDecorator(decoder=decoder) 2736 else: 2737 raise ValueError(self._UNSUPPORTED_DECODER_ERROR.format(decoder_type=type(decoder))) 2738 else: 2739 decoder_to_use = PaginationDecoderDecorator(decoder=JsonDecoder(parameters={})) 2740 page_size_option = ( 2741 self._create_component_from_model(model=model.page_size_option, config=config) 2742 if model.page_size_option 2743 else None 2744 ) 2745 page_token_option = ( 2746 self._create_component_from_model(model=model.page_token_option, config=config) 2747 if model.page_token_option 2748 else None 2749 ) 2750 pagination_strategy = self._create_component_from_model( 2751 model=model.pagination_strategy, 2752 config=config, 2753 decoder=decoder_to_use, 2754 extractor_model=extractor_model, 2755 ) 2756 if cursor_used_for_stop_condition: 2757 pagination_strategy = StopConditionPaginationStrategyDecorator( 2758 pagination_strategy, CursorStopCondition(cursor_used_for_stop_condition) 2759 ) 2760 paginator = DefaultPaginator( 2761 decoder=decoder_to_use, 2762 page_size_option=page_size_option, 2763 page_token_option=page_token_option, 2764 pagination_strategy=pagination_strategy, 2765 url_base=url_base, 2766 config=config, 2767 parameters=model.parameters or {}, 2768 ) 2769 if self._limit_pages_fetched_per_slice: 2770 return PaginatorTestReadDecorator(paginator, self._limit_pages_fetched_per_slice) 2771 return paginator 2772 2773 @staticmethod 2774 def _is_decoder_downgraded_by_connector_builder(decoder: Decoder) -> bool: 2775 """Is this the buffered stand-in the Connector Builder builds for a streaming decoder? 2776 2777 `create_csv_decoder`, `create_jsonl_decoder`, `create_json_items_decoder` and 2778 `create_gzip_decoder` build a `CompositeRawDecoder` with `stream_response=False` when 2779 `_emit_connector_builder_messages` is set, so a Builder test read can be replayed. Those 2780 four instances are the only ones that stream in production while reporting 2781 `is_stream_response() == False` in the Builder. 2782 2783 The match is on the exact class and on the parser, not on `isinstance`: a `CustomDecoder` 2784 subclassing `CompositeRawDecoder` with `stream_response=False`, or a buffered 2785 `CompositeRawDecoder` built around a `JsonParser`, reads the body from `response.content` 2786 in production too and must not be rejected. 2787 """ 2788 return type(decoder) is CompositeRawDecoder and isinstance( 2789 decoder.parser, (CsvParser, JsonLineParser, JsonItemsParser, GzipParser) 2790 ) 2791 2792 def _decoder_streams_in_production(self, decoder: Decoder) -> bool: 2793 """Can this decoder read a response body only once in production? 2794 2795 True for decoders that stream (`is_stream_response()`), and, in the Connector Builder, for 2796 the buffered stand-ins it builds for them (`_is_decoder_downgraded_by_connector_builder`). 2797 """ 2798 inner_decoder = ( 2799 decoder.decoder if isinstance(decoder, PaginationDecoderDecorator) else decoder 2800 ) 2801 return inner_decoder.is_stream_response() or ( 2802 self._emit_connector_builder_messages 2803 and self._is_decoder_downgraded_by_connector_builder(inner_decoder) 2804 ) 2805 2806 def _reject_combined_extractor_over_streaming_decoder(self, decoder: Optional[Decoder]) -> None: 2807 """Refuse to build a `CombinedExtractor` whose response body can only be read once. 2808 2809 Every sub-extractor is handed the same `requests.Response`. A streaming decoder consumes 2810 and closes `response.raw`, so every sub-extractor after the first reads a closed 2811 `urllib3.HTTPResponse`, which returns an empty body instead of raising: `union` emits only 2812 the first sub-extractor's records and `first_match` emits nothing when the first path 2813 misses. Rejecting the manifest is the only way to make that loud. 2814 2815 The Connector Builder downgrades those same decoders to `stream_response=False` so a test 2816 read can be replayed, which would let a manifest look correct in the Builder and lose 2817 records once published. The downgraded decoders are therefore rejected in the Builder too. 2818 """ 2819 if decoder is None or not self._decoder_streams_in_production(decoder): 2820 return 2821 inner_decoder = ( 2822 decoder.decoder if isinstance(decoder, PaginationDecoderDecorator) else decoder 2823 ) 2824 raise AirbyteTracedException( 2825 message="CombinedExtractor is not supported with a streaming decoder.", 2826 internal_message=( 2827 f"CombinedExtractor was configured with {type(inner_decoder).__name__}, which " 2828 f"streams the response and can only be read once. Sub-extractors after the first " 2829 f"would read a closed response and silently return no records. The streaming " 2830 f"decoders are CsvDecoder, JsonlDecoder, JsonItemsDecoder, GzipDecoder and " 2831 f"IterableDecoder; use JsonDecoder, XmlDecoder or ZipfileDecoder, or declare a " 2832 f"single extractor." 2833 ), 2834 failure_type=FailureType.config_error, 2835 ) 2836 2837 def create_combined_extractor( 2838 self, 2839 model: CombinedExtractorModel, 2840 config: Config, 2841 decoder: Optional[Decoder] = None, 2842 **kwargs: Any, 2843 ) -> CombinedExtractor: 2844 self._reject_combined_extractor_over_streaming_decoder(decoder) 2845 extractors = [ 2846 self._create_component_from_model(model=sub_extractor, config=config, decoder=decoder) 2847 for sub_extractor in model.extractors 2848 ] 2849 return CombinedExtractor( 2850 extractors=extractors, 2851 mode=CombineMode(model.mode.value) if model.mode else CombineMode.union, 2852 skip_empty_records=bool(model.skip_empty_records), 2853 parameters=model.parameters or {}, 2854 ) 2855 2856 def create_dpath_extractor( 2857 self, 2858 model: DpathExtractorModel, 2859 config: Config, 2860 decoder: Optional[Decoder] = None, 2861 **kwargs: Any, 2862 ) -> DpathExtractor: 2863 if decoder: 2864 decoder_to_use = decoder 2865 else: 2866 decoder_to_use = JsonDecoder(parameters={}) 2867 model_field_path: List[Union[InterpolatedString, str]] = [x for x in model.field_path] 2868 2869 record_expander = None 2870 if model.record_expander: 2871 record_expander = self._create_component_from_model( 2872 model=model.record_expander, 2873 config=config, 2874 ) 2875 2876 return DpathExtractor( 2877 decoder=decoder_to_use, 2878 field_path=model_field_path, 2879 config=config, 2880 parameters=model.parameters or {}, 2881 record_expander=record_expander, 2882 ) 2883 2884 def create_record_expander( 2885 self, 2886 model: RecordExpanderModel, 2887 config: Config, 2888 **kwargs: Any, 2889 ) -> RecordExpander: 2890 truncated_list_retriever = None 2891 suppress_incomplete_fetch_warning = False 2892 if model.truncated_list_retriever: 2893 retriever_model = model.truncated_list_retriever 2894 name = "record_expander_truncated_list" 2895 # `CustomRetriever` allows extra fields, so read from the dumped model to cover both types. 2896 retriever_fields = retriever_model.dict() 2897 for unsupported_option in ("partition_router", "pagination_reset"): 2898 if retriever_fields.get(unsupported_option): 2899 raise ValueError( 2900 f"`{unsupported_option}` is not supported on `truncated_list_retriever`." 2901 ) 2902 log_formatter = lambda response: format_http_message( 2903 response, 2904 f"Record expander '{name}' request", 2905 "Request performed in order to fetch the complete nested list of a truncated record.", 2906 name, 2907 is_auxiliary=True, 2908 ) 2909 # `name`/`primary_key`/`transformations` are also what a `CustomRetriever` forwards to its 2910 # nested `requester`/`record_selector`, so they are passed for both retriever types. 2911 truncated_list_retriever = self._create_component_from_model( 2912 model=retriever_model, 2913 config=config, 2914 name=name, 2915 primary_key=None, 2916 transformations=[], 2917 log_formatter=log_formatter, 2918 ) 2919 # Only a capped paginator makes a shortfall expected; `NoPagination` and retrievers 2920 # without a paginator are never capped. 2921 suppress_incomplete_fetch_warning = isinstance( 2922 truncated_list_retriever, SimpleRetriever 2923 ) and isinstance(truncated_list_retriever.paginator, PaginatorTestReadDecorator) 2924 return RecordExpander( 2925 expand_records_from_field=model.expand_records_from_field, 2926 config=config, 2927 parameters=model.parameters or {}, 2928 remain_original_record=model.remain_original_record or False, 2929 parent_fields=[ 2930 self._create_component_from_model(model=parent_field, config=config) 2931 for parent_field in model.parent_fields 2932 ] 2933 if model.parent_fields 2934 else None, 2935 merge_parent=model.merge_parent or False, 2936 on_no_records=OnNoRecords(model.on_no_records.value) 2937 if model.on_no_records 2938 else OnNoRecords.skip, 2939 truncation_indicator_path=model.truncation_indicator_path, 2940 truncated_list_retriever=truncated_list_retriever, 2941 message_repository=self._message_repository, 2942 suppress_incomplete_fetch_warning=suppress_incomplete_fetch_warning, 2943 ) 2944 2945 @staticmethod 2946 def create_parent_field_path( 2947 model: ParentFieldPathModel, 2948 config: Config, 2949 **kwargs: Any, 2950 ) -> ParentFieldPath: 2951 return ParentFieldPath( 2952 parent_path=model.parent_path, 2953 record_path=model.record_path, 2954 config=config, 2955 parameters=model.parameters or {}, 2956 ) 2957 2958 @staticmethod 2959 def create_response_to_file_extractor( 2960 model: ResponseToFileExtractorModel, 2961 **kwargs: Any, 2962 ) -> ResponseToFileExtractor: 2963 return ResponseToFileExtractor( 2964 parameters=model.parameters or {}, 2965 preserve_na_values=model.preserve_na_values or False, 2966 ) 2967 2968 @staticmethod 2969 def create_exponential_backoff_strategy( 2970 model: ExponentialBackoffStrategyModel, config: Config 2971 ) -> ExponentialBackoffStrategy: 2972 ModelToComponentFactory._validate_jitter_range(model.jitter_range_in_seconds) 2973 return ExponentialBackoffStrategy( 2974 factor=model.factor or 5, 2975 jitter_range_in_seconds=model.jitter_range_in_seconds, 2976 parameters=model.parameters or {}, 2977 config=config, 2978 ) 2979 2980 @staticmethod 2981 def create_group_by_key(model: GroupByKeyMergeStrategyModel, config: Config) -> GroupByKey: 2982 return GroupByKey(model.key, config=config, parameters=model.parameters or {}) 2983 2984 def create_http_requester( 2985 self, 2986 model: HttpRequesterModel, 2987 config: Config, 2988 decoder: Decoder = JsonDecoder(parameters={}), 2989 query_properties_key: Optional[str] = None, 2990 use_cache: Optional[bool] = None, 2991 *, 2992 name: str, 2993 ) -> HttpRequester: 2994 authenticator = ( 2995 self._create_component_from_model( 2996 model=model.authenticator, 2997 config=config, 2998 url_base=model.url or model.url_base, 2999 name=name, 3000 decoder=decoder, 3001 ) 3002 if model.authenticator 3003 else None 3004 ) 3005 error_handler = ( 3006 self._create_component_from_model(model=model.error_handler, config=config) 3007 if model.error_handler 3008 else DefaultErrorHandler( 3009 backoff_strategies=[], 3010 response_filters=[], 3011 config=config, 3012 parameters=model.parameters or {}, 3013 ) 3014 ) 3015 3016 api_budget = self._api_budget 3017 3018 request_options_provider = InterpolatedRequestOptionsProvider( 3019 request_body=model.request_body, 3020 request_body_data=model.request_body_data, 3021 request_body_json=model.request_body_json, 3022 request_headers=model.request_headers, 3023 request_parameters=model.request_parameters, # type: ignore # QueryProperties have been removed in `create_simple_retriever` 3024 query_properties_key=query_properties_key, 3025 config=config, 3026 parameters=model.parameters or {}, 3027 ) 3028 3029 assert model.use_cache is not None # for mypy 3030 assert model.http_method is not None # for mypy 3031 3032 should_use_cache = (model.use_cache or bool(use_cache)) and not self._disable_cache 3033 3034 return HttpRequester( 3035 name=name, 3036 url=model.url, 3037 url_base=model.url_base, 3038 path=model.path, 3039 authenticator=authenticator, 3040 error_handler=error_handler, 3041 api_budget=api_budget, 3042 http_method=HttpMethod[model.http_method.value], 3043 request_options_provider=request_options_provider, 3044 config=config, 3045 disable_retries=self._disable_retries, 3046 parameters=model.parameters or {}, 3047 message_repository=self._message_repository, 3048 use_cache=should_use_cache, 3049 connect_timeout_in_seconds=model.connect_timeout_in_seconds, 3050 read_timeout_in_seconds=model.read_timeout_in_seconds, 3051 use_tcp_keepalive=model.use_tcp_keepalive or False, 3052 decoder=decoder, 3053 stream_response=decoder.is_stream_response() if decoder else False, 3054 ) 3055 3056 @staticmethod 3057 def create_http_response_filter( 3058 model: HttpResponseFilterModel, config: Config, **kwargs: Any 3059 ) -> HttpResponseFilter: 3060 if model.action: 3061 action = ResponseAction(model.action.value) 3062 else: 3063 action = None 3064 3065 failure_type = FailureType(model.failure_type.value) if model.failure_type else None 3066 3067 http_codes = ( 3068 set(model.http_codes) if model.http_codes else set() 3069 ) # JSON schema notation has no set data type. The schema enforces an array of unique elements 3070 3071 return HttpResponseFilter( 3072 action=action, 3073 failure_type=failure_type, 3074 error_message=model.error_message or "", 3075 error_message_contains=model.error_message_contains or "", 3076 http_codes=http_codes, 3077 predicate=model.predicate or "", 3078 config=config, 3079 parameters=model.parameters or {}, 3080 ) 3081 3082 @staticmethod 3083 def create_inline_schema_loader( 3084 model: InlineSchemaLoaderModel, config: Config, **kwargs: Any 3085 ) -> InlineSchemaLoader: 3086 return InlineSchemaLoader(schema=model.schema_ or {}, parameters={}) 3087 3088 def create_complex_field_type( 3089 self, model: ComplexFieldTypeModel, config: Config, **kwargs: Any 3090 ) -> ComplexFieldType: 3091 items = ( 3092 self._create_component_from_model(model=model.items, config=config) 3093 if isinstance(model.items, ComplexFieldTypeModel) 3094 else model.items 3095 ) 3096 3097 return ComplexFieldType(field_type=model.field_type, items=items) 3098 3099 def create_types_map(self, model: TypesMapModel, config: Config, **kwargs: Any) -> TypesMap: 3100 target_type = ( 3101 self._create_component_from_model(model=model.target_type, config=config) 3102 if isinstance(model.target_type, ComplexFieldTypeModel) 3103 else model.target_type 3104 ) 3105 3106 return TypesMap( 3107 target_type=target_type, 3108 current_type=model.current_type, 3109 condition=model.condition if model.condition is not None else "True", 3110 ) 3111 3112 def create_schema_type_identifier( 3113 self, model: SchemaTypeIdentifierModel, config: Config, **kwargs: Any 3114 ) -> SchemaTypeIdentifier: 3115 types_mapping = [] 3116 if model.types_mapping: 3117 types_mapping.extend( 3118 [ 3119 self._create_component_from_model(types_map, config=config) 3120 for types_map in model.types_mapping 3121 ] 3122 ) 3123 model_schema_pointer: List[Union[InterpolatedString, str]] = ( 3124 [x for x in model.schema_pointer] if model.schema_pointer else [] 3125 ) 3126 model_key_pointer: List[Union[InterpolatedString, str]] = [x for x in model.key_pointer] 3127 model_type_pointer: Optional[List[Union[InterpolatedString, str]]] = ( 3128 [x for x in model.type_pointer] if model.type_pointer else None 3129 ) 3130 3131 return SchemaTypeIdentifier( 3132 schema_pointer=model_schema_pointer, 3133 key_pointer=model_key_pointer, 3134 type_pointer=model_type_pointer, 3135 types_mapping=types_mapping, 3136 parameters=model.parameters or {}, 3137 ) 3138 3139 def create_dynamic_schema_loader( 3140 self, model: DynamicSchemaLoaderModel, config: Config, **kwargs: Any 3141 ) -> DynamicSchemaLoader: 3142 schema_transformations = [] 3143 if model.schema_transformations: 3144 for transformation_model in model.schema_transformations: 3145 schema_transformations.append( 3146 self._create_component_from_model(model=transformation_model, config=config) 3147 ) 3148 name = "dynamic_properties" 3149 partition_router = self._build_stream_slicer_from_partition_router(model.retriever, config) 3150 retriever_kwargs: Dict[str, Any] = {} 3151 if isinstance(model.retriever, AsyncRetrieverModel): 3152 if model.retriever.partition_router: 3153 raise ValueError( 3154 "DynamicSchemaLoader does not support an AsyncRetriever with a partition_router: only the first record is read, so jobs for other partitions would be orphaned. Remove the partition_router from the schema loader's AsyncRetriever." 3155 ) 3156 # create_async_retriever requires stream_slicer; passing it to a CustomRetriever would override its own 3157 retriever_kwargs["stream_slicer"] = partition_router 3158 retriever = self._create_component_from_model( 3159 model=model.retriever, 3160 config=config, 3161 name=name, 3162 primary_key=None, 3163 partition_router=partition_router, 3164 **retriever_kwargs, 3165 transformations=[], 3166 use_cache=True, 3167 log_formatter=( 3168 lambda response: format_http_message( 3169 response, 3170 f"Schema loader '{name}' request", 3171 f"Request performed in order to extract schema.", 3172 name, 3173 is_auxiliary=True, 3174 ) 3175 ), 3176 ) 3177 schema_type_identifier = self._create_component_from_model( 3178 model.schema_type_identifier, config=config, parameters=model.parameters or {} 3179 ) 3180 schema_filter = ( 3181 self._create_component_from_model( 3182 model.schema_filter, config=config, parameters=model.parameters or {} 3183 ) 3184 if model.schema_filter is not None 3185 else None 3186 ) 3187 3188 return DynamicSchemaLoader( 3189 retriever=retriever, 3190 config=config, 3191 schema_transformations=schema_transformations, 3192 schema_filter=schema_filter, 3193 schema_type_identifier=schema_type_identifier, 3194 parameters=model.parameters or {}, 3195 ) 3196 3197 @staticmethod 3198 def create_json_decoder(model: JsonDecoderModel, config: Config, **kwargs: Any) -> Decoder: 3199 return JsonDecoder(parameters={}) 3200 3201 def create_csv_decoder(self, model: CsvDecoderModel, config: Config, **kwargs: Any) -> Decoder: 3202 return CompositeRawDecoder( 3203 parser=ModelToComponentFactory._get_parser(model, config), 3204 stream_response=False if self._emit_connector_builder_messages else True, 3205 ) 3206 3207 def create_jsonl_decoder( 3208 self, model: JsonlDecoderModel, config: Config, **kwargs: Any 3209 ) -> Decoder: 3210 return CompositeRawDecoder( 3211 parser=ModelToComponentFactory._get_parser(model, config), 3212 stream_response=False if self._emit_connector_builder_messages else True, 3213 ) 3214 3215 def create_json_items_decoder( 3216 self, model: JsonItemsDecoderModel, config: Config, **kwargs: Any 3217 ) -> Decoder: 3218 return CompositeRawDecoder( 3219 parser=ModelToComponentFactory._get_parser(model, config), 3220 stream_response=False if self._emit_connector_builder_messages else True, 3221 ) 3222 3223 def create_gzip_decoder( 3224 self, model: GzipDecoderModel, config: Config, **kwargs: Any 3225 ) -> Decoder: 3226 _compressed_response_types = { 3227 "gzip", 3228 "x-gzip", 3229 "gzip, deflate", 3230 "x-gzip, deflate", 3231 "application/zip", 3232 "application/gzip", 3233 "application/x-gzip", 3234 "application/x-zip-compressed", 3235 } 3236 3237 gzip_parser: GzipParser = ModelToComponentFactory._get_parser(model, config) # type: ignore # based on the model, we know this will be a GzipParser 3238 3239 if self._emit_connector_builder_messages: 3240 return CompositeRawDecoder(gzip_parser, False) 3241 3242 transport_gzip_parser = GzipParser(inner_parser=gzip_parser) 3243 return CompositeRawDecoder.by_headers( 3244 [ 3245 ({"Content-Encoding"}, {"gzip"}, transport_gzip_parser), 3246 ({"Content-Type"}, _compressed_response_types, gzip_parser), 3247 ], 3248 stream_response=True, 3249 fallback_parser=gzip_parser, 3250 ) 3251 3252 @staticmethod 3253 def create_iterable_decoder( 3254 model: IterableDecoderModel, config: Config, **kwargs: Any 3255 ) -> IterableDecoder: 3256 return IterableDecoder(parameters={}) 3257 3258 @staticmethod 3259 def create_xml_decoder(model: XmlDecoderModel, config: Config, **kwargs: Any) -> XmlDecoder: 3260 return XmlDecoder(parameters={}) 3261 3262 def create_zipfile_decoder( 3263 self, model: ZipfileDecoderModel, config: Config, **kwargs: Any 3264 ) -> ZipfileDecoder: 3265 return ZipfileDecoder(parser=ModelToComponentFactory._get_parser(model.decoder, config)) 3266 3267 @staticmethod 3268 def _get_parser(model: BaseModel, config: Config) -> Parser: 3269 if isinstance(model, JsonDecoderModel): 3270 # Note that the logic is a bit different from the JsonDecoder as there is some legacy that is maintained to return {} on error cases 3271 return JsonParser() 3272 elif isinstance(model, JsonItemsDecoderModel): 3273 return JsonItemsParser( 3274 items_path=model.items_path, 3275 encoding=model.encoding, 3276 ) 3277 elif isinstance(model, JsonlDecoderModel): 3278 return JsonLineParser() 3279 elif isinstance(model, CsvDecoderModel): 3280 return CsvParser( 3281 encoding=model.encoding, 3282 delimiter=model.delimiter, 3283 set_values_to_none=model.set_values_to_none, 3284 ) 3285 elif isinstance(model, GzipDecoderModel): 3286 return GzipParser( 3287 inner_parser=ModelToComponentFactory._get_parser(model.decoder, config) 3288 ) 3289 elif isinstance( 3290 model, (CustomDecoderModel, IterableDecoderModel, XmlDecoderModel, ZipfileDecoderModel) 3291 ): 3292 raise ValueError(f"Decoder type {model} does not have parser associated to it") 3293 3294 raise ValueError(f"Unknown decoder type {model}") 3295 3296 @staticmethod 3297 def create_json_file_schema_loader( 3298 model: JsonFileSchemaLoaderModel, config: Config, **kwargs: Any 3299 ) -> JsonFileSchemaLoader: 3300 return JsonFileSchemaLoader( 3301 file_path=model.file_path or "", config=config, parameters=model.parameters or {} 3302 ) 3303 3304 def create_jwt_authenticator( 3305 self, model: JwtAuthenticatorModel, config: Config, **kwargs: Any 3306 ) -> JwtAuthenticator: 3307 jwt_headers = model.jwt_headers or JwtHeadersModel(kid=None, typ="JWT", cty=None) 3308 jwt_payload = model.jwt_payload or JwtPayloadModel(iss=None, sub=None, aud=None) 3309 request_option = ( 3310 self._create_component_from_model(model.request_option, config) 3311 if model.request_option 3312 else None 3313 ) 3314 return JwtAuthenticator( 3315 config=config, 3316 parameters=model.parameters or {}, 3317 algorithm=JwtAlgorithm(model.algorithm.value), 3318 secret_key=model.secret_key, 3319 base64_encode_secret_key=model.base64_encode_secret_key, 3320 token_duration=model.token_duration, 3321 header_prefix=model.header_prefix, 3322 kid=jwt_headers.kid, 3323 typ=jwt_headers.typ, 3324 cty=jwt_headers.cty, 3325 iss=jwt_payload.iss, 3326 sub=jwt_payload.sub, 3327 aud=jwt_payload.aud, 3328 additional_jwt_headers=model.additional_jwt_headers, 3329 additional_jwt_payload=model.additional_jwt_payload, 3330 passphrase=model.passphrase, 3331 request_option=request_option, 3332 ) 3333 3334 def create_list_partition_router( 3335 self, model: ListPartitionRouterModel, config: Config, **kwargs: Any 3336 ) -> ListPartitionRouter: 3337 request_option = ( 3338 self._create_component_from_model(model.request_option, config) 3339 if model.request_option 3340 else None 3341 ) 3342 return ListPartitionRouter( 3343 cursor_field=model.cursor_field, 3344 request_option=request_option, 3345 values=model.values, 3346 config=config, 3347 parameters=model.parameters or {}, 3348 ) 3349 3350 @staticmethod 3351 def create_min_max_datetime( 3352 model: MinMaxDatetimeModel, config: Config, **kwargs: Any 3353 ) -> MinMaxDatetime: 3354 return MinMaxDatetime( 3355 datetime=model.datetime, 3356 datetime_format=model.datetime_format or "", 3357 max_datetime=model.max_datetime or "", 3358 min_datetime=model.min_datetime or "", 3359 parameters=model.parameters or {}, 3360 ) 3361 3362 @staticmethod 3363 def create_no_auth(model: NoAuthModel, config: Config, **kwargs: Any) -> NoAuth: 3364 return NoAuth(parameters=model.parameters or {}) 3365 3366 @staticmethod 3367 def create_no_pagination( 3368 model: NoPaginationModel, config: Config, **kwargs: Any 3369 ) -> NoPagination: 3370 return NoPagination(parameters={}) 3371 3372 def create_oauth_authenticator( 3373 self, model: OAuthAuthenticatorModel, config: Config, **kwargs: Any 3374 ) -> DeclarativeOauth2Authenticator: 3375 profile_assertion = ( 3376 self._create_component_from_model(model.profile_assertion, config=config) 3377 if model.profile_assertion 3378 else None 3379 ) 3380 3381 refresh_token_error_status_codes, refresh_token_error_key, refresh_token_error_values = ( 3382 self._get_refresh_token_error_information(model) 3383 ) 3384 if model.refresh_token_updater: 3385 # ignore type error because fixing it would have a lot of dependencies, revisit later 3386 return DeclarativeSingleUseRefreshTokenOauth2Authenticator( # type: ignore 3387 config, 3388 InterpolatedString.create( 3389 model.token_refresh_endpoint, # type: ignore 3390 parameters=model.parameters or {}, 3391 ).eval(config), 3392 access_token_name=InterpolatedString.create( 3393 model.access_token_name or "access_token", parameters=model.parameters or {} 3394 ).eval(config), 3395 refresh_token_name=model.refresh_token_updater.refresh_token_name, 3396 expires_in_name=InterpolatedString.create( 3397 model.expires_in_name or "expires_in", parameters=model.parameters or {} 3398 ).eval(config), 3399 client_id_name=InterpolatedString.create( 3400 model.client_id_name or "client_id", parameters=model.parameters or {} 3401 ).eval(config), 3402 client_id=InterpolatedString.create( 3403 model.client_id, parameters=model.parameters or {} 3404 ).eval(config) 3405 if model.client_id 3406 else model.client_id, 3407 client_secret_name=InterpolatedString.create( 3408 model.client_secret_name or "client_secret", parameters=model.parameters or {} 3409 ).eval(config), 3410 client_secret=InterpolatedString.create( 3411 model.client_secret, parameters=model.parameters or {} 3412 ).eval(config) 3413 if model.client_secret 3414 else model.client_secret, 3415 access_token_config_path=model.refresh_token_updater.access_token_config_path, 3416 refresh_token_config_path=model.refresh_token_updater.refresh_token_config_path, 3417 token_expiry_date_config_path=model.refresh_token_updater.token_expiry_date_config_path, 3418 grant_type_name=InterpolatedString.create( 3419 model.grant_type_name or "grant_type", parameters=model.parameters or {} 3420 ).eval(config), 3421 grant_type=InterpolatedString.create( 3422 model.grant_type or "refresh_token", parameters=model.parameters or {} 3423 ).eval(config), 3424 refresh_request_body=InterpolatedMapping( 3425 model.refresh_request_body or {}, parameters=model.parameters or {} 3426 ).eval(config), 3427 refresh_request_headers=InterpolatedMapping( 3428 model.refresh_request_headers or {}, parameters=model.parameters or {} 3429 ).eval(config), 3430 send_refresh_request_as_query_params=bool( 3431 model.send_refresh_request_as_query_params 3432 ), 3433 scopes=model.scopes, 3434 token_expiry_date_format=model.token_expiry_date_format, 3435 token_expiry_is_time_of_expiration=bool(model.token_expiry_date_format), 3436 message_repository=self._message_repository, 3437 emit_control_message_to_message_repository=self._emit_connector_builder_messages, 3438 refresh_token_error_status_codes=refresh_token_error_status_codes, 3439 refresh_token_error_key=refresh_token_error_key, 3440 refresh_token_error_values=refresh_token_error_values, 3441 refresh_token_error_messages=self._get_refresh_token_error_messages(model), 3442 ) 3443 # ignore type error because fixing it would have a lot of dependencies, revisit later 3444 return DeclarativeOauth2Authenticator( # type: ignore 3445 access_token_name=model.access_token_name or "access_token", 3446 access_token_value=model.access_token_value, 3447 client_id_name=model.client_id_name or "client_id", 3448 client_id=model.client_id, 3449 client_secret_name=model.client_secret_name or "client_secret", 3450 client_secret=model.client_secret, 3451 expires_in_name=model.expires_in_name or "expires_in", 3452 grant_type_name=model.grant_type_name or "grant_type", 3453 grant_type=model.grant_type or "refresh_token", 3454 refresh_request_body=model.refresh_request_body, 3455 refresh_request_headers=model.refresh_request_headers, 3456 send_refresh_request_as_query_params=bool(model.send_refresh_request_as_query_params), 3457 refresh_token_name=model.refresh_token_name or "refresh_token", 3458 refresh_token=model.refresh_token, 3459 scopes=model.scopes, 3460 token_expiry_date=model.token_expiry_date, 3461 token_expiry_date_format=model.token_expiry_date_format, 3462 token_expiry_is_time_of_expiration=bool(model.token_expiry_date_format), 3463 token_refresh_endpoint=model.token_refresh_endpoint, 3464 config=config, 3465 parameters=model.parameters or {}, 3466 message_repository=self._message_repository, 3467 profile_assertion=profile_assertion, 3468 use_profile_assertion=model.use_profile_assertion, 3469 refresh_token_error_status_codes=refresh_token_error_status_codes, 3470 refresh_token_error_key=refresh_token_error_key, 3471 refresh_token_error_values=refresh_token_error_values, 3472 refresh_token_error_messages=self._get_refresh_token_error_messages(model), 3473 ) 3474 3475 @staticmethod 3476 def _get_refresh_token_error_information( 3477 model: OAuthAuthenticatorModel, 3478 ) -> Tuple[Tuple[int, ...], str, Tuple[str, ...]]: 3479 """ 3480 In a previous version of the CDK, the auth error as config_error was only done if a refresh token updater was 3481 defined. As a transition, we added those fields on the OAuthAuthenticatorModel. This method ensures that the 3482 information is defined only once and return the right fields. 3483 """ 3484 refresh_token_updater = model.refresh_token_updater 3485 is_defined_on_refresh_token_updated = refresh_token_updater and ( 3486 refresh_token_updater.refresh_token_error_status_codes 3487 or refresh_token_updater.refresh_token_error_key 3488 or refresh_token_updater.refresh_token_error_values 3489 ) 3490 is_defined_on_oauth_authenticator = ( 3491 model.refresh_token_error_status_codes 3492 or model.refresh_token_error_key 3493 or model.refresh_token_error_values 3494 ) 3495 if is_defined_on_refresh_token_updated and is_defined_on_oauth_authenticator: 3496 raise ValueError( 3497 "refresh_token_error should either be defined on the OAuthAuthenticatorModel or the RefreshTokenUpdaterModel, not both" 3498 ) 3499 3500 if is_defined_on_refresh_token_updated: 3501 not_optional_refresh_token_updater: RefreshTokenUpdaterModel = refresh_token_updater # type: ignore # we know from the condition that this is not None 3502 return ( 3503 tuple(not_optional_refresh_token_updater.refresh_token_error_status_codes) 3504 if not_optional_refresh_token_updater.refresh_token_error_status_codes 3505 else (), 3506 not_optional_refresh_token_updater.refresh_token_error_key or "", 3507 tuple(not_optional_refresh_token_updater.refresh_token_error_values) 3508 if not_optional_refresh_token_updater.refresh_token_error_values 3509 else (), 3510 ) 3511 elif is_defined_on_oauth_authenticator: 3512 return ( 3513 tuple(model.refresh_token_error_status_codes) 3514 if model.refresh_token_error_status_codes 3515 else (), 3516 model.refresh_token_error_key or "", 3517 tuple(model.refresh_token_error_values) if model.refresh_token_error_values else (), 3518 ) 3519 3520 # returning default values we think cover most cases 3521 return (400,), "error", ("invalid_grant", "invalid_permissions") 3522 3523 @staticmethod 3524 def _get_refresh_token_error_messages(model: OAuthAuthenticatorModel) -> Dict[str, str]: 3525 """Resolve messages separately so they never disable the default gate.""" 3526 return dict(model.refresh_token_error_messages or {}) 3527 3528 @staticmethod 3529 def _reject_union_combined_extractor_for_offset_increment( 3530 extractor_model: Optional[BaseModel], 3531 ) -> None: 3532 """Refuse an `OffsetIncrement` paginator driven by a `union` `CombinedExtractor`. 3533 3534 `OffsetIncrement.next_page_token` advances the offset by the number of records its 3535 extractor returns for the page. Under `union` that number is the sum over all 3536 sub-extractors, so the offset overshoots the API page size and every page after the first 3537 starts past the records that were never read: with two sub-extractors returning two records 3538 each and `page_size: 2`, the requested offsets are 0, 4, 8 instead of 0, 2, 4 and two 3539 thirds of the records are silently dropped. 3540 3541 `first_match` returns the winning sub-extractor's count, which does not inflate the count, 3542 and is left alone. A `union` nested anywhere in the tree inflates the count of the node 3543 above it, so the whole tree is walked. 3544 """ 3545 if not isinstance(extractor_model, CombinedExtractorModel): 3546 return 3547 if not ModelToComponentFactory._combined_extractor_tree_contains_union(extractor_model): 3548 return 3549 raise AirbyteTracedException( 3550 message=( 3551 'CombinedExtractor mode "union" is not supported with an OffsetIncrement paginator.' 3552 ), 3553 internal_message=( 3554 "OffsetIncrement counts the records of its extractor to advance the offset. A " 3555 "`union` CombinedExtractor returns the sum of its sub-extractors' records, which " 3556 "overshoots the page the API returned, so records would be skipped. Use the " 3557 "`first_match` mode, a CursorPagination or PageIncrement paginator, or a single " 3558 "extractor." 3559 ), 3560 failure_type=FailureType.config_error, 3561 ) 3562 3563 @staticmethod 3564 def _combined_extractor_tree_contains_union(model: CombinedExtractorModel) -> bool: 3565 mode = CombineMode(model.mode.value) if model.mode else CombineMode.union 3566 if mode == CombineMode.union: 3567 return True 3568 return any( 3569 isinstance(sub_extractor, CombinedExtractorModel) 3570 and ModelToComponentFactory._combined_extractor_tree_contains_union(sub_extractor) 3571 for sub_extractor in model.extractors 3572 ) 3573 3574 @staticmethod 3575 def _count_dropped_empty_records(extractor: RecordExtractor) -> None: 3576 """Make the record selector count the empty records a `CombinedExtractor` drops. 3577 3578 `OffsetIncrement` and `PageIncrement` stop when that count falls below the page size, so a 3579 full page holding a null would otherwise look short and silently end pagination. 3580 """ 3581 if not isinstance(extractor, CombinedExtractor): 3582 return 3583 extractor.count_dropped_empty_records = True 3584 for sub_extractor in extractor.extractors: 3585 ModelToComponentFactory._count_dropped_empty_records(sub_extractor) 3586 3587 def create_offset_increment( 3588 self, 3589 model: OffsetIncrementModel, 3590 config: Config, 3591 decoder: Optional[Decoder] = None, 3592 extractor_model: Optional[ 3593 Union[CustomRecordExtractorModel, DpathExtractorModel, CombinedExtractorModel] 3594 ] = None, 3595 **kwargs: Any, 3596 ) -> OffsetIncrement: 3597 # None when nested under a custom component, which cannot pass a decoder 3598 if decoder is None: 3599 decoder = JsonDecoder(parameters={}) 3600 if isinstance(decoder, PaginationDecoderDecorator): 3601 inner_decoder = decoder.decoder 3602 else: 3603 inner_decoder = decoder 3604 decoder = PaginationDecoderDecorator(decoder=decoder) 3605 3606 if self._is_supported_decoder_for_pagination(inner_decoder): 3607 decoder_to_use = decoder 3608 else: 3609 raise ValueError( 3610 self._UNSUPPORTED_DECODER_ERROR.format(decoder_type=type(inner_decoder)) 3611 ) 3612 3613 self._reject_union_combined_extractor_for_offset_increment(extractor_model) 3614 3615 # Pydantic v1 Union type coercion can convert int to string depending on Union order. 3616 # If page_size is a string that represents an integer (not an interpolation), convert it back. 3617 page_size = model.page_size 3618 if isinstance(page_size, str) and page_size.isdigit(): 3619 page_size = int(page_size) 3620 3621 return OffsetIncrement( 3622 page_size=page_size, 3623 config=config, 3624 decoder=decoder_to_use, 3625 inject_on_first_request=model.inject_on_first_request or False, 3626 parameters=model.parameters or {}, 3627 ) 3628 3629 @staticmethod 3630 def create_page_increment( 3631 model: PageIncrementModel, config: Config, **kwargs: Any 3632 ) -> PageIncrement: 3633 # Pydantic v1 Union type coercion can convert int to string depending on Union order. 3634 # If page_size is a string that represents an integer (not an interpolation), convert it back. 3635 page_size = model.page_size 3636 if isinstance(page_size, str) and page_size.isdigit(): 3637 page_size = int(page_size) 3638 3639 return PageIncrement( 3640 page_size=page_size, 3641 config=config, 3642 start_from_page=model.start_from_page or 0, 3643 inject_on_first_request=model.inject_on_first_request or False, 3644 parameters=model.parameters or {}, 3645 ) 3646 3647 def create_parent_stream_config( 3648 self, model: ParentStreamConfigModel, config: Config, *, stream_name: str, **kwargs: Any 3649 ) -> ParentStreamConfig: 3650 declarative_stream = self._create_component_from_model( 3651 model.stream, 3652 config=config, 3653 is_parent=True, 3654 **kwargs, 3655 ) 3656 request_option = ( 3657 self._create_component_from_model(model.request_option, config=config) 3658 if model.request_option 3659 else None 3660 ) 3661 3662 if model.lazy_read_pointer and any("*" in pointer for pointer in model.lazy_read_pointer): 3663 raise ValueError( 3664 "The '*' wildcard in 'lazy_read_pointer' is not supported — only direct paths are allowed." 3665 ) 3666 3667 model_lazy_read_pointer: List[Union[InterpolatedString, str]] = ( 3668 [x for x in model.lazy_read_pointer] if model.lazy_read_pointer else [] 3669 ) 3670 3671 return ParentStreamConfig( 3672 parent_key=model.parent_key, 3673 request_option=request_option, 3674 stream=declarative_stream, 3675 partition_field=model.partition_field, 3676 config=config, 3677 incremental_dependency=model.incremental_dependency or False, 3678 parameters=model.parameters or {}, 3679 extra_fields=model.extra_fields, 3680 lazy_read_pointer=model_lazy_read_pointer, 3681 ) 3682 3683 def create_properties_from_endpoint( 3684 self, model: PropertiesFromEndpointModel, config: Config, **kwargs: Any 3685 ) -> PropertiesFromEndpoint: 3686 retriever = self._create_component_from_model( 3687 model=model.retriever, 3688 config=config, 3689 name="dynamic_properties", 3690 primary_key=None, 3691 stream_slicer=None, 3692 transformations=[], 3693 use_cache=True, # Enable caching on the HttpRequester/HttpClient because the properties endpoint will be called for every slice being processed, and it is highly unlikely for the response to different 3694 ) 3695 return PropertiesFromEndpoint( 3696 property_field_path=model.property_field_path, 3697 retriever=retriever, 3698 config=config, 3699 parameters=model.parameters or {}, 3700 ) 3701 3702 def create_property_chunking( 3703 self, model: PropertyChunkingModel, config: Config, **kwargs: Any 3704 ) -> PropertyChunking: 3705 record_merge_strategy = ( 3706 self._create_component_from_model( 3707 model=model.record_merge_strategy, config=config, **kwargs 3708 ) 3709 if model.record_merge_strategy 3710 else None 3711 ) 3712 3713 property_limit_type: PropertyLimitType 3714 match model.property_limit_type: 3715 case PropertyLimitTypeModel.property_count: 3716 property_limit_type = PropertyLimitType.property_count 3717 case PropertyLimitTypeModel.characters: 3718 property_limit_type = PropertyLimitType.characters 3719 case _: 3720 raise ValueError(f"Invalid PropertyLimitType {property_limit_type}") 3721 3722 return PropertyChunking( 3723 property_limit_type=property_limit_type, 3724 property_limit=model.property_limit, 3725 record_merge_strategy=record_merge_strategy, 3726 config=config, 3727 parameters=model.parameters or {}, 3728 ) 3729 3730 def create_query_properties( 3731 self, model: QueryPropertiesModel, config: Config, *, stream_name: str, **kwargs: Any 3732 ) -> QueryProperties: 3733 if isinstance(model.property_list, list): 3734 property_list = model.property_list 3735 else: 3736 property_list = self._create_component_from_model( 3737 model=model.property_list, config=config, **kwargs 3738 ) 3739 3740 property_chunking = ( 3741 self._create_component_from_model( 3742 model=model.property_chunking, config=config, **kwargs 3743 ) 3744 if model.property_chunking 3745 else None 3746 ) 3747 3748 property_selector = ( 3749 self._create_component_from_model( 3750 model=model.property_selector, config=config, stream_name=stream_name, **kwargs 3751 ) 3752 if model.property_selector 3753 else None 3754 ) 3755 3756 return QueryProperties( 3757 property_list=property_list, 3758 always_include_properties=model.always_include_properties, 3759 property_chunking=property_chunking, 3760 property_selector=property_selector, 3761 config=config, 3762 parameters=model.parameters or {}, 3763 ) 3764 3765 def create_json_schema_property_selector( 3766 self, 3767 model: JsonSchemaPropertySelectorModel, 3768 config: Config, 3769 *, 3770 stream_name: str, 3771 **kwargs: Any, 3772 ) -> JsonSchemaPropertySelector: 3773 configured_stream = self._stream_name_to_configured_stream.get(stream_name) 3774 3775 transformations = [] 3776 if model.transformations: 3777 for transformation_model in model.transformations: 3778 transformations.append( 3779 self._create_component_from_model(model=transformation_model, config=config) 3780 ) 3781 3782 return JsonSchemaPropertySelector( 3783 configured_stream=configured_stream, 3784 properties_transformations=transformations, 3785 config=config, 3786 parameters=model.parameters or {}, 3787 ) 3788 3789 @staticmethod 3790 def create_record_filter( 3791 model: RecordFilterModel, config: Config, **kwargs: Any 3792 ) -> RecordFilter: 3793 return RecordFilter( 3794 condition=model.condition or "", config=config, parameters=model.parameters or {} 3795 ) 3796 3797 @staticmethod 3798 def create_request_path(model: RequestPathModel, config: Config, **kwargs: Any) -> RequestPath: 3799 return RequestPath(parameters={}) 3800 3801 @staticmethod 3802 def create_request_option( 3803 model: RequestOptionModel, config: Config, **kwargs: Any 3804 ) -> RequestOption: 3805 inject_into = RequestOptionType(model.inject_into.value) 3806 field_path: Optional[List[Union[InterpolatedString, str]]] = ( 3807 [ 3808 InterpolatedString.create(segment, parameters=kwargs.get("parameters", {})) 3809 for segment in model.field_path 3810 ] 3811 if model.field_path 3812 else None 3813 ) 3814 field_name = ( 3815 InterpolatedString.create(model.field_name, parameters=kwargs.get("parameters", {})) 3816 if model.field_name 3817 else None 3818 ) 3819 return RequestOption( 3820 field_name=field_name, 3821 field_path=field_path, 3822 inject_into=inject_into, 3823 parameters=kwargs.get("parameters", {}), 3824 ) 3825 3826 def create_record_selector( 3827 self, 3828 model: RecordSelectorModel, 3829 config: Config, 3830 *, 3831 name: str, 3832 transformations: List[RecordTransformation] | None = None, 3833 decoder: Decoder | None = None, 3834 client_side_incremental_sync_cursor: Optional[Cursor] = None, 3835 is_client_side_incremental_sync: bool = False, 3836 file_uploader: Optional[DefaultFileUploader] = None, 3837 **kwargs: Any, 3838 ) -> RecordSelector: 3839 extractor = self._create_component_from_model( 3840 model=model.extractor, decoder=decoder, config=config 3841 ) 3842 self._count_dropped_empty_records(extractor) 3843 record_filter = ( 3844 self._create_component_from_model(model.record_filter, config=config) 3845 if model.record_filter 3846 else None 3847 ) 3848 3849 # A client-side incremental stream transforms before filtering by default. That default belongs to the flag, 3850 # not to the component that ends up doing the cursor comparison: a data feed does it in the retriever and 3851 # receives no cursor here, but its `record_filter` condition must keep running after the transformations. 3852 default_transform_before_filtering = bool( 3853 client_side_incremental_sync_cursor or is_client_side_incremental_sync 3854 ) 3855 transform_before_filtering = ( 3856 default_transform_before_filtering 3857 if model.transform_before_filtering is None 3858 else model.transform_before_filtering 3859 ) 3860 if client_side_incremental_sync_cursor: 3861 record_filter = ClientSideIncrementalRecordFilterDecorator( 3862 config=config, 3863 parameters=model.parameters, 3864 condition=model.record_filter.condition 3865 if (model.record_filter and hasattr(model.record_filter, "condition")) 3866 else None, 3867 cursor=client_side_incremental_sync_cursor, 3868 ) 3869 3870 if model.schema_normalization is None: 3871 # default to no schema normalization if not set 3872 model.schema_normalization = SchemaNormalizationModel.None_ 3873 3874 schema_normalization = ( 3875 TypeTransformer(SCHEMA_TRANSFORMER_TYPE_MAPPING[model.schema_normalization]) 3876 if isinstance(model.schema_normalization, SchemaNormalizationModel) 3877 else self._create_component_from_model(model.schema_normalization, config=config) # type: ignore[arg-type] # custom normalization model expected here 3878 ) 3879 3880 return RecordSelector( 3881 extractor=extractor, 3882 name=name, 3883 config=config, 3884 record_filter=record_filter, 3885 transformations=transformations or [], 3886 file_uploader=file_uploader, 3887 schema_normalization=schema_normalization, 3888 parameters=model.parameters or {}, 3889 transform_before_filtering=transform_before_filtering, 3890 ) 3891 3892 @staticmethod 3893 def create_remove_fields( 3894 model: RemoveFieldsModel, config: Config, **kwargs: Any 3895 ) -> RemoveFields: 3896 return RemoveFields( 3897 field_pointers=model.field_pointers, condition=model.condition or "", parameters={} 3898 ) 3899 3900 def create_selective_authenticator( 3901 self, 3902 model: SelectiveAuthenticatorModel, 3903 config: Config, 3904 *, 3905 name: Optional[str] = None, 3906 url_base: Optional[str] = None, 3907 **kwargs: Any, 3908 ) -> DeclarativeAuthenticator: 3909 # Keyword-only so that _create_nested_component also fills them under a custom component, 3910 # from the parent's kwargs or $parameters. None is not forwarded, so a missing required one 3911 # keeps its hint. 3912 nested_kwargs = { 3913 key: value 3914 for key, value in {"name": name, "url_base": url_base}.items() 3915 if value is not None 3916 } 3917 authenticators = { 3918 key: self._create_component_from_model( 3919 model=auth, config=config, **nested_kwargs, **kwargs 3920 ) 3921 for key, auth in model.authenticators.items() 3922 } 3923 # SelectiveAuthenticator will return instance of DeclarativeAuthenticator or raise ValueError error 3924 return SelectiveAuthenticator( # type: ignore[abstract] 3925 config=config, 3926 authenticators=authenticators, 3927 authenticator_selection_path=model.authenticator_selection_path, 3928 **kwargs, 3929 ) 3930 3931 @staticmethod 3932 def create_legacy_session_token_authenticator( 3933 model: LegacySessionTokenAuthenticatorModel, config: Config, *, url_base: str, **kwargs: Any 3934 ) -> LegacySessionTokenAuthenticator: 3935 return LegacySessionTokenAuthenticator( 3936 api_url=url_base, 3937 header=model.header, 3938 login_url=model.login_url, 3939 password=model.password or "", 3940 session_token=model.session_token or "", 3941 session_token_response_key=model.session_token_response_key or "", 3942 username=model.username or "", 3943 validate_session_url=model.validate_session_url, 3944 config=config, 3945 parameters=model.parameters or {}, 3946 ) 3947 3948 def create_simple_retriever( 3949 self, 3950 model: SimpleRetrieverModel, 3951 config: Config, 3952 *, 3953 name: str, 3954 primary_key: Optional[Union[str, List[str], List[List[str]]]], 3955 request_options_provider: Optional[RequestOptionsProvider] = None, 3956 cursor: Optional[Cursor] = None, 3957 has_stop_condition_cursor: bool = False, 3958 is_client_side_incremental_sync: bool = False, 3959 transformations: List[RecordTransformation], 3960 file_uploader: Optional[DefaultFileUploader] = None, 3961 incremental_sync: Optional[ 3962 Union[ 3963 IncrementingCountCursorModel, 3964 DatetimeBasedCursorModel, 3965 SyncTokenCursorModel, 3966 ] 3967 ] = None, 3968 use_cache: Optional[bool] = None, 3969 log_formatter: Optional[Callable[[Response], Any]] = None, 3970 partition_router: Optional[PartitionRouter] = None, 3971 **kwargs: Any, 3972 ) -> SimpleRetriever: 3973 def _get_url(req: Requester) -> str: 3974 """ 3975 Closure to get the URL from the requester. This is used to get the URL in the case of a lazy retriever. 3976 This is needed because the URL is not set until the requester is created. 3977 """ 3978 3979 _url: str = ( 3980 model.requester.url 3981 if hasattr(model.requester, "url") and model.requester.url is not None 3982 else req.get_url(stream_state=None, stream_slice=None, next_page_token=None) 3983 ) 3984 _url_base: str = ( 3985 model.requester.url_base 3986 if hasattr(model.requester, "url_base") and model.requester.url_base is not None 3987 else req.get_url_base(stream_state=None, stream_slice=None, next_page_token=None) 3988 ) 3989 3990 return _url or _url_base 3991 3992 if cursor is None: 3993 cursor = FinalStateCursor(name, None, self._message_repository) 3994 3995 # A data feed drops the records the cursor considers already synced in the retriever, which sits downstream of 3996 # the paginator. Letting the record selector drop them as well would be redundant and would hide them from the 3997 # pagination stop condition, so a data feed never delegates that filtering to the record selector, whether 3998 # `is_client_side_incremental` is set or not. The `condition` from `record_filter` is intentionally left out of 3999 # the post-pagination filter and stays in the record selector, which preserves the existing behaviour: the 4000 # selector runs inside the page loop, so the records the condition rejects never reach the paginator's 4001 # accounting. Moving it downstream would start counting them. 4002 post_pagination_filter = ( 4003 ClientSideIncrementalRecordFilterDecorator( 4004 config=config, 4005 parameters=model.parameters or {}, 4006 condition=None, 4007 cursor=cursor, 4008 ) 4009 if has_stop_condition_cursor 4010 else None 4011 ) 4012 client_side_incremental_cursor = ( 4013 cursor if is_client_side_incremental_sync and not post_pagination_filter else None 4014 ) 4015 if post_pagination_filter and is_client_side_incremental_sync: 4016 LOGGER.warning( 4017 f"Stream {name}: `is_client_side_incremental` adds no record filtering when `is_data_feed` is set, " 4018 "as a data feed already filters on the cursor value. It still makes the record selector apply the " 4019 "transformations before the `record_filter` condition." 4020 ) 4021 4022 decoder = ( 4023 cursor.decoder 4024 if isinstance(cursor, SyncTokenCursor) 4025 else ( 4026 self._create_component_from_model(model=model.decoder, config=config) 4027 if model.decoder 4028 else JsonDecoder(parameters={}) 4029 ) 4030 ) 4031 record_selector = self._create_component_from_model( 4032 model=model.record_selector, 4033 name=name, 4034 config=config, 4035 decoder=decoder, 4036 transformations=transformations, 4037 client_side_incremental_sync_cursor=client_side_incremental_cursor, 4038 is_client_side_incremental_sync=is_client_side_incremental_sync, 4039 file_uploader=file_uploader, 4040 ) 4041 4042 query_properties: Optional[QueryProperties] = None 4043 query_properties_key: Optional[str] = None 4044 self._ensure_query_properties_to_model(model.requester) 4045 if self._has_query_properties_in_request_parameters(model.requester): 4046 # It is better to be explicit about an error if PropertiesFromEndpoint is defined in multiple 4047 # places instead of default to request_parameters which isn't clearly documented 4048 if ( 4049 hasattr(model.requester, "fetch_properties_from_endpoint") 4050 and model.requester.fetch_properties_from_endpoint 4051 ): 4052 raise ValueError( 4053 f"PropertiesFromEndpoint should only be specified once per stream, but found in {model.requester.type}.fetch_properties_from_endpoint and {model.requester.type}.request_parameters" 4054 ) 4055 4056 query_properties_definitions = [] 4057 for key, request_parameter in model.requester.request_parameters.items(): # type: ignore # request_parameters is already validated to be a Mapping using _has_query_properties_in_request_parameters() 4058 if isinstance(request_parameter, QueryPropertiesModel): 4059 query_properties_key = key 4060 query_properties_definitions.append(request_parameter) 4061 4062 if len(query_properties_definitions) > 1: 4063 raise ValueError( 4064 f"request_parameters only supports defining one QueryProperties field, but found {len(query_properties_definitions)} usages" 4065 ) 4066 4067 if len(query_properties_definitions) == 1: 4068 query_properties = self._create_component_from_model( 4069 model=query_properties_definitions[0], stream_name=name, config=config 4070 ) 4071 4072 # Removes QueryProperties components from the interpolated mappings because it has been designed 4073 # to be used by the SimpleRetriever and will be resolved from the provider from the slice directly 4074 # instead of through jinja interpolation 4075 if hasattr(model.requester, "request_parameters") and isinstance( 4076 model.requester.request_parameters, Mapping 4077 ): 4078 model.requester.request_parameters = self._remove_query_properties( 4079 model.requester.request_parameters 4080 ) 4081 elif ( 4082 hasattr(model.requester, "fetch_properties_from_endpoint") 4083 and model.requester.fetch_properties_from_endpoint 4084 ): 4085 # todo: Deprecate this condition once dependent connectors migrate to query_properties 4086 query_properties_definition = QueryPropertiesModel( 4087 type="QueryProperties", 4088 property_list=model.requester.fetch_properties_from_endpoint, 4089 always_include_properties=None, 4090 property_chunking=None, 4091 ) # type: ignore # $parameters has a default value 4092 4093 query_properties = self.create_query_properties( 4094 model=query_properties_definition, 4095 stream_name=name, 4096 config=config, 4097 ) 4098 elif hasattr(model.requester, "query_properties") and model.requester.query_properties: 4099 query_properties = self.create_query_properties( 4100 model=model.requester.query_properties, 4101 stream_name=name, 4102 config=config, 4103 ) 4104 4105 requester = self._create_component_from_model( 4106 model=model.requester, 4107 decoder=decoder, 4108 name=name, 4109 query_properties_key=query_properties_key, 4110 use_cache=use_cache, 4111 config=config, 4112 ) 4113 4114 if not request_options_provider: 4115 request_options_provider = DefaultRequestOptionsProvider(parameters={}) 4116 if isinstance(request_options_provider, DefaultRequestOptionsProvider) and isinstance( 4117 partition_router, PartitionRouter 4118 ): 4119 request_options_provider = partition_router 4120 4121 paginator = ( 4122 self._create_component_from_model( 4123 model=model.paginator, 4124 config=config, 4125 url_base=_get_url(requester), 4126 extractor_model=model.record_selector.extractor, 4127 decoder=decoder, 4128 cursor_used_for_stop_condition=cursor if has_stop_condition_cursor else None, 4129 ) 4130 if model.paginator 4131 else NoPagination(parameters={}) 4132 ) 4133 4134 ignore_stream_slicer_parameters_on_paginated_requests = ( 4135 model.ignore_stream_slicer_parameters_on_paginated_requests or False 4136 ) 4137 4138 reads_parent_stream_lazily = bool( 4139 model.partition_router 4140 and isinstance(model.partition_router, SubstreamPartitionRouterModel) 4141 and any( 4142 parent_stream_config.lazy_read_pointer 4143 for parent_stream_config in model.partition_router.parent_stream_configs 4144 ) 4145 ) 4146 if reads_parent_stream_lazily and ( 4147 model.page_size_reduction 4148 or self._uses_reduce_page_size_action(getattr(model.requester, "error_handler", None)) 4149 ): 4150 # Checked outside of the LazySimpleRetriever branch below, which only applies on the first 4151 # sync of a stream: gating it on the absence of state would accept the same manifest from the 4152 # second sync onwards. LazySimpleRetriever paginates the parent's embedded pages, so there is 4153 # no page of its own to re-issue with a smaller page size. 4154 raise ValueError( 4155 f"`page_size_reduction` and the REDUCE_PAGE_SIZE response action are not supported when " 4156 f"reading a parent stream lazily. Remove either the page size reduction or the parent " 4157 f"stream's `lazy_read_pointer` for stream {name}." 4158 ) 4159 4160 if reads_parent_stream_lazily and ( 4161 model.request_window_splitting 4162 or self._uses_split_request_window_action( 4163 getattr(model.requester, "error_handler", None) 4164 ) 4165 ): 4166 # Same reasoning as the page_size_reduction check above: LazySimpleRetriever reads records 4167 # embedded in the parent's own pages rather than requesting a window of its own, so there is 4168 # nothing here for a split child window to be read from. 4169 raise ValueError( 4170 f"`request_window_splitting` and the SPLIT_REQUEST_WINDOW response action are not " 4171 f"supported when reading a parent stream lazily. Remove either the request window " 4172 f"splitting or the parent stream's `lazy_read_pointer` for stream {name}." 4173 ) 4174 4175 if reads_parent_stream_lazily and not bool( 4176 self._connector_state_manager.get_stream_state(name, None) 4177 ): 4178 if incremental_sync: 4179 if incremental_sync.type != "DatetimeBasedCursor": 4180 raise ValueError( 4181 f"LazySimpleRetriever only supports DatetimeBasedCursor. Found: {incremental_sync.type}." 4182 ) 4183 4184 elif incremental_sync.step or incremental_sync.cursor_granularity: 4185 raise ValueError( 4186 f"Found more that one slice per parent. LazySimpleRetriever only supports single slice read for stream - {name}." 4187 ) 4188 4189 if model.decoder and model.decoder.type != "JsonDecoder": 4190 raise ValueError( 4191 f"LazySimpleRetriever only supports JsonDecoder. Found: {model.decoder.type}." 4192 ) 4193 4194 return LazySimpleRetriever( 4195 name=name, 4196 paginator=paginator, 4197 primary_key=primary_key, 4198 requester=requester, 4199 record_selector=record_selector, 4200 stream_slicer=_NO_STREAM_SLICING, 4201 request_option_provider=request_options_provider, 4202 config=config, 4203 ignore_stream_slicer_parameters_on_paginated_requests=ignore_stream_slicer_parameters_on_paginated_requests, 4204 post_pagination_filter=post_pagination_filter, 4205 parameters=model.parameters or {}, 4206 ) 4207 4208 if ( 4209 model.record_selector.record_filter 4210 and model.pagination_reset 4211 and model.pagination_reset.limits 4212 ): 4213 raise ValueError("PaginationResetLimits are not supported while having record filter.") 4214 4215 request_window_splitting = self._create_request_window_splitting( 4216 model, name, cursor, incremental_sync, query_properties, file_uploader 4217 ) 4218 return SimpleRetriever( 4219 name=name, 4220 paginator=paginator, 4221 primary_key=primary_key, 4222 requester=requester, 4223 record_selector=record_selector, 4224 stream_slicer=_NO_STREAM_SLICING, 4225 request_option_provider=request_options_provider, 4226 config=config, 4227 ignore_stream_slicer_parameters_on_paginated_requests=ignore_stream_slicer_parameters_on_paginated_requests, 4228 additional_query_properties=query_properties, 4229 log_formatter=self._get_log_formatter(log_formatter, name), 4230 pagination_tracker_factory=self._create_pagination_tracker_factory( 4231 model.pagination_reset, cursor 4232 ), 4233 page_size_reduction=self._create_page_size_reduction( 4234 model, name, query_properties, file_uploader 4235 ), 4236 request_window_splitting=request_window_splitting, 4237 request_window_splitter=( 4238 cursor.split_request_window 4239 if request_window_splitting and hasattr(cursor, "split_request_window") 4240 else None 4241 ), 4242 post_pagination_filter=post_pagination_filter, 4243 page_observer=( 4244 cursor.on_page_completed if isinstance(cursor, SyncTokenCursor) else None 4245 ), 4246 parameters=model.parameters or {}, 4247 ) 4248 4249 def _create_page_size_reduction( 4250 self, 4251 model: SimpleRetrieverModel, 4252 name: str, 4253 query_properties: Optional[QueryProperties], 4254 file_uploader: Optional[DefaultFileUploader] = None, 4255 ) -> Optional[PageSizeReduction]: 4256 # A CustomRequester does not necessarily define an error handler. A CustomRequester that does define 4257 # one keeps it as a raw dict rather than a typed model, so this returns False for it as well. 4258 error_handler = getattr(model.requester, "error_handler", None) 4259 uses_action = self._uses_reduce_page_size_action(error_handler) 4260 if uses_action and not model.page_size_reduction: 4261 raise ValueError( 4262 f"Stream {name} has a response filter with the REDUCE_PAGE_SIZE action but the retriever does not " 4263 f"define `page_size_reduction`. Add a `page_size_reduction` block to the retriever." 4264 ) 4265 4266 if not model.page_size_reduction: 4267 return None 4268 4269 if not uses_action: 4270 # Not raised: a CustomErrorHandler can resolve to REDUCE_PAGE_SIZE without us being able to see 4271 # it, so the only safe reaction to a block we cannot tie to an action is a warning. Without it a 4272 # misspelled action leaves the feature silently dead on a stream that only exists because it 4273 # would otherwise fail. 4274 LOGGER.warning( 4275 f"Stream {name} defines `page_size_reduction` but no response filter with the " 4276 f"REDUCE_PAGE_SIZE action was found on its requester. The page size will never be reduced " 4277 f"unless a custom error handler resolves to that action." 4278 ) 4279 4280 self._validate_page_size_reduction_is_supported( 4281 model, name, query_properties, file_uploader 4282 ) 4283 4284 reset_policy = model.page_size_reduction.reset_policy 4285 return PageSizeReduction( 4286 reduction_factor=model.page_size_reduction.reduction_factor, # type: ignore[arg-type] # the schema defines a default 4287 minimum_page_size=model.page_size_reduction.minimum_page_size, # type: ignore[arg-type] # the schema defines a default 4288 max_attempts=model.page_size_reduction.max_attempts, # type: ignore[arg-type] # the schema defines a default 4289 backoff_seconds=model.page_size_reduction.backoff_seconds, # type: ignore[arg-type] # the schema defines a default 4290 retries_at_minimum_page_size=model.page_size_reduction.retries_at_minimum_page_size, # type: ignore[arg-type] # the schema defines a default 4291 failure_message=model.page_size_reduction.failure_message, 4292 reset_policy=PageSizeResetPolicy(reset_policy.value) 4293 if reset_policy is not None 4294 else PageSizeResetPolicy.NEVER, 4295 ) 4296 4297 def _validate_page_size_reduction_is_supported( 4298 self, 4299 model: SimpleRetrieverModel, 4300 name: str, 4301 query_properties: Optional[QueryProperties], 4302 file_uploader: Optional[DefaultFileUploader] = None, 4303 ) -> None: 4304 """ 4305 Page size reduction re-issues the same page with a smaller page size. That is only correct when the next 4306 page does not depend on the page size, and it only has an effect when the paginator injects the page size 4307 in the request. A custom pagination strategy is accepted when it can receive the reduced page size, which 4308 is checked by inspecting its signature rather than by recognizing its type. 4309 """ 4310 if query_properties: 4311 raise ValueError( 4312 f"`page_size_reduction` cannot be used together with query properties on stream {name}. Records " 4313 f"from the earlier property chunks have already been emitted when a chunk asks for a smaller page, " 4314 f"so retrying the page would emit them twice." 4315 ) 4316 4317 if file_uploader: 4318 raise ValueError( 4319 f"`page_size_reduction` cannot be used together with a `file_uploader` on stream {name}. The " 4320 f"file uploader sends one request per record from inside the page's record generator, so a " 4321 f"reduction asked for halfway through a page would re-emit the records already yielded by it." 4322 ) 4323 4324 if not isinstance(model.paginator, DefaultPaginatorModel): 4325 raise ValueError( 4326 f"`page_size_reduction` requires a DefaultPaginator on stream {name} so that the connector can " 4327 f"send a smaller page size." 4328 ) 4329 4330 if not model.paginator.page_size_option: 4331 raise ValueError( 4332 f"`page_size_reduction` requires `page_size_option` on the paginator of stream {name}: without it " 4333 f"the connector cannot tell the API to send a smaller page." 4334 ) 4335 4336 if isinstance(model.paginator.page_token_option, RequestPathModel): 4337 # A RequestPath page token is a full URL built by the API, and it already carries the page size the 4338 # API echoed back. The reduced page size is injected as a request option on top of that URL, so the 4339 # request goes out with the page size twice - the original one from the URL and the reduced one - 4340 # and which of the two the API honors is up to the API. Every page after the first would then keep 4341 # asking for the page size that just failed. 4342 raise ValueError( 4343 f"`page_size_reduction` does not support a `page_token_option` of type RequestPath on stream " 4344 f"{name}. The next page is then requested through a URL returned by the API, which already " 4345 f"carries the page size, so the reduced page size would be sent alongside the original one. Use " 4346 f"a CursorPagination strategy with a `page_token_option` of type RequestOption instead." 4347 ) 4348 4349 strategy = model.paginator.pagination_strategy 4350 if isinstance(strategy, PageIncrementModel): 4351 raise ValueError( 4352 f"`page_size_reduction` does not support the PageIncrement pagination strategy used by stream " 4353 f"{name}. Pages are addressed as page number * page size, so a smaller page size shifts every " 4354 f"following page boundary and would skip records. Use OffsetIncrement or CursorPagination." 4355 ) 4356 if isinstance(strategy, CustomPaginationStrategyModel): 4357 # A custom strategy is written by the same person enabling the reduction, so the 4358 # question is not whether we recognize it but whether it can be told the reduced 4359 # page size. Checking the signature keeps a strategy that would raise TypeError 4360 # mid-sync from being accepted at config time. 4361 custom_class = self._get_class_from_fully_qualified_class_name(strategy.class_name) 4362 if isinstance(custom_class, type) and issubclass(custom_class, PageIncrement): 4363 # `PageIncrement.next_page_token` declares `page_size_override` only to reject it, so a 4364 # subclass that does not override the method would pass the signature check below and only 4365 # fail once the first reduction is requested, mid-sync. 4366 raise ValueError( 4367 f"`page_size_reduction` does not support the PageIncrement pagination strategy that the " 4368 f"custom pagination strategy {strategy.class_name} used by stream {name} inherits from. " 4369 f"Pages are addressed as page number * page size, so a smaller page size shifts every " 4370 f"following page boundary and would skip records. Use OffsetIncrement or CursorPagination." 4371 ) 4372 try: 4373 parameters = inspect.signature(custom_class.next_page_token).parameters 4374 except (AttributeError, TypeError, ValueError) as exception: 4375 raise ValueError( 4376 f"`page_size_reduction` could not check the signature of `next_page_token` on the custom " 4377 f"pagination strategy {strategy.class_name} used by stream {name}: {exception}. Make sure " 4378 f"`class_name` points at a PaginationStrategy subclass whose `next_page_token` accepts a " 4379 f"`page_size_override` keyword argument." 4380 ) 4381 accepts_override = "page_size_override" in parameters or any( 4382 parameter.kind == inspect.Parameter.VAR_KEYWORD for parameter in parameters.values() 4383 ) 4384 if not accepts_override: 4385 raise ValueError( 4386 f"`page_size_reduction` requires the custom pagination strategy " 4387 f"{strategy.class_name} used by stream {name} to accept a `page_size_override` keyword " 4388 f"argument in `next_page_token`, so that it can honor the reduced page size. Add " 4389 f"`page_size_override: Optional[int] = None` to its signature; a strategy that does not " 4390 f"use its page size as a stop condition can ignore the value." 4391 ) 4392 elif not isinstance(strategy, (CursorPaginationModel, OffsetIncrementModel)): 4393 raise ValueError( 4394 f"`page_size_reduction` only supports the CursorPagination, OffsetIncrement and " 4395 f"CustomPaginationStrategy pagination strategies. Stream {name} uses " 4396 f"{type(strategy).__name__}." 4397 ) 4398 else: 4399 # A CustomPaginationStrategy carries its page size in its own code, so this is only checkable 4400 # for the strategies the CDK defines. `page_size` is optional on both of them, and without it 4401 # `get_page_size` returns None, the paginator injects nothing, and the reduction is a dead end 4402 # that only surfaces on the first failing response - after records have been emitted. 4403 self._validate_page_size_is_reducible(strategy, model.page_size_reduction, name) 4404 if isinstance(strategy, CursorPaginationModel): 4405 self._validate_stop_condition_is_reduction_aware( 4406 strategy, model.page_size_reduction, name 4407 ) 4408 4409 @staticmethod 4410 def _validate_stop_condition_is_reduction_aware( 4411 strategy: CursorPaginationModel, 4412 page_size_reduction: Optional[PageSizeReductionModel], 4413 name: str, 4414 ) -> None: 4415 """ 4416 A `stop_condition` comparing `last_page_size` to a hardcoded page size reads a full reduced page as a 4417 short page and ends the pagination early, dropping the rest of the partition without failing. The 4418 strategy exposes the page size that was actually requested as `page_size`, so the condition can be 4419 written correctly - but only if it is, which is what this checks. 4420 4421 The condition is parsed as a Jinja expression rather than matched as a string: only the AST tells 4422 `page_size`, which follows the reduction, apart from `config['page_size']`, which does not, and only the 4423 AST tells an inequality, which a reduction can invalidate, apart from `last_page_size == 0`, which it 4424 cannot. A condition that never names `last_page_size` is not waved through: a count read from the 4425 response body - `{{ response['data'] | length < 100 }}` - truncates in exactly the same way, and which 4426 response field counts the records of a page is not knowable here. A shape the analysis does not 4427 understand is warned about rather than rejected - this runs at stream construction, so a false 4428 rejection takes `check`, `discover` and `read` down with it. 4429 """ 4430 stop_condition = strategy.stop_condition 4431 if not stop_condition: 4432 return 4433 4434 minimum_page_size = ( 4435 page_size_reduction.minimum_page_size if page_size_reduction else None 4436 ) or 1 4437 verdict, reason = classify_stop_condition(stop_condition, minimum_page_size) 4438 if verdict is StopConditionSafety.TRUNCATES: 4439 raise ValueError( 4440 f"`page_size_reduction` on stream {name} cannot be used with the `stop_condition` " 4441 f"{stop_condition!r}: {reason}. The pagination would then end early, silently dropping the " 4442 f"rest of the partition. Compare against the `page_size` interpolation variable instead, " 4443 f"which holds the page size that was actually requested (for example " 4444 f"`{{{{ last_page_size < page_size }}}}`), or test the page for emptiness with " 4445 f"`{{{{ last_page_size == 0 }}}}`." 4446 ) 4447 if verdict is StopConditionSafety.UNKNOWN: 4448 LOGGER.warning( 4449 f"Stream {name} uses `page_size_reduction` with the `stop_condition` {stop_condition!r}, " 4450 f"which could not be checked against the reduction because {reason}. Make sure a page that is " 4451 f"full at a reduced page size does not satisfy it, otherwise the pagination ends early and the " 4452 f"rest of the partition is silently dropped. Comparing against the `page_size` interpolation " 4453 f"variable, which holds the page size that was actually requested, is always safe." 4454 ) 4455 4456 @staticmethod 4457 def _validate_page_size_is_reducible( 4458 strategy: Union[CursorPaginationModel, OffsetIncrementModel], 4459 page_size_reduction: Optional[PageSizeReductionModel], 4460 name: str, 4461 ) -> None: 4462 page_size = strategy.page_size 4463 if page_size is None: 4464 raise ValueError( 4465 f"`page_size_reduction` requires `page_size` on the pagination strategy of stream {name}: " 4466 f"without it the paginator does not send a page size, so there is nothing to reduce." 4467 ) 4468 4469 # The schema allows a string so that the page size can be interpolated. `page_size` is 4470 # `Optional[Union[int, str]]` and pydantic v1 tries `int` first, so both `100` and `"100"` arrive as 4471 # an int and only a genuinely non-numeric template stays a str. Such a template is only known once 4472 # the config is available, so it is left to the runtime check in `PageSizeReducer.reduce`. 4473 try: 4474 configured_page_size: Optional[int] = int(page_size) 4475 except ValueError: 4476 configured_page_size = None 4477 4478 minimum_page_size = ( 4479 page_size_reduction.minimum_page_size if page_size_reduction else None 4480 ) or 1 4481 if configured_page_size is not None and configured_page_size <= minimum_page_size: 4482 raise ValueError( 4483 f"`page_size_reduction` on stream {name} can never reduce its page size: the pagination " 4484 f"strategy's `page_size` is {configured_page_size} and `minimum_page_size` is " 4485 f"{minimum_page_size}. Lower `minimum_page_size` or raise `page_size`." 4486 ) 4487 4488 # Each reduction divides the page size by `reduction_factor` and spends one attempt, so an unbroken 4489 # run of failures bottoms out at `page_size / reduction_factor ** max_attempts` whatever 4490 # `minimum_page_size` says. Only warned about, and not raised: pages that succeed in between restart 4491 # the budget while `NEVER` keeps the page size, so the floor is reachable over a partition even when 4492 # it is out of reach of a single run - and a manifest that deliberately gives up earlier than its 4493 # floor is not wrong, only worth pointing out. 4494 reduction_factor = ( 4495 page_size_reduction.reduction_factor if page_size_reduction else None 4496 ) or 2.0 4497 max_attempts = (page_size_reduction.max_attempts if page_size_reduction else None) or 5 4498 # Only when there is a floor worth reaching. The default of 1 is out of reach of the default budget on 4499 # any page size above 32, so this would otherwise fire on nearly every stream that opts in - and 4500 # `__fields_set__` would not help, since it records that a value was supplied and not that it differs 4501 # from the default, so spelling `minimum_page_size: 1` out longhand would earn the warning. 4502 if ( 4503 configured_page_size is not None 4504 and minimum_page_size > 1 4505 and reduction_factor**max_attempts < configured_page_size / minimum_page_size 4506 ): 4507 reachable_page_size = max( 4508 minimum_page_size, int(configured_page_size // reduction_factor**max_attempts) 4509 ) 4510 LOGGER.warning( 4511 f"Stream {name} sets `minimum_page_size` to {minimum_page_size}, which `page_size_reduction` " 4512 f"cannot reach in one run of failing pages: `max_attempts` is {max_attempts} and " 4513 f"`reduction_factor` is {reduction_factor}, so {max_attempts} reductions of a page size of " 4514 f"{configured_page_size} stop at {reachable_page_size} records per page and the sync then " 4515 f"fails with a transient error. Whichever of the two bounds is tighter wins; reaching " 4516 f"{minimum_page_size} in one run needs `max_attempts` of at least " 4517 f"{math.ceil(math.log(configured_page_size / minimum_page_size, reduction_factor))}." 4518 ) 4519 4520 def _reject_reduce_page_size_action(self, requester: Any, description: str) -> None: 4521 """ 4522 `error_handler` is defined on `HttpRequester`, which is referenced by requesters that have no page of 4523 their own, so REDUCE_PAGE_SIZE is schema-legal in places where nothing can honor it. Only 4524 `SimpleRetriever._read_pages` re-issues a page, so every other requester is rejected here rather than 4525 surfacing the exception mid-sync as a generic failure. 4526 """ 4527 if requester is None: 4528 return 4529 if self._uses_reduce_page_size_action(getattr(requester, "error_handler", None)): 4530 raise ValueError( 4531 f"The REDUCE_PAGE_SIZE response action is not supported on the {description}: only the main " 4532 f"requester of a SimpleRetriever can re-issue its page with a smaller page size. Use a " 4533 f"different action on that error handler." 4534 ) 4535 4536 def _uses_reduce_page_size_action(self, error_handler: Any) -> bool: 4537 if isinstance(error_handler, CompositeErrorHandlerModel): 4538 return any( 4539 self._uses_reduce_page_size_action(nested) 4540 for nested in error_handler.error_handlers 4541 ) 4542 if isinstance(error_handler, DefaultErrorHandlerModel): 4543 return any( 4544 response_filter.action == HttpResponseFilterActionModel.REDUCE_PAGE_SIZE 4545 for response_filter in error_handler.response_filters or [] 4546 ) 4547 # A CustomErrorHandler can return any action and we cannot inspect it, so we do not validate it. 4548 return False 4549 4550 def _create_request_window_splitting( 4551 self, 4552 model: SimpleRetrieverModel, 4553 name: str, 4554 cursor: Optional[Cursor], 4555 incremental_sync: Optional[ 4556 Union[IncrementingCountCursorModel, DatetimeBasedCursorModel, SyncTokenCursorModel] 4557 ] = None, 4558 query_properties: Optional[QueryProperties] = None, 4559 file_uploader: Optional[DefaultFileUploader] = None, 4560 ) -> Optional[RequestWindowSplitting]: 4561 # A CustomRequester does not necessarily define an error handler. A CustomRequester that does define 4562 # one keeps it as a raw dict rather than a typed model, so this returns False for it as well. 4563 error_handler = getattr(model.requester, "error_handler", None) 4564 uses_action = self._uses_split_request_window_action(error_handler) 4565 if uses_action and not model.request_window_splitting: 4566 raise ValueError( 4567 f"Stream {name} has a response filter with the SPLIT_REQUEST_WINDOW action but the " 4568 f"retriever does not define `request_window_splitting`. Add a `request_window_splitting` " 4569 f"block to the retriever." 4570 ) 4571 4572 if not model.request_window_splitting: 4573 return None 4574 4575 if not uses_action: 4576 # Not raised: a CustomErrorHandler can resolve to SPLIT_REQUEST_WINDOW without us being able to 4577 # see it, and custom code can raise RequestWindowSplitRequiredException directly without 4578 # going through any error handler at all - so the only safe reaction to a block we cannot tie to 4579 # a known trigger is a warning. 4580 LOGGER.warning( 4581 f"Stream {name} defines `request_window_splitting` but no response filter with the " 4582 f"SPLIT_REQUEST_WINDOW action was found on its requester. The request window will never " 4583 f"be split unless a custom error handler or custom code resolves to that action." 4584 ) 4585 4586 self._validate_request_window_splitting_is_supported( 4587 name, 4588 cursor, 4589 incremental_sync, 4590 model.request_window_splitting.min_split_window, 4591 query_properties, 4592 file_uploader, 4593 ) 4594 4595 return RequestWindowSplitting( 4596 failure_message=model.request_window_splitting.failure_message, 4597 min_split_window=parse_duration(model.request_window_splitting.min_split_window) 4598 if model.request_window_splitting.min_split_window 4599 else None, 4600 ) 4601 4602 def _validate_request_window_splitting_is_supported( 4603 self, 4604 name: str, 4605 cursor: Optional[Cursor], 4606 incremental_sync: Optional[ 4607 Union[IncrementingCountCursorModel, DatetimeBasedCursorModel, SyncTokenCursorModel] 4608 ] = None, 4609 min_split_window: Optional[str] = None, 4610 query_properties: Optional[QueryProperties] = None, 4611 file_uploader: Optional[DefaultFileUploader] = None, 4612 ) -> None: 4613 """ 4614 Request window splitting replaces a failing slice with smaller children derived from the stream's own 4615 cursor, so it only makes sense on a stream whose cursor exposes a `split_request_window` method and it 4616 is rejected for the same structural reasons `page_size_reduction` is: `additional_query_properties` and 4617 a `file_uploader` both mean records of the failing window may already have been emitted by the time the 4618 split is requested, from inside the record generator rather than from a page boundary this retriever 4619 controls. 4620 """ 4621 if not hasattr(cursor, "split_request_window"): 4622 raise ValueError( 4623 f"`request_window_splitting` requires an incremental cursor that supports window splitting on " 4624 f"stream {name}. Found {type(cursor).__name__ if cursor is not None else 'no cursor'}: this is " 4625 f"only supported today for a `DatetimeBasedCursor` with `cursor_granularity` set." 4626 ) 4627 4628 if not ( 4629 isinstance(incremental_sync, DatetimeBasedCursorModel) 4630 and incremental_sync.cursor_granularity 4631 ): 4632 # `cursor_granularity` is both the smallest window the connector will ever request and what keeps 4633 # two child windows from overlapping at their shared edge, so `split_request_window` cannot split 4634 # at all without it - checked here, on the manifest model, rather than by reaching into the 4635 # cursor's private state from outside the class it belongs to. 4636 raise ValueError( 4637 f"`request_window_splitting` requires `cursor_granularity` on the `DatetimeBasedCursor` of " 4638 f"stream {name}: without it there is no smallest window to stop splitting at, and no way to " 4639 f"keep two child windows from overlapping at their shared edge." 4640 ) 4641 4642 if incremental_sync.is_client_side_incremental: 4643 # A client-side-incremental cursor filters records after they are read rather than sending the 4644 # window as request parameters, so every child window sends the exact same request as its parent: 4645 # splitting changes nothing, and the sync would keep splitting all the way to `min_split_window` 4646 # or the safety-net depth before failing. 4647 raise ValueError( 4648 f"`request_window_splitting` cannot be used together with `is_client_side_incremental` on " 4649 f"stream {name}: the window is filtered client-side rather than sent to the API, so splitting " 4650 f"it would not change the request and could not resolve a SPLIT_REQUEST_WINDOW response." 4651 ) 4652 4653 parsed_cursor_granularity = parse_duration(incremental_sync.cursor_granularity) 4654 if isinstance( 4655 parsed_cursor_granularity, datetime.timedelta 4656 ) and parsed_cursor_granularity <= datetime.timedelta(0): 4657 # Passes the truthy check above (the string is non-empty), but a zero-or-negative granularity can 4658 # never produce a child strictly smaller than the parent. 4659 raise ValueError( 4660 f"`request_window_splitting` requires a `cursor_granularity` greater than zero on stream " 4661 f"{name}: `{incremental_sync.cursor_granularity}` parses to a zero-length duration, which can " 4662 f"never produce a smaller child window." 4663 ) 4664 4665 smallest_format_unit = self._smallest_datetime_format_unit(incremental_sync.datetime_format) 4666 if ( 4667 smallest_format_unit is not None 4668 and isinstance(parsed_cursor_granularity, datetime.timedelta) 4669 and parsed_cursor_granularity < smallest_format_unit 4670 ): 4671 # Two children split closer together than `datetime_format` can render would format to the same 4672 # boundary value instead of splitting. 4673 raise ValueError( 4674 f"`request_window_splitting` requires `cursor_granularity` to be no finer than what " 4675 f"`datetime_format` can represent on stream {name}: `datetime_format` " 4676 f"{incremental_sync.datetime_format!r} cannot represent a value finer than " 4677 f"{smallest_format_unit}, but `cursor_granularity` {incremental_sync.cursor_granularity!r} is " 4678 f"finer than that. Use a `datetime_format` precise enough to represent `cursor_granularity`, " 4679 f"or a coarser `cursor_granularity`." 4680 ) 4681 4682 if min_split_window: 4683 parsed_min_split_window = parse_duration(min_split_window) 4684 if not isinstance( 4685 parsed_min_split_window, datetime.timedelta 4686 ) or parsed_min_split_window <= datetime.timedelta(0): 4687 # A duration with years or months (e.g. `P1M`) parses to an `isodate.Duration`, which has no 4688 # fixed length to compare a window's span against - `split_request_window` would raise 4689 # mid-sync trying to. A zero-or-negative duration would never stop splitting. 4690 raise ValueError( 4691 f"`min_split_window` must be a positive duration expressible as a fixed number of " 4692 f"days/hours/minutes/seconds on stream {name}: `{min_split_window}` is not." 4693 ) 4694 4695 if query_properties: 4696 raise ValueError( 4697 f"`request_window_splitting` cannot be used together with query properties on stream {name}. " 4698 f"Records from the earlier property chunks may have already been emitted when a chunk asks to " 4699 f"split the window, so splitting and re-reading it could duplicate them." 4700 ) 4701 4702 if file_uploader: 4703 raise ValueError( 4704 f"`request_window_splitting` cannot be used together with a `file_uploader` on stream {name}. " 4705 f"The file uploader sends one request per record from inside the window's record generator, so " 4706 f"a split requested partway through could re-emit records already yielded by it." 4707 ) 4708 4709 @staticmethod 4710 def _smallest_datetime_format_unit(datetime_format: str) -> Optional[datetime.timedelta]: 4711 """ 4712 :return: the finest duration `datetime_format` can distinguish, per 4713 `_DATETIME_FORMAT_DIRECTIVE_GRANULARITY`, or `None` if it contains none of those directives. 4714 """ 4715 matches = [ 4716 granularity 4717 for directive, granularity in _DATETIME_FORMAT_DIRECTIVE_GRANULARITY 4718 if directive in datetime_format 4719 ] 4720 return min(matches) if matches else None 4721 4722 def _reject_split_request_window_action(self, requester: Any, description: str) -> None: 4723 """ 4724 `error_handler` is defined on `HttpRequester`, which is referenced by requesters that have no window of 4725 their own to split, so SPLIT_REQUEST_WINDOW is schema-legal in places where nothing can honor it. Only 4726 the main requester of a `SimpleRetriever` reads a cursor-sliced window, so every other requester is 4727 rejected here rather than surfacing the exception mid-sync as a generic failure. 4728 """ 4729 if requester is None: 4730 return 4731 if self._uses_split_request_window_action(getattr(requester, "error_handler", None)): 4732 raise ValueError( 4733 f"The SPLIT_REQUEST_WINDOW response action is not supported on the {description}: only the " 4734 f"main requester of a SimpleRetriever reads a window that can be split. Use a different " 4735 f"action on that error handler." 4736 ) 4737 4738 def _uses_split_request_window_action(self, error_handler: Any) -> bool: 4739 if isinstance(error_handler, CompositeErrorHandlerModel): 4740 return any( 4741 self._uses_split_request_window_action(nested) 4742 for nested in error_handler.error_handlers 4743 ) 4744 if isinstance(error_handler, DefaultErrorHandlerModel): 4745 return any( 4746 response_filter.action == HttpResponseFilterActionModel.SPLIT_REQUEST_WINDOW 4747 for response_filter in error_handler.response_filters or [] 4748 ) 4749 # A CustomErrorHandler can return any action and we cannot inspect it, so we do not validate it. 4750 return False 4751 4752 def _create_pagination_tracker_factory( 4753 self, model: Optional[PaginationResetModel], cursor: Cursor 4754 ) -> Callable[[], PaginationTracker]: 4755 if model is None: 4756 return lambda: PaginationTracker() 4757 4758 # Until we figure out a way to use any cursor for PaginationTracker, we will have to have this cursor selector logic 4759 cursor_factory: Callable[[], Optional[ConcurrentCursor]] = lambda: None 4760 if model.action == PaginationResetActionModel.RESET: 4761 # in that case, we will let cursor_factory to return None even if the stream has a cursor 4762 pass 4763 elif model.action == PaginationResetActionModel.SPLIT_USING_CURSOR: 4764 if isinstance(cursor, ConcurrentCursor): 4765 cursor_factory = lambda: cursor.copy_without_state() # type: ignore # the if condition validates that it is a ConcurrentCursor 4766 elif isinstance(cursor, ConcurrentPerPartitionCursor): 4767 cursor_factory = lambda: cursor._cursor_factory.create( # type: ignore # if this becomes a problem, we would need to extract the cursor_factory instantiation logic and make it accessible here 4768 {}, datetime.timedelta(0) 4769 ) 4770 elif not isinstance(cursor, FinalStateCursor): 4771 LOGGER.warning( 4772 "Unknown cursor for PaginationTracker. Pagination resets might not work properly" 4773 ) 4774 else: 4775 raise ValueError(f"Unknown PaginationReset action: {model.action}") 4776 4777 limit = model.limits.number_of_records if model and model.limits else None 4778 return lambda: PaginationTracker(cursor_factory(), limit) 4779 4780 def _get_log_formatter( 4781 self, log_formatter: Callable[[Response], Any] | None, name: str 4782 ) -> Callable[[Response], Any] | None: 4783 if self._should_limit_slices_fetched(): 4784 return ( 4785 ( 4786 lambda response: format_http_message( 4787 response, 4788 f"Stream '{name}' request", 4789 f"Request performed in order to extract records for stream '{name}'", 4790 name, 4791 ) 4792 ) 4793 if not log_formatter 4794 else log_formatter 4795 ) 4796 return None 4797 4798 def _should_limit_slices_fetched(self) -> bool: 4799 """ 4800 Returns True if the number of slices fetched should be limited, False otherwise. 4801 This is used to limit the number of slices fetched during tests. 4802 """ 4803 return bool(self._limit_slices_fetched or self._emit_connector_builder_messages) 4804 4805 @staticmethod 4806 def _has_query_properties_in_request_parameters( 4807 requester: Union[HttpRequesterModel, CustomRequesterModel], 4808 ) -> bool: 4809 if not hasattr(requester, "request_parameters"): 4810 return False 4811 request_parameters = requester.request_parameters 4812 if request_parameters and isinstance(request_parameters, Mapping): 4813 for request_parameter in request_parameters.values(): 4814 if isinstance(request_parameter, QueryPropertiesModel): 4815 return True 4816 return False 4817 4818 @staticmethod 4819 def _remove_query_properties( 4820 request_parameters: Mapping[str, Union[str, QueryPropertiesModel]], 4821 ) -> Mapping[str, str]: 4822 return { 4823 parameter_field: request_parameter 4824 for parameter_field, request_parameter in request_parameters.items() 4825 if not isinstance(request_parameter, QueryPropertiesModel) 4826 } 4827 4828 def create_state_delegating_stream( 4829 self, 4830 model: StateDelegatingStreamModel, 4831 config: Config, 4832 **kwargs: Any, 4833 ) -> DefaultStream: 4834 if ( 4835 model.full_refresh_stream.name != model.name 4836 or model.name != model.incremental_stream.name 4837 ): 4838 raise ValueError( 4839 f"state_delegating_stream, full_refresh_stream name and incremental_stream must have equal names. Instead has {model.name}, {model.full_refresh_stream.name} and {model.incremental_stream.name}." 4840 ) 4841 4842 # Resolve api_retention_period with config context (supports Jinja2 interpolation) 4843 resolved_retention_period: Optional[str] = None 4844 if model.api_retention_period: 4845 interpolated_retention = InterpolatedString.create( 4846 model.api_retention_period, parameters=model.parameters or {} 4847 ) 4848 resolved_value = interpolated_retention.eval(config=config) 4849 if resolved_value: 4850 resolved_retention_period = str(resolved_value) 4851 4852 if resolved_retention_period: 4853 for stream_model in (model.full_refresh_stream, model.incremental_stream): 4854 if isinstance(stream_model.incremental_sync, IncrementingCountCursorModel): 4855 raise ValueError( 4856 f"Stream '{model.name}' uses IncrementingCountCursor which is not supported " 4857 f"with api_retention_period. IncrementingCountCursor does not use datetime-based " 4858 f"cursors, so cursor age validation cannot be performed." 4859 ) 4860 if isinstance(stream_model.incremental_sync, SyncTokenCursorModel): 4861 raise ValueError( 4862 f"Stream '{model.name}' uses SyncTokenCursor which is not supported " 4863 f"with api_retention_period. A sync token has no datetime cursor, so " 4864 f"cursor age validation cannot be performed." 4865 ) 4866 4867 stream_state = self._connector_state_manager.get_stream_state(model.name, None) 4868 4869 if not stream_state: 4870 return self._create_component_from_model( # type: ignore[no-any-return] 4871 model.full_refresh_stream, config=config, **kwargs 4872 ) 4873 4874 incremental_stream: DefaultStream = self._create_component_from_model( 4875 model.incremental_stream, config=config, **kwargs 4876 ) # type: ignore[assignment] 4877 4878 # Only run cursor age validation for streams that are in the configured 4879 # catalog (or when no catalog was provided, e.g. during discover / connector 4880 # builder). Streams not selected by the user but instantiated as parent-stream 4881 # dependencies must not go through this path because it emits state messages 4882 # that the destination does not know about, causing "Stream not found" crashes. 4883 stream_is_in_catalog = ( 4884 not self._stream_name_to_configured_stream # no catalog → validate by default 4885 or model.name in self._stream_name_to_configured_stream 4886 ) 4887 if resolved_retention_period and stream_is_in_catalog: 4888 full_refresh_stream: DefaultStream = self._create_component_from_model( 4889 model.full_refresh_stream, config=config, **kwargs 4890 ) # type: ignore[assignment] 4891 if self._is_cursor_older_than_retention_period( 4892 stream_state, 4893 full_refresh_stream.cursor, 4894 incremental_stream.cursor, 4895 resolved_retention_period, 4896 model.name, 4897 ): 4898 # Clear state BEFORE constructing the full_refresh_stream so that 4899 # its cursor starts from start_date instead of the stale cursor. 4900 self._connector_state_manager.update_state_for_stream(model.name, None, {}) 4901 state_message = self._connector_state_manager.create_state_message(model.name, None) 4902 self._message_repository.emit_message(state_message) 4903 return self._create_component_from_model( # type: ignore[no-any-return] 4904 model.full_refresh_stream, config=config, **kwargs 4905 ) 4906 4907 return incremental_stream 4908 4909 @staticmethod 4910 def _is_cursor_older_than_retention_period( 4911 stream_state: Mapping[str, Any], 4912 full_refresh_cursor: Cursor, 4913 incremental_cursor: Cursor, 4914 api_retention_period: str, 4915 stream_name: str, 4916 ) -> bool: 4917 """Check if the cursor value in the state is older than the API's retention period. 4918 4919 Checks cursors in sequence: full refresh cursor first, then incremental cursor. 4920 FinalStateCursor returns now() for completed full refresh state (NO_CURSOR_STATE_KEY), 4921 which is always within retention, so we use incremental. For other states, it returns 4922 None and we fall back to checking the incremental cursor. 4923 4924 Returns True if the cursor is older than the retention period (should use full refresh). 4925 Returns False if the cursor is within the retention period (safe to use incremental). 4926 """ 4927 retention_duration = parse_duration(api_retention_period) 4928 retention_cutoff = datetime.datetime.now(datetime.timezone.utc) - retention_duration 4929 4930 # Check full refresh cursor first 4931 cursor_datetime = full_refresh_cursor.get_cursor_datetime_from_state(stream_state) 4932 4933 # If full refresh cursor returns None, check incremental cursor 4934 if cursor_datetime is None: 4935 cursor_datetime = incremental_cursor.get_cursor_datetime_from_state(stream_state) 4936 4937 if cursor_datetime is None: 4938 # Neither cursor could parse the state - fall back to full refresh to be safe 4939 return True 4940 4941 if cursor_datetime < retention_cutoff: 4942 logging.warning( 4943 f"Stream '{stream_name}' has a cursor value older than " 4944 f"the API's retention period of {api_retention_period} " 4945 f"(cutoff: {retention_cutoff.isoformat()}). " 4946 f"Falling back to full refresh to avoid data loss." 4947 ) 4948 return True 4949 4950 return False 4951 4952 def _get_state_delegating_stream_model( 4953 self, 4954 model: StateDelegatingStreamModel, 4955 parent_state: Optional[Mapping[str, Any]] = None, 4956 ) -> DeclarativeStreamModel: 4957 """Return the appropriate underlying stream model based on state.""" 4958 return ( 4959 model.incremental_stream 4960 if self._connector_state_manager.get_stream_state(model.name, None) or parent_state 4961 else model.full_refresh_stream 4962 ) 4963 4964 _OPTIONAL_ASYNC_STATUS_FIELDS = {"skipped"} 4965 4966 def _create_async_job_status_mapping( 4967 self, model: AsyncJobStatusMapModel, config: Config, **kwargs: Any 4968 ) -> Mapping[str, AsyncJobStatus]: 4969 api_status_to_cdk_status = {} 4970 for cdk_status, api_statuses in model.dict().items(): 4971 if cdk_status == "type": 4972 # This is an element of the dict because of the typing of the CDK but it is not a CDK status 4973 continue 4974 4975 if api_statuses is None: 4976 if cdk_status in self._OPTIONAL_ASYNC_STATUS_FIELDS: 4977 continue 4978 raise ValueError( 4979 f"Required CDK status '{cdk_status}' has no API statuses mapped. " 4980 f"Please provide at least an empty list for required status fields." 4981 ) 4982 4983 for status in api_statuses: 4984 if status in api_status_to_cdk_status: 4985 raise ValueError( 4986 f"API status {status} is already set for CDK status {cdk_status}. Please ensure API statuses are only provided once" 4987 ) 4988 api_status_to_cdk_status[status] = self._get_async_job_status(cdk_status) 4989 return api_status_to_cdk_status 4990 4991 def _get_async_job_status(self, status: str) -> AsyncJobStatus: 4992 match status: 4993 case "running": 4994 return AsyncJobStatus.RUNNING 4995 case "completed": 4996 return AsyncJobStatus.COMPLETED 4997 case "failed": 4998 return AsyncJobStatus.FAILED 4999 case "timeout": 5000 return AsyncJobStatus.TIMED_OUT 5001 case "skipped": 5002 return AsyncJobStatus.SKIPPED 5003 case _: 5004 raise ValueError(f"Unsupported CDK status {status}") 5005 5006 def create_async_retriever( 5007 self, 5008 model: AsyncRetrieverModel, 5009 config: Config, 5010 *, 5011 name: str, 5012 primary_key: Optional[ 5013 Union[str, List[str], List[List[str]]] 5014 ], # this seems to be needed to match create_simple_retriever 5015 stream_slicer: Optional[StreamSlicer], 5016 client_side_incremental_sync: Optional[Dict[str, Any]] = None, 5017 transformations: List[RecordTransformation], 5018 **kwargs: Any, 5019 ) -> AsyncRetriever: 5020 if model.download_target_requester and not model.download_target_extractor: 5021 raise ValueError( 5022 f"`download_target_extractor` required if using a `download_target_requester`" 5023 ) 5024 5025 for requester_field in ( 5026 "creation_requester", 5027 "polling_requester", 5028 "download_requester", 5029 "download_target_requester", 5030 "abort_requester", 5031 "delete_requester", 5032 ): 5033 self._reject_reduce_page_size_action( 5034 getattr(model, requester_field, None), 5035 f"`{requester_field}` of the AsyncRetriever of stream {name}", 5036 ) 5037 self._reject_split_request_window_action( 5038 getattr(model, requester_field, None), 5039 f"`{requester_field}` of the AsyncRetriever of stream {name}", 5040 ) 5041 5042 def _get_download_retriever( 5043 requester: Requester, extractor: RecordExtractor, _decoder: Decoder 5044 ) -> SimpleRetriever: 5045 self._count_dropped_empty_records(extractor) 5046 # We create a record selector for the download retriever 5047 # with no schema normalization and no transformations, neither record filter 5048 # as all this occurs in the record_selector of the AsyncRetriever 5049 record_selector = RecordSelector( 5050 extractor=extractor, 5051 name=name, 5052 record_filter=None, 5053 transformations=[], 5054 schema_normalization=TypeTransformer(TransformConfig.NoTransform), 5055 config=config, 5056 parameters={}, 5057 ) 5058 paginator = ( 5059 self._create_component_from_model( 5060 model=model.download_paginator, 5061 decoder=_decoder, 5062 config=config, 5063 url_base="", 5064 # Only read by the `union` rejection of `OffsetIncrement`. 5065 extractor_model=( 5066 model.download_extractor 5067 if isinstance(model.download_extractor, CombinedExtractorModel) 5068 else None 5069 ), 5070 ) 5071 if model.download_paginator 5072 else NoPagination(parameters={}) 5073 ) 5074 5075 return SimpleRetriever( 5076 requester=requester, 5077 record_selector=record_selector, 5078 primary_key=None, 5079 name=name, 5080 paginator=paginator, 5081 config=config, 5082 parameters={}, 5083 log_formatter=self._get_log_formatter(None, name), 5084 ) 5085 5086 def _get_job_timeout() -> datetime.timedelta: 5087 user_defined_timeout: Optional[int] = ( 5088 int( 5089 InterpolatedString.create( 5090 str(model.polling_job_timeout), 5091 parameters={}, 5092 ).eval(config) 5093 ) 5094 if model.polling_job_timeout 5095 else None 5096 ) 5097 5098 # check for user defined timeout during the test read or 15 minutes 5099 test_read_timeout = datetime.timedelta(minutes=user_defined_timeout or 15) 5100 # default value for non-connector builder is 60 minutes. 5101 default_sync_timeout = datetime.timedelta(minutes=user_defined_timeout or 60) 5102 5103 return ( 5104 test_read_timeout if self._emit_connector_builder_messages else default_sync_timeout 5105 ) 5106 5107 decoder = ( 5108 self._create_component_from_model(model=model.decoder, config=config) 5109 if model.decoder 5110 else JsonDecoder(parameters={}) 5111 ) 5112 record_selector = self._create_component_from_model( 5113 model=model.record_selector, 5114 config=config, 5115 decoder=decoder, 5116 name=name, 5117 transformations=transformations, 5118 client_side_incremental_sync=client_side_incremental_sync, 5119 ) 5120 5121 stream_slicer = stream_slicer or SinglePartitionRouter(parameters={}) 5122 if self._should_limit_slices_fetched(): 5123 stream_slicer = cast( 5124 StreamSlicer, 5125 StreamSlicerTestReadDecorator( 5126 wrapped_slicer=stream_slicer, 5127 maximum_number_of_slices=self._limit_slices_fetched or 5, 5128 ), 5129 ) 5130 5131 creation_requester = self._create_component_from_model( 5132 model=model.creation_requester, 5133 decoder=decoder, 5134 config=config, 5135 name=f"job creation - {name}", 5136 ) 5137 polling_requester = self._create_component_from_model( 5138 model=model.polling_requester, 5139 decoder=decoder, 5140 config=config, 5141 name=f"job polling - {name}", 5142 ) 5143 job_download_components_name = f"job download - {name}" 5144 download_decoder = ( 5145 self._create_component_from_model(model=model.download_decoder, config=config) 5146 if model.download_decoder 5147 else JsonDecoder(parameters={}) 5148 ) 5149 download_extractor = ( 5150 self._create_component_from_model( 5151 model=model.download_extractor, 5152 config=config, 5153 decoder=download_decoder, 5154 parameters=model.parameters, 5155 ) 5156 if model.download_extractor 5157 else DpathExtractor( 5158 [], 5159 config=config, 5160 decoder=download_decoder, 5161 parameters=model.parameters or {}, 5162 ) 5163 ) 5164 download_requester = self._create_component_from_model( 5165 model=model.download_requester, 5166 decoder=download_decoder, 5167 config=config, 5168 name=job_download_components_name, 5169 ) 5170 download_retriever = _get_download_retriever( 5171 download_requester, download_extractor, download_decoder 5172 ) 5173 abort_requester = ( 5174 self._create_component_from_model( 5175 model=model.abort_requester, 5176 decoder=decoder, 5177 config=config, 5178 name=f"job abort - {name}", 5179 ) 5180 if model.abort_requester 5181 else None 5182 ) 5183 delete_requester = ( 5184 self._create_component_from_model( 5185 model=model.delete_requester, 5186 decoder=decoder, 5187 config=config, 5188 name=f"job delete - {name}", 5189 ) 5190 if model.delete_requester 5191 else None 5192 ) 5193 download_target_requester = ( 5194 self._create_component_from_model( 5195 model=model.download_target_requester, 5196 decoder=decoder, 5197 config=config, 5198 name=f"job extract_url - {name}", 5199 ) 5200 if model.download_target_requester 5201 else None 5202 ) 5203 status_extractor = self._create_component_from_model( 5204 model=model.status_extractor, decoder=decoder, config=config, name=name 5205 ) 5206 download_target_extractor = ( 5207 self._create_component_from_model( 5208 model=model.download_target_extractor, 5209 decoder=decoder, 5210 config=config, 5211 name=name, 5212 ) 5213 if model.download_target_extractor 5214 else None 5215 ) 5216 5217 job_repository: AsyncJobRepository = AsyncHttpJobRepository( 5218 creation_requester=creation_requester, 5219 polling_requester=polling_requester, 5220 download_retriever=download_retriever, 5221 download_target_requester=download_target_requester, 5222 abort_requester=abort_requester, 5223 delete_requester=delete_requester, 5224 status_extractor=status_extractor, 5225 status_mapping=self._create_async_job_status_mapping(model.status_mapping, config), 5226 download_target_extractor=download_target_extractor, 5227 job_timeout=_get_job_timeout(), 5228 ) 5229 5230 failed_retry_wait_time_in_seconds: Optional[int] = ( 5231 int( 5232 InterpolatedString.create( 5233 str(model.failed_retry_wait_time_in_seconds), 5234 parameters={}, 5235 ).eval(config) 5236 ) 5237 if model.failed_retry_wait_time_in_seconds 5238 else None 5239 ) 5240 5241 async_job_partition_router = AsyncJobPartitionRouter( 5242 job_orchestrator_factory=lambda stream_slices: AsyncJobOrchestrator( 5243 job_repository, 5244 stream_slices, 5245 self._job_tracker, 5246 self._message_repository, 5247 # FIXME work would need to be done here in order to detect if a stream as a parent stream that is bulk 5248 has_bulk_parent=False, 5249 # set the `job_max_retry` to 1 for the `Connector Builder`` use-case. 5250 # `None` == default retry is set to 3 attempts, under the hood. 5251 job_max_retry=1 if self._emit_connector_builder_messages else None, 5252 failed_retry_wait_time_in_seconds=failed_retry_wait_time_in_seconds, 5253 ), 5254 stream_slicer=stream_slicer, 5255 config=config, 5256 parameters=model.parameters or {}, 5257 ) 5258 5259 return AsyncRetriever( 5260 record_selector=record_selector, 5261 stream_slicer=async_job_partition_router, 5262 config=config, 5263 parameters=model.parameters or {}, 5264 ) 5265 5266 def create_spec(self, model: SpecModel, config: Config, **kwargs: Any) -> Spec: 5267 config_migrations = [ 5268 self._create_component_from_model(migration, config) 5269 for migration in ( 5270 model.config_normalization_rules.config_migrations 5271 if ( 5272 model.config_normalization_rules 5273 and model.config_normalization_rules.config_migrations 5274 ) 5275 else [] 5276 ) 5277 ] 5278 config_transformations = [ 5279 self._create_component_from_model(transformation, config) 5280 for transformation in ( 5281 model.config_normalization_rules.transformations 5282 if ( 5283 model.config_normalization_rules 5284 and model.config_normalization_rules.transformations 5285 ) 5286 else [] 5287 ) 5288 ] 5289 config_validations = [ 5290 self._create_component_from_model(validation, config) 5291 for validation in ( 5292 model.config_normalization_rules.validations 5293 if ( 5294 model.config_normalization_rules 5295 and model.config_normalization_rules.validations 5296 ) 5297 else [] 5298 ) 5299 ] 5300 5301 return Spec( 5302 connection_specification=model.connection_specification, 5303 documentation_url=model.documentation_url, 5304 advanced_auth=model.advanced_auth, 5305 parameters={}, 5306 config_migrations=config_migrations, 5307 config_transformations=config_transformations, 5308 config_validations=config_validations, 5309 ) 5310 5311 def create_substream_partition_router( 5312 self, 5313 model: SubstreamPartitionRouterModel, 5314 config: Config, 5315 *, 5316 stream_name: str, 5317 **kwargs: Any, 5318 ) -> SubstreamPartitionRouter: 5319 parent_stream_configs = [] 5320 if model.parent_stream_configs: 5321 parent_stream_configs.extend( 5322 [ 5323 self.create_parent_stream_config_with_substream_wrapper( 5324 model=parent_stream_config, config=config, stream_name=stream_name, **kwargs 5325 ) 5326 for parent_stream_config in model.parent_stream_configs 5327 ] 5328 ) 5329 5330 return SubstreamPartitionRouter( 5331 parent_stream_configs=parent_stream_configs, 5332 parameters=model.parameters or {}, 5333 config=config, 5334 ) 5335 5336 def create_parent_stream_config_with_substream_wrapper( 5337 self, model: ParentStreamConfigModel, config: Config, *, stream_name: str, **kwargs: Any 5338 ) -> Any: 5339 child_state = self._connector_state_manager.get_stream_state(stream_name, None) 5340 if NO_CURSOR_STATE_KEY in child_state: 5341 # Full refresh streams checkpoint a `{NO_CURSOR_STATE_KEY: true}` sentinel. When such a 5342 # stream is later converted to incremental with an incremental_dependency parent, 5343 # `_instantiate_parent_stream_state_manager` would treat the sentinel's boolean as a legacy 5344 # cursor value and re-key it under the parent's cursor field, crashing cursor initialization. 5345 child_state = { 5346 key: value for key, value in child_state.items() if key != NO_CURSOR_STATE_KEY 5347 } 5348 5349 parent_state: Optional[Mapping[str, Any]] = ( 5350 child_state if model.incremental_dependency and child_state else None 5351 ) 5352 connector_state_manager = self._instantiate_parent_stream_state_manager( 5353 child_state, config, model, parent_state 5354 ) 5355 5356 substream_factory = ModelToComponentFactory( 5357 custom_components_trusted=self._custom_components_trusted, 5358 connector_state_manager=connector_state_manager, 5359 limit_pages_fetched_per_slice=self._limit_pages_fetched_per_slice, 5360 limit_slices_fetched=self._limit_slices_fetched, 5361 emit_connector_builder_messages=self._emit_connector_builder_messages, 5362 disable_retries=self._disable_retries, 5363 disable_cache=self._disable_cache, 5364 message_repository=StateFilteringMessageRepository( 5365 LogAppenderMessageRepositoryDecorator( 5366 { 5367 "airbyte_cdk": {"stream": {"is_substream": True}}, 5368 "http": {"is_auxiliary": True}, 5369 }, 5370 self._message_repository, 5371 self._evaluate_log_level(self._emit_connector_builder_messages), 5372 ), 5373 ), 5374 api_budget=self._api_budget, 5375 # Share the authenticator registry so parent and child streams draw from the 5376 # same token quota counters 5377 rate_limited_authenticators=self._rate_limited_authenticators, 5378 ) 5379 5380 return substream_factory.create_parent_stream_config( 5381 model=model, config=config, stream_name=stream_name, **kwargs 5382 ) 5383 5384 def _instantiate_parent_stream_state_manager( 5385 self, 5386 child_state: MutableMapping[str, Any], 5387 config: Config, 5388 model: ParentStreamConfigModel, 5389 parent_state: Optional[Mapping[str, Any]] = None, 5390 ) -> ConnectorStateManager: 5391 """ 5392 With DefaultStream, the state needs to be provided during __init__ of the cursor as opposed to the 5393 `set_initial_state` flow that existed for the declarative cursors. This state is taken from 5394 self._connector_state_manager.get_stream_state (`self` being a newly created ModelToComponentFactory to account 5395 for the MessageRepository being different). So we need to pass a ConnectorStateManager to the 5396 ModelToComponentFactory that has the parent states. This method populates this if there is a child state and if 5397 incremental_dependency is set. 5398 """ 5399 if model.incremental_dependency and child_state: 5400 parent_stream_name = model.stream.name or "" 5401 extracted_parent_state = ConcurrentPerPartitionCursor.get_parent_state( 5402 child_state, parent_stream_name 5403 ) 5404 5405 if not extracted_parent_state: 5406 extracted_parent_state = ConcurrentPerPartitionCursor.get_global_state( 5407 child_state, parent_stream_name 5408 ) 5409 5410 if not extracted_parent_state and not isinstance(extracted_parent_state, dict): 5411 cursor_values = list(child_state.values()) 5412 # Only a scalar legacy cursor value can seed the parent. Sentinels such as 5413 # `{"__ab_full_refresh_sync_complete": true}` or legacy per-partition `states` 5414 # would crash the parent cursor initialization. 5415 if ( 5416 len(cursor_values) == 1 5417 and isinstance(cursor_values[0], (str, int, float)) 5418 and not isinstance(cursor_values[0], bool) 5419 ): 5420 incremental_sync_model: Union[ 5421 DatetimeBasedCursorModel, 5422 IncrementingCountCursorModel, 5423 ] = ( 5424 model.stream.incremental_sync # type: ignore # if we are there, it is because there is incremental_dependency and therefore there is an incremental_sync on the parent stream 5425 if isinstance(model.stream, DeclarativeStreamModel) 5426 else self._get_state_delegating_stream_model( 5427 model.stream, parent_state=parent_state 5428 ).incremental_sync 5429 ) 5430 cursor_field = InterpolatedString.create( 5431 incremental_sync_model.cursor_field, 5432 parameters=incremental_sync_model.parameters or {}, 5433 ).eval(config) 5434 extracted_parent_state = AirbyteStateMessage( 5435 type=AirbyteStateType.STREAM, 5436 stream=AirbyteStreamState( 5437 stream_descriptor=StreamDescriptor( 5438 name=parent_stream_name, namespace=None 5439 ), 5440 stream_state=AirbyteStateBlob({cursor_field: cursor_values[0]}), 5441 ), 5442 ) 5443 return ConnectorStateManager([extracted_parent_state] if extracted_parent_state else []) 5444 5445 return ConnectorStateManager([]) 5446 5447 @staticmethod 5448 def create_wait_time_from_header( 5449 model: WaitTimeFromHeaderModel, config: Config, **kwargs: Any 5450 ) -> WaitTimeFromHeaderBackoffStrategy: 5451 return WaitTimeFromHeaderBackoffStrategy( 5452 header=model.header, 5453 parameters=model.parameters or {}, 5454 config=config, 5455 regex=model.regex, 5456 max_waiting_time_in_seconds=model.max_waiting_time_in_seconds, 5457 ) 5458 5459 @staticmethod 5460 def create_wait_time_from_response( 5461 model: WaitTimeFromResponseModel, config: Config, **kwargs: Any 5462 ) -> WaitTimeFromResponseBackoffStrategy: 5463 return WaitTimeFromResponseBackoffStrategy( 5464 wait_time_in_seconds=model.wait_time_in_seconds, 5465 parameters=model.parameters or {}, 5466 config=config, 5467 max_waiting_time_in_seconds=model.max_waiting_time_in_seconds, 5468 ) 5469 5470 @staticmethod 5471 def create_wait_until_time_from_header( 5472 model: WaitUntilTimeFromHeaderModel, config: Config, **kwargs: Any 5473 ) -> WaitUntilTimeFromHeaderBackoffStrategy: 5474 return WaitUntilTimeFromHeaderBackoffStrategy( 5475 header=model.header, 5476 parameters=model.parameters or {}, 5477 config=config, 5478 min_wait=model.min_wait, 5479 regex=model.regex, 5480 max_waiting_time_in_seconds=model.max_waiting_time_in_seconds, 5481 ) 5482 5483 def get_message_repository(self) -> MessageRepository: 5484 return self._message_repository 5485 5486 def _evaluate_log_level(self, emit_connector_builder_messages: bool) -> Level: 5487 return Level.DEBUG if emit_connector_builder_messages else Level.INFO 5488 5489 @staticmethod 5490 def create_components_mapping_definition( 5491 model: ComponentMappingDefinitionModel, config: Config, **kwargs: Any 5492 ) -> ComponentMappingDefinition: 5493 interpolated_value = InterpolatedString.create( 5494 model.value, parameters=model.parameters or {} 5495 ) 5496 field_path = [ 5497 InterpolatedString.create(path, parameters=model.parameters or {}) 5498 for path in model.field_path 5499 ] 5500 return ComponentMappingDefinition( 5501 field_path=field_path, # type: ignore[arg-type] # field_path can be str and InterpolatedString 5502 value=interpolated_value, 5503 value_type=ModelToComponentFactory._json_schema_type_name_to_type(model.value_type), 5504 create_or_update=model.create_or_update, 5505 condition=model.condition, 5506 parameters=model.parameters or {}, 5507 ) 5508 5509 def create_http_components_resolver( 5510 self, model: HttpComponentsResolverModel, config: Config, stream_name: Optional[str] = None 5511 ) -> Any: 5512 partition_router = self._build_stream_slicer_from_partition_router(model.retriever, config) 5513 retriever = self._create_component_from_model( 5514 model=model.retriever, 5515 config=config, 5516 name=f"{stream_name if stream_name else '__http_components_resolver'}", 5517 primary_key=None, 5518 stream_slicer=partition_router, 5519 partition_router=partition_router, 5520 transformations=[], 5521 ) 5522 5523 components_mapping = [] 5524 for component_mapping_definition_model in model.components_mapping: 5525 if component_mapping_definition_model.condition: 5526 raise ValueError("`condition` is only supported for `ConfigComponentsResolver`") 5527 components_mapping.append( 5528 self._create_component_from_model( 5529 model=component_mapping_definition_model, 5530 value_type=ModelToComponentFactory._json_schema_type_name_to_type( 5531 component_mapping_definition_model.value_type 5532 ), 5533 config=config, 5534 ) 5535 ) 5536 5537 return HttpComponentsResolver( 5538 retriever=retriever, 5539 # AsyncRetriever reads records only from the job slices its own slicer yields 5540 stream_slicer=retriever.stream_slicer 5541 if isinstance(retriever, AsyncRetriever) 5542 else partition_router, 5543 config=config, 5544 components_mapping=components_mapping, 5545 parameters=model.parameters or {}, 5546 ) 5547 5548 @staticmethod 5549 def create_stream_config( 5550 model: StreamConfigModel, config: Config, **kwargs: Any 5551 ) -> StreamConfig: 5552 model_configs_pointer: List[Union[InterpolatedString, str]] = ( 5553 [x for x in model.configs_pointer] if model.configs_pointer else [] 5554 ) 5555 5556 return StreamConfig( 5557 configs_pointer=model_configs_pointer, 5558 default_values=model.default_values, 5559 parameters=model.parameters or {}, 5560 ) 5561 5562 def create_config_components_resolver( 5563 self, 5564 model: ConfigComponentsResolverModel, 5565 config: Config, 5566 ) -> Any: 5567 model_stream_configs = ( 5568 model.stream_config if isinstance(model.stream_config, list) else [model.stream_config] 5569 ) 5570 5571 stream_configs = [ 5572 self._create_component_from_model( 5573 stream_config, config=config, parameters=model.parameters or {} 5574 ) 5575 for stream_config in model_stream_configs 5576 ] 5577 5578 components_mapping = [ 5579 self._create_component_from_model( 5580 model=components_mapping_definition_model, 5581 value_type=ModelToComponentFactory._json_schema_type_name_to_type( 5582 components_mapping_definition_model.value_type 5583 ), 5584 config=config, 5585 parameters=model.parameters, 5586 ) 5587 for components_mapping_definition_model in model.components_mapping 5588 ] 5589 5590 return ConfigComponentsResolver( 5591 stream_configs=stream_configs, 5592 config=config, 5593 components_mapping=components_mapping, 5594 parameters=model.parameters or {}, 5595 ) 5596 5597 def create_parametrized_components_resolver( 5598 self, 5599 model: ParametrizedComponentsResolverModel, 5600 config: Config, 5601 ) -> ParametrizedComponentsResolver: 5602 stream_parameters = StreamParametersDefinition( 5603 list_of_parameters_for_stream=model.stream_parameters.list_of_parameters_for_stream 5604 ) 5605 5606 components_mapping = [] 5607 for components_mapping_definition_model in model.components_mapping: 5608 if components_mapping_definition_model.condition: 5609 raise ValueError("`condition` is only supported for `ConfigComponentsResolver`") 5610 components_mapping.append( 5611 self._create_component_from_model( 5612 model=components_mapping_definition_model, 5613 value_type=ModelToComponentFactory._json_schema_type_name_to_type( 5614 components_mapping_definition_model.value_type 5615 ), 5616 config=config, 5617 ) 5618 ) 5619 return ParametrizedComponentsResolver( 5620 stream_parameters=stream_parameters, 5621 config=config, 5622 components_mapping=components_mapping, 5623 parameters=model.parameters or {}, 5624 ) 5625 5626 _UNSUPPORTED_DECODER_ERROR = ( 5627 "Specified decoder of {decoder_type} is not supported for pagination." 5628 "Please set as `JsonDecoder`, `XmlDecoder`, or a `CompositeRawDecoder` with an inner_parser of `JsonParser` or `GzipParser` instead." 5629 "If using `GzipParser`, please ensure that the lowest level inner_parser is a `JsonParser`." 5630 ) 5631 5632 def _is_supported_decoder_for_pagination(self, decoder: Decoder) -> bool: 5633 if isinstance(decoder, (JsonDecoder, XmlDecoder)): 5634 return True 5635 elif isinstance(decoder, CompositeRawDecoder): 5636 return self._is_supported_parser_for_pagination(decoder.parser) 5637 else: 5638 return False 5639 5640 def _is_supported_parser_for_pagination(self, parser: Parser) -> bool: 5641 if isinstance(parser, JsonParser): 5642 return True 5643 elif isinstance(parser, GzipParser): 5644 return isinstance(parser.inner_parser, JsonParser) 5645 else: 5646 return False 5647 5648 def create_http_api_budget( 5649 self, model: HTTPAPIBudgetModel, config: Config, **kwargs: Any 5650 ) -> HttpAPIBudget: 5651 policies = [ 5652 self._create_component_from_model(model=policy, config=config) 5653 for policy in model.policies 5654 ] 5655 5656 return HttpAPIBudget( 5657 policies=policies, 5658 ratelimit_reset_header=model.ratelimit_reset_header or "ratelimit-reset", 5659 ratelimit_remaining_header=model.ratelimit_remaining_header or "ratelimit-remaining", 5660 status_codes_for_ratelimit_hit=model.status_codes_for_ratelimit_hit or [429], 5661 ) 5662 5663 def create_fixed_window_call_rate_policy( 5664 self, model: FixedWindowCallRatePolicyModel, config: Config, **kwargs: Any 5665 ) -> FixedWindowCallRatePolicy: 5666 matchers = [ 5667 self._create_component_from_model(model=matcher, config=config) 5668 for matcher in model.matchers 5669 ] 5670 5671 # Set the initial reset timestamp to 10 days from now. 5672 # This value will be updated by the first request. 5673 return FixedWindowCallRatePolicy( 5674 next_reset_ts=datetime.datetime.now() + datetime.timedelta(days=10), 5675 period=parse_duration(model.period), 5676 call_limit=model.call_limit, 5677 matchers=matchers, 5678 ) 5679 5680 def create_file_uploader( 5681 self, model: FileUploaderModel, config: Config, **kwargs: Any 5682 ) -> FileUploader: 5683 self._reject_reduce_page_size_action(model.requester, "requester of a `file_uploader`") 5684 self._reject_split_request_window_action(model.requester, "requester of a `file_uploader`") 5685 name = "File Uploader" 5686 requester = self._create_component_from_model( 5687 model=model.requester, 5688 config=config, 5689 name=name, 5690 **kwargs, 5691 ) 5692 download_target_extractor = self._create_component_from_model( 5693 model=model.download_target_extractor, 5694 config=config, 5695 name=name, 5696 **kwargs, 5697 ) 5698 emit_connector_builder_messages = self._emit_connector_builder_messages 5699 file_uploader = DefaultFileUploader( 5700 requester=requester, 5701 download_target_extractor=download_target_extractor, 5702 config=config, 5703 file_writer=NoopFileWriter() 5704 if emit_connector_builder_messages 5705 else LocalFileSystemFileWriter(), 5706 parameters=model.parameters or {}, 5707 filename_extractor=model.filename_extractor if model.filename_extractor else None, 5708 ) 5709 5710 return ( 5711 ConnectorBuilderFileUploader(file_uploader) 5712 if emit_connector_builder_messages 5713 else file_uploader 5714 ) 5715 5716 def create_moving_window_call_rate_policy( 5717 self, model: MovingWindowCallRatePolicyModel, config: Config, **kwargs: Any 5718 ) -> MovingWindowCallRatePolicy: 5719 rates = [ 5720 self._create_component_from_model(model=rate, config=config) for rate in model.rates 5721 ] 5722 matchers = [ 5723 self._create_component_from_model(model=matcher, config=config) 5724 for matcher in model.matchers 5725 ] 5726 return MovingWindowCallRatePolicy( 5727 rates=rates, 5728 matchers=matchers, 5729 ) 5730 5731 def create_unlimited_call_rate_policy( 5732 self, model: UnlimitedCallRatePolicyModel, config: Config, **kwargs: Any 5733 ) -> UnlimitedCallRatePolicy: 5734 matchers = [ 5735 self._create_component_from_model(model=matcher, config=config) 5736 for matcher in model.matchers 5737 ] 5738 5739 return UnlimitedCallRatePolicy( 5740 matchers=matchers, 5741 ) 5742 5743 def create_rate(self, model: RateModel, config: Config, **kwargs: Any) -> Rate: 5744 interpolated_limit = InterpolatedString.create(str(model.limit), parameters={}) 5745 return Rate( 5746 limit=int(interpolated_limit.eval(config=config)), 5747 interval=parse_duration(model.interval), 5748 ) 5749 5750 def create_http_request_matcher( 5751 self, model: HttpRequestRegexMatcherModel, config: Config, **kwargs: Any 5752 ) -> HttpRequestRegexMatcher: 5753 weight = model.weight 5754 if weight is not None: 5755 if isinstance(weight, str): 5756 weight = int(InterpolatedString.create(weight, parameters={}).eval(config)) 5757 else: 5758 weight = int(weight) 5759 if weight < 1: 5760 raise ValueError(f"weight must be >= 1, got {weight}") 5761 return HttpRequestRegexMatcher( 5762 method=model.method, 5763 url_base=model.url_base, 5764 url_path_pattern=model.url_path_pattern, 5765 params=model.params, 5766 headers=model.headers, 5767 weight=weight, 5768 ) 5769 5770 def create_rate_limited_multiple_token_authenticator( 5771 self, 5772 model: RateLimitedMultipleTokenAuthenticatorModel, 5773 config: Config, 5774 **kwargs: Any, 5775 ) -> RateLimitedMultipleTokenAuthenticator: 5776 if isinstance(model.tokens, str): 5777 tokens_value = InterpolatedString.create(model.tokens, parameters={}).eval(config) 5778 delimiter = model.token_delimiter or "," 5779 tokens = [ 5780 token.strip() for token in str(tokens_value).split(delimiter) if token.strip() 5781 ] 5782 else: 5783 tokens = [ 5784 token_value 5785 for token in model.tokens 5786 if ( 5787 token_value := str( 5788 InterpolatedString.create(token, parameters={}).eval(config) 5789 ).strip() 5790 ) 5791 ] 5792 5793 quota_specs = [ 5794 { 5795 "name": quota_model.name, 5796 "remaining_path": quota_model.remaining_path, 5797 "reset_path": quota_model.reset_path, 5798 "limit_path": quota_model.limit_path, 5799 "remaining_header": quota_model.remaining_header, 5800 "reset_header": quota_model.reset_header, 5801 "limit_header": quota_model.limit_header, 5802 # Normalize the same way as the runtime TokenQuota below, so an omitted field 5803 # and an explicit `[]` key identically and keep sharing one set of counters. 5804 "exhaustion_status_codes": quota_model.exhaustion_status_codes or [], 5805 "matchers": [ 5806 { 5807 "method": matcher_model.method, 5808 "url_base": matcher_model.url_base, 5809 "url_path_pattern": matcher_model.url_path_pattern, 5810 "params": matcher_model.params, 5811 "headers": matcher_model.headers, 5812 "weight": matcher_model.weight, 5813 } 5814 for matcher_model in quota_model.matchers or [] 5815 ], 5816 } 5817 for quota_model in model.quotas 5818 ] 5819 5820 quota_status_url = str( 5821 InterpolatedString.create(model.quota_status_source.url, parameters={}).eval(config) 5822 ) 5823 quota_status_http_method = ( 5824 model.quota_status_source.http_method.value 5825 if model.quota_status_source.http_method 5826 else "GET" 5827 ) 5828 quota_status_headers = { 5829 key: str(InterpolatedString.create(value, parameters={}).eval(config)) 5830 for key, value in (model.quota_status_source.request_headers or {}).items() 5831 } 5832 # Normalize the same way as the quota specs above, so an omitted field and an explicit 5833 # `[]` key identically and keep sharing one set of counters. Deduplicated as well as 5834 # sorted, because the runtime turns this into a set: without it `[404]` and `[404, 404]` 5835 # would key differently and stop sharing counters while behaving identically. 5836 quota_status_unavailable_status_codes = sorted( 5837 set(model.quota_status_source.unavailable_status_codes or []) 5838 ) 5839 auth_method = model.auth_method or "Bearer" 5840 header = model.header or "Authorization" 5841 max_wait_time_str = str( 5842 InterpolatedString.create(model.max_wait_time or "PT2H", parameters={}).eval(config) 5843 ) 5844 max_wait_time = parse_duration(max_wait_time_str) 5845 if not isinstance(max_wait_time, datetime.timedelta): 5846 raise ValueError( 5847 f"max_wait_time must be a fixed-length ISO 8601 duration (e.g. 'PT2H'); " 5848 f"calendar-unit durations like '{max_wait_time_str}' are not supported" 5849 ) 5850 budget_reserve_fraction = ( 5851 model.budget_reserve_fraction if model.budget_reserve_fraction is not None else 0.1 5852 ) 5853 budget_min_reserve = ( 5854 model.budget_min_reserve if model.budget_min_reserve is not None else 50 5855 ) 5856 5857 # Reuse the same instance for identical definitions so that all streams share the 5858 # same token quota counters (similar to how api_budget is shared). The key is built 5859 # from the resolved constructor arguments rather than the raw model so that 5860 # stream-specific `$parameters` propagated onto the model (and its nested components) 5861 # cannot break instance sharing. 5862 cache_key = json.dumps( 5863 { 5864 "tokens": tokens, 5865 "quotas": quota_specs, 5866 "quota_status_url": quota_status_url, 5867 "quota_status_http_method": quota_status_http_method, 5868 "quota_status_headers": quota_status_headers, 5869 "quota_status_unavailable_status_codes": quota_status_unavailable_status_codes, 5870 "auth_method": auth_method, 5871 "header": header, 5872 "max_wait_time": max_wait_time.total_seconds(), 5873 "budget_reserve_fraction": budget_reserve_fraction, 5874 "budget_min_reserve": budget_min_reserve, 5875 }, 5876 sort_keys=True, 5877 ) 5878 if cache_key in self._rate_limited_authenticators: 5879 return self._rate_limited_authenticators[cache_key] 5880 5881 quotas = [ 5882 TokenQuota( 5883 name=quota_model.name, 5884 remaining_path=quota_model.remaining_path, 5885 reset_path=quota_model.reset_path, 5886 limit_path=quota_model.limit_path, 5887 remaining_header=quota_model.remaining_header, 5888 reset_header=quota_model.reset_header, 5889 limit_header=quota_model.limit_header, 5890 exhaustion_status_codes=quota_model.exhaustion_status_codes or [], 5891 matchers=[ 5892 self.create_http_request_matcher(matcher_model, config) 5893 for matcher_model in quota_model.matchers or [] 5894 ], 5895 ) 5896 for quota_model in model.quotas 5897 ] 5898 5899 authenticator = RateLimitedMultipleTokenAuthenticator( 5900 tokens=tokens, 5901 quotas=quotas, 5902 quota_status_url=quota_status_url, 5903 quota_status_http_method=quota_status_http_method, 5904 quota_status_headers=quota_status_headers, 5905 quota_status_unavailable_status_codes=quota_status_unavailable_status_codes, 5906 auth_method=auth_method, 5907 header=header, 5908 max_wait_time=max_wait_time, 5909 budget_reserve_fraction=budget_reserve_fraction, 5910 budget_min_reserve=budget_min_reserve, 5911 ) 5912 self._rate_limited_authenticators[cache_key] = authenticator 5913 return authenticator 5914 5915 def set_api_budget(self, component_definition: ComponentDefinition, config: Config) -> None: 5916 self._api_budget = self.create_component( 5917 model_type=HTTPAPIBudgetModel, component_definition=component_definition, config=config 5918 ) 5919 5920 def create_grouping_partition_router( 5921 self, 5922 model: GroupingPartitionRouterModel, 5923 config: Config, 5924 *, 5925 stream_name: str, 5926 **kwargs: Any, 5927 ) -> GroupingPartitionRouter: 5928 underlying_router = self._create_component_from_model( 5929 model=model.underlying_partition_router, 5930 config=config, 5931 stream_name=stream_name, 5932 **kwargs, 5933 ) 5934 if model.group_size < 1: 5935 raise ValueError(f"Group size must be greater than 0, got {model.group_size}") 5936 5937 # Request options in underlying partition routers are not supported for GroupingPartitionRouter 5938 # because they are specific to individual partitions and cannot be aggregated or handled 5939 # when grouping, potentially leading to incorrect API calls. Any request customization 5940 # should be managed at the stream level through the requester's configuration. 5941 if isinstance(underlying_router, SubstreamPartitionRouter): 5942 if any( 5943 parent_config.request_option 5944 for parent_config in underlying_router.parent_stream_configs 5945 ): 5946 raise ValueError("Request options are not supported for GroupingPartitionRouter.") 5947 5948 if isinstance(underlying_router, ListPartitionRouter): 5949 if underlying_router.request_option: 5950 raise ValueError("Request options are not supported for GroupingPartitionRouter.") 5951 5952 return GroupingPartitionRouter( 5953 group_size=model.group_size, 5954 underlying_partition_router=underlying_router, 5955 deduplicate=model.deduplicate if model.deduplicate is not None else True, 5956 config=config, 5957 ) 5958 5959 def create_union_partition_router( 5960 self, 5961 model: UnionPartitionRouterModel, 5962 config: Config, 5963 *, 5964 stream_name: str, 5965 **kwargs: Any, 5966 ) -> UnionPartitionRouter: 5967 # The schema enforces minItems: 2 for manifests; this guard covers construction paths 5968 # that bypass JSON-schema validation (the generated model carries no min_items constraint). 5969 if len(model.partition_routers) < 2: 5970 raise ValueError( 5971 f"UnionPartitionRouter for stream {stream_name} needs at least 2 child partition routers" 5972 ) 5973 5974 partition_routers = [ 5975 self._create_component_from_model( 5976 model=child, 5977 config=config, 5978 stream_name=stream_name, 5979 **kwargs, 5980 ) 5981 for child in model.partition_routers 5982 ] 5983 5984 # partition_field depends only on config/parameters, so it is evaluated once at build 5985 # time; the runtime component always receives a plain string. 5986 partition_field = InterpolatedString.create( 5987 model.partition_field, parameters=model.parameters or {} 5988 ).eval(config) 5989 5990 # Fail fast at build time when a built-in child router is statically known to emit a 5991 # partition field different from the union's. CustomPartitionRouter children are opaque 5992 # and can only be validated at runtime. 5993 for child_model in model.partition_routers: 5994 child_partition_fields: List[str] = [] 5995 if isinstance(child_model, ListPartitionRouterModel): 5996 child_partition_fields.append( 5997 InterpolatedString.create( 5998 child_model.cursor_field, parameters=child_model.parameters or {} 5999 ).eval(config) 6000 ) 6001 elif isinstance(child_model, SubstreamPartitionRouterModel): 6002 for parent_stream_config in child_model.parent_stream_configs: 6003 child_partition_fields.append( 6004 InterpolatedString.create( 6005 parent_stream_config.partition_field, 6006 parameters=parent_stream_config.parameters 6007 or child_model.parameters 6008 or {}, 6009 ).eval(config) 6010 ) 6011 elif isinstance(child_model, UnionPartitionRouterModel): 6012 child_partition_fields.append( 6013 InterpolatedString.create( 6014 child_model.partition_field, parameters=child_model.parameters or {} 6015 ).eval(config) 6016 ) 6017 for child_partition_field in child_partition_fields: 6018 if child_partition_field != partition_field: 6019 raise ValueError( 6020 f"UnionPartitionRouter expects all child partition routers to emit the " 6021 f"partition field '{partition_field}', but a " 6022 f"{child_model.type} child emits '{child_partition_field}'." 6023 ) 6024 6025 # A union slice comes from exactly one child partition router, so request options 6026 # declared on children cannot be applied consistently to requests built from the 6027 # normalized union slices. Partition values should be consumed via interpolation 6028 # (e.g. stream_partition) instead. Note that this validation only covers built-in 6029 # router types; CustomPartitionRouter children are opaque, so any request options 6030 # they implement internally cannot be detected or rejected here. 6031 for router in partition_routers: 6032 if isinstance(router, SubstreamPartitionRouter): 6033 if any( 6034 parent_config.request_option for parent_config in router.parent_stream_configs 6035 ): 6036 raise ValueError("Request options are not supported for UnionPartitionRouter.") 6037 if isinstance(router, ListPartitionRouter) and router.request_option: 6038 raise ValueError("Request options are not supported for UnionPartitionRouter.") 6039 6040 return UnionPartitionRouter( 6041 partition_routers=partition_routers, 6042 partition_field=partition_field, 6043 parameters=model.parameters or {}, 6044 ) 6045 6046 def _ensure_query_properties_to_model( 6047 self, requester: Union[HttpRequesterModel, CustomRequesterModel] 6048 ) -> None: 6049 """ 6050 For some reason, it seems like CustomRequesterModel request_parameters stays as dictionaries which means that 6051 the other conditions relying on it being QueryPropertiesModel instead of a dict fail. Here, we migrate them to 6052 proper model. 6053 """ 6054 if not hasattr(requester, "request_parameters"): 6055 return 6056 6057 request_parameters = requester.request_parameters 6058 if request_parameters and isinstance(request_parameters, Dict): 6059 for request_parameter_key in request_parameters.keys(): 6060 request_parameter = request_parameters[request_parameter_key] 6061 if ( 6062 isinstance(request_parameter, Dict) 6063 and request_parameter.get("type") == "QueryProperties" 6064 ): 6065 request_parameters[request_parameter_key] = QueryPropertiesModel.parse_obj( 6066 request_parameter 6067 ) 6068 6069 def _get_catalog_defined_cursor_field( 6070 self, stream_name: str, allow_catalog_defined_cursor_field: bool 6071 ) -> Optional[CursorField]: 6072 if not allow_catalog_defined_cursor_field: 6073 return None 6074 6075 configured_stream = self._stream_name_to_configured_stream.get(stream_name) 6076 6077 # Depending on the operation is being performed, there may not be a configured stream yet. In this 6078 # case we return None which will then use the default cursor field defined on the cursor model. 6079 # We also treat cursor_field: [""] (list with empty string) as no cursor field, since this can 6080 # occur when the platform serializes "no cursor configured" streams incorrectly. 6081 if ( 6082 not configured_stream 6083 or not configured_stream.cursor_field 6084 or not configured_stream.cursor_field[0] 6085 ): 6086 return None 6087 elif len(configured_stream.cursor_field) > 1: 6088 raise ValueError( 6089 f"The `{stream_name}` stream does not support nested cursor_field. Please specify only a single cursor_field for the stream in the configured catalog." 6090 ) 6091 else: 6092 return CursorField( 6093 cursor_field_key=configured_stream.cursor_field[0], 6094 supports_catalog_defined_cursor_field=allow_catalog_defined_cursor_field, 6095 )
ModelToComponentFactory( limit_pages_fetched_per_slice: Optional[int] = None, limit_slices_fetched: Optional[int] = None, emit_connector_builder_messages: bool = False, disable_retries: bool = False, disable_cache: bool = False, message_repository: Optional[airbyte_cdk.MessageRepository] = None, connector_state_manager: Optional[airbyte_cdk.ConnectorStateManager] = None, max_concurrent_async_job_count: Optional[int] = None, configured_catalog: Optional[airbyte_protocol_dataclasses.models.airbyte_protocol.ConfiguredAirbyteCatalog] = None, api_budget: Optional[airbyte_cdk.sources.streams.call_rate.APIBudget] = None, rate_limited_authenticators: Optional[Dict[str, airbyte_cdk.sources.declarative.auth.RateLimitedMultipleTokenAuthenticator]] = None, custom_components_trusted: bool = True)
764 def __init__( 765 self, 766 limit_pages_fetched_per_slice: Optional[int] = None, 767 limit_slices_fetched: Optional[int] = None, 768 emit_connector_builder_messages: bool = False, 769 disable_retries: bool = False, 770 disable_cache: bool = False, 771 message_repository: Optional[MessageRepository] = None, 772 connector_state_manager: Optional[ConnectorStateManager] = None, 773 max_concurrent_async_job_count: Optional[int] = None, 774 configured_catalog: Optional[ConfiguredAirbyteCatalog] = None, 775 api_budget: Optional[APIBudget] = None, 776 rate_limited_authenticators: Optional[ 777 Dict[str, RateLimitedMultipleTokenAuthenticator] 778 ] = None, 779 custom_components_trusted: bool = True, 780 ): 781 self._init_mappings() 782 self._custom_components_trusted = custom_components_trusted 783 self._limit_pages_fetched_per_slice = limit_pages_fetched_per_slice 784 self._limit_slices_fetched = limit_slices_fetched 785 self._emit_connector_builder_messages = emit_connector_builder_messages 786 self._disable_retries = disable_retries 787 self._disable_cache = disable_cache 788 self._message_repository = message_repository or InMemoryMessageRepository( 789 self._evaluate_log_level(emit_connector_builder_messages) 790 ) 791 self._stream_name_to_configured_stream = self._create_stream_name_to_configured_stream( 792 configured_catalog 793 ) 794 self._connector_state_manager = connector_state_manager or ConnectorStateManager() 795 self._api_budget: Optional[Union[APIBudget]] = api_budget 796 # Shared instances so all streams see the same token quota counters (like api_budget) 797 self._rate_limited_authenticators: Dict[str, RateLimitedMultipleTokenAuthenticator] = ( 798 rate_limited_authenticators if rate_limited_authenticators is not None else {} 799 ) 800 self._job_tracker: JobTracker = JobTracker(max_concurrent_async_job_count or 1) 801 # placeholder for deprecation warnings 802 self._collected_deprecation_logs: List[ConnectorBuilderLogMessage] = []
def
create_component( self, model_type: Type[pydantic.v1.main.BaseModel], component_definition: Mapping[str, Any], config: Mapping[str, Any], **kwargs: Any) -> Any:
932 def create_component( 933 self, 934 model_type: Type[BaseModel], 935 component_definition: ComponentDefinition, 936 config: Config, 937 **kwargs: Any, 938 ) -> Any: 939 """ 940 Takes a given Pydantic model type and Mapping representing a component definition and creates a declarative component and 941 subcomponents which will be used at runtime. This is done by first parsing the mapping into a Pydantic model and then creating 942 creating declarative components from that model. 943 944 :param model_type: The type of declarative component that is being initialized 945 :param component_definition: The mapping that represents a declarative component 946 :param config: The connector config that is provided by the customer 947 :return: The declarative component to be used at runtime 948 """ 949 950 component_type = component_definition.get("type") 951 if component_definition.get("type") != model_type.__name__: 952 raise ValueError( 953 f"Expected manifest component of type {model_type.__name__}, but received {component_type} instead" 954 ) 955 956 declarative_component_model = model_type.parse_obj(component_definition) 957 958 if not isinstance(declarative_component_model, model_type): 959 raise ValueError( 960 f"Expected {model_type.__name__} component, but received {declarative_component_model.__class__.__name__}" 961 ) 962 963 return self._create_component_from_model( 964 model=declarative_component_model, config=config, **kwargs 965 )
Takes a given Pydantic model type and Mapping representing a component definition and creates a declarative component and subcomponents which will be used at runtime. This is done by first parsing the mapping into a Pydantic model and then creating creating declarative components from that model.
Parameters
- model_type: The type of declarative component that is being initialized
- component_definition: The mapping that represents a declarative component
- config: The connector config that is provided by the customer
Returns
The declarative component to be used at runtime
982 def get_model_deprecations(self) -> List[ConnectorBuilderLogMessage]: 983 """ 984 Returns the deprecation warnings that were collected during the creation of components. 985 """ 986 return self._collected_deprecation_logs
Returns the deprecation warnings that were collected during the creation of components.
def
create_config_migration( self, model: airbyte_cdk.sources.declarative.models.declarative_component_schema.ConfigMigration, config: Mapping[str, Any]) -> airbyte_cdk.sources.declarative.spec.ConfigMigration:
1003 def create_config_migration( 1004 self, model: ConfigMigrationModel, config: Config 1005 ) -> ConfigMigration: 1006 transformations: List[ConfigTransformation] = [ 1007 self._create_component_from_model(transformation, config) 1008 for transformation in model.transformations 1009 ] 1010 1011 return ConfigMigration( 1012 description=model.description, 1013 transformations=transformations, 1014 )
def
create_config_add_fields( self, model: airbyte_cdk.sources.declarative.models.declarative_component_schema.ConfigAddFields, config: Mapping[str, Any], **kwargs: Any) -> airbyte_cdk.sources.declarative.transformations.config_transformations.ConfigAddFields:
1016 def create_config_add_fields( 1017 self, model: ConfigAddFieldsModel, config: Config, **kwargs: Any 1018 ) -> ConfigAddFields: 1019 fields = [self._create_component_from_model(field, config) for field in model.fields] 1020 return ConfigAddFields( 1021 fields=fields, 1022 condition=model.condition or "", 1023 )
@staticmethod
def
create_config_remove_fields( model: airbyte_cdk.sources.declarative.models.declarative_component_schema.ConfigRemoveFields, config: Mapping[str, Any], **kwargs: Any) -> airbyte_cdk.sources.declarative.transformations.config_transformations.ConfigRemoveFields:
@staticmethod
def
create_config_remap_field( model: airbyte_cdk.sources.declarative.models.declarative_component_schema.ConfigRemapField, config: Mapping[str, Any], **kwargs: Any) -> airbyte_cdk.sources.declarative.transformations.config_transformations.ConfigRemapField:
1034 @staticmethod 1035 def create_config_remap_field( 1036 model: ConfigRemapFieldModel, config: Config, **kwargs: Any 1037 ) -> ConfigRemapField: 1038 mapping = cast(Mapping[str, Any], model.map) 1039 return ConfigRemapField( 1040 map=mapping, 1041 field_path=model.field_path, 1042 config=config, 1043 )
def
create_dpath_validator( self, model: airbyte_cdk.sources.declarative.models.declarative_component_schema.DpathValidator, config: Mapping[str, Any]) -> airbyte_cdk.sources.declarative.validators.DpathValidator:
def
create_predicate_validator( self, model: airbyte_cdk.sources.declarative.models.declarative_component_schema.PredicateValidator, config: Mapping[str, Any]) -> airbyte_cdk.sources.declarative.validators.PredicateValidator:
1053 def create_predicate_validator( 1054 self, model: PredicateValidatorModel, config: Config 1055 ) -> PredicateValidator: 1056 strategy = self._create_component_from_model(model.validation_strategy, config) 1057 1058 return PredicateValidator( 1059 value=model.value, 1060 strategy=strategy, 1061 )
@staticmethod
def
create_validate_adheres_to_schema( model: airbyte_cdk.sources.declarative.models.declarative_component_schema.ValidateAdheresToSchema, config: Mapping[str, Any], **kwargs: Any) -> airbyte_cdk.sources.declarative.validators.ValidateAdheresToSchema:
1063 @staticmethod 1064 def create_validate_adheres_to_schema( 1065 model: ValidateAdheresToSchemaModel, config: Config, **kwargs: Any 1066 ) -> ValidateAdheresToSchema: 1067 base_schema = cast(Mapping[str, Any], model.base_schema) 1068 return ValidateAdheresToSchema( 1069 schema=base_schema, 1070 )
@staticmethod
def
create_added_field_definition( model: airbyte_cdk.sources.declarative.models.declarative_component_schema.AddedFieldDefinition, config: Mapping[str, Any], **kwargs: Any) -> airbyte_cdk.AddedFieldDefinition:
1072 @staticmethod 1073 def create_added_field_definition( 1074 model: AddedFieldDefinitionModel, config: Config, **kwargs: Any 1075 ) -> AddedFieldDefinition: 1076 interpolated_value = InterpolatedString.create( 1077 model.value, parameters=model.parameters or {} 1078 ) 1079 return AddedFieldDefinition( 1080 path=model.path, 1081 value=interpolated_value, 1082 value_type=ModelToComponentFactory._json_schema_type_name_to_type(model.value_type), 1083 parameters=model.parameters or {}, 1084 )
def
create_add_fields( self, model: airbyte_cdk.sources.declarative.models.declarative_component_schema.AddFields, config: Mapping[str, Any], **kwargs: Any) -> airbyte_cdk.AddFields:
1086 def create_add_fields(self, model: AddFieldsModel, config: Config, **kwargs: Any) -> AddFields: 1087 added_field_definitions = [ 1088 self._create_component_from_model( 1089 model=added_field_definition_model, 1090 value_type=ModelToComponentFactory._json_schema_type_name_to_type( 1091 added_field_definition_model.value_type 1092 ), 1093 config=config, 1094 ) 1095 for added_field_definition_model in model.fields 1096 ] 1097 return AddFields( 1098 fields=added_field_definitions, 1099 condition=model.condition or "", 1100 parameters=model.parameters or {}, 1101 )
def
create_keys_to_lower_transformation( self, model: airbyte_cdk.sources.declarative.models.declarative_component_schema.KeysToLower, config: Mapping[str, Any], **kwargs: Any) -> airbyte_cdk.sources.declarative.transformations.keys_to_lower_transformation.KeysToLowerTransformation:
def
create_keys_to_snake_transformation( self, model: airbyte_cdk.sources.declarative.models.declarative_component_schema.KeysToSnakeCase, config: Mapping[str, Any], **kwargs: Any) -> airbyte_cdk.sources.declarative.transformations.keys_to_snake_transformation.KeysToSnakeCaseTransformation:
def
create_keys_replace_transformation( self, model: airbyte_cdk.sources.declarative.models.declarative_component_schema.KeysReplace, config: Mapping[str, Any], **kwargs: Any) -> airbyte_cdk.sources.declarative.transformations.keys_replace_transformation.KeysReplaceTransformation:
def
create_flatten_fields( self, model: airbyte_cdk.sources.declarative.models.declarative_component_schema.FlattenFields, config: Mapping[str, Any], **kwargs: Any) -> airbyte_cdk.sources.declarative.transformations.flatten_fields.FlattenFields:
def
create_dpath_flatten_fields( self, model: airbyte_cdk.sources.declarative.models.declarative_component_schema.DpathFlattenFields, config: Mapping[str, Any], **kwargs: Any) -> airbyte_cdk.sources.declarative.transformations.dpath_flatten_fields.DpathFlattenFields:
1127 def create_dpath_flatten_fields( 1128 self, model: DpathFlattenFieldsModel, config: Config, **kwargs: Any 1129 ) -> DpathFlattenFields: 1130 model_field_path: List[Union[InterpolatedString, str]] = [x for x in model.field_path] 1131 key_transformation = ( 1132 KeyTransformation( 1133 config=config, 1134 prefix=model.key_transformation.prefix, 1135 suffix=model.key_transformation.suffix, 1136 parameters=model.parameters or {}, 1137 ) 1138 if model.key_transformation is not None 1139 else None 1140 ) 1141 return DpathFlattenFields( 1142 config=config, 1143 field_path=model_field_path, 1144 delete_origin_value=model.delete_origin_value 1145 if model.delete_origin_value is not None 1146 else False, 1147 replace_record=model.replace_record if model.replace_record is not None else False, 1148 key_transformation=key_transformation, 1149 parameters=model.parameters or {}, 1150 )
def
create_api_key_authenticator( self, model: airbyte_cdk.sources.declarative.models.declarative_component_schema.ApiKeyAuthenticator, config: Mapping[str, Any], token_provider: Optional[airbyte_cdk.sources.declarative.auth.token_provider.TokenProvider] = None, **kwargs: Any) -> airbyte_cdk.ApiKeyAuthenticator:
1164 def create_api_key_authenticator( 1165 self, 1166 model: ApiKeyAuthenticatorModel, 1167 config: Config, 1168 token_provider: Optional[TokenProvider] = None, 1169 **kwargs: Any, 1170 ) -> ApiKeyAuthenticator: 1171 if model.inject_into is None and model.header is None: 1172 raise ValueError( 1173 "Expected either inject_into or header to be set for ApiKeyAuthenticator" 1174 ) 1175 1176 if model.inject_into is not None and model.header is not None: 1177 raise ValueError( 1178 "inject_into and header cannot be set both for ApiKeyAuthenticator - remove the deprecated header option" 1179 ) 1180 1181 if token_provider is not None and model.api_token != "": 1182 raise ValueError( 1183 "If token_provider is set, api_token is ignored and has to be set to empty string." 1184 ) 1185 1186 request_option = ( 1187 self._create_component_from_model( 1188 model.inject_into, config, parameters=model.parameters or {} 1189 ) 1190 if model.inject_into 1191 else RequestOption( 1192 inject_into=RequestOptionType.header, 1193 field_name=model.header or "", 1194 parameters=model.parameters or {}, 1195 ) 1196 ) 1197 1198 return ApiKeyAuthenticator( 1199 token_provider=( 1200 token_provider 1201 if token_provider is not None 1202 else InterpolatedStringTokenProvider( 1203 api_token=model.api_token or "", 1204 config=config, 1205 parameters=model.parameters or {}, 1206 ) 1207 ), 1208 request_option=request_option, 1209 config=config, 1210 parameters=model.parameters or {}, 1211 )
def
create_legacy_to_per_partition_state_migration( self, model: airbyte_cdk.sources.declarative.models.declarative_component_schema.LegacyToPerPartitionStateMigration, config: Mapping[str, Any], declarative_stream: airbyte_cdk.sources.declarative.models.declarative_component_schema.DeclarativeStream) -> airbyte_cdk.LegacyToPerPartitionStateMigration:
1213 def create_legacy_to_per_partition_state_migration( 1214 self, 1215 model: LegacyToPerPartitionStateMigrationModel, 1216 config: Mapping[str, Any], 1217 declarative_stream: DeclarativeStreamModel, 1218 ) -> LegacyToPerPartitionStateMigration: 1219 retriever = declarative_stream.retriever 1220 if not isinstance(retriever, (SimpleRetrieverModel, AsyncRetrieverModel)): 1221 raise ValueError( 1222 f"LegacyToPerPartitionStateMigrations can only be applied on a DeclarativeStream with a SimpleRetriever or AsyncRetriever. Got {type(retriever)}" 1223 ) 1224 partition_router = retriever.partition_router 1225 if not isinstance( 1226 partition_router, 1227 ( 1228 SubstreamPartitionRouterModel, 1229 CustomPartitionRouterModel, 1230 UnionPartitionRouterModel, 1231 ), 1232 ): 1233 raise ValueError( 1234 f"LegacyToPerPartitionStateMigrations can only be applied on a SimpleRetriever with a SubstreamPartitionRouter, UnionPartitionRouter or CustomPartitionRouter. Got {type(partition_router)}" 1235 ) 1236 if not isinstance(partition_router, UnionPartitionRouterModel) and not hasattr( 1237 partition_router, "parent_stream_configs" 1238 ): 1239 raise ValueError( 1240 "LegacyToPerPartitionStateMigrations can only be applied with a parent stream configuration." 1241 ) 1242 1243 if not hasattr(declarative_stream, "incremental_sync"): 1244 raise ValueError( 1245 "LegacyToPerPartitionStateMigrations can only be applied with an incremental_sync configuration." 1246 ) 1247 1248 return LegacyToPerPartitionStateMigration( 1249 partition_router, # type: ignore # was already checked above 1250 declarative_stream.incremental_sync, # type: ignore # was already checked. Migration can be applied only to incremental streams. 1251 config, 1252 declarative_stream.parameters, # type: ignore # different type is expected here Mapping[str, Any], got Dict[str, Any] 1253 )
def
create_session_token_authenticator( self, model: airbyte_cdk.sources.declarative.models.declarative_component_schema.SessionTokenAuthenticator, config: Mapping[str, Any], *, name: str, **kwargs: Any) -> Union[airbyte_cdk.ApiKeyAuthenticator, airbyte_cdk.BearerAuthenticator]:
1255 def create_session_token_authenticator( 1256 self, model: SessionTokenAuthenticatorModel, config: Config, *, name: str, **kwargs: Any 1257 ) -> Union[ApiKeyAuthenticator, BearerAuthenticator]: 1258 self._reject_reduce_page_size_action( 1259 model.login_requester, f"`login_requester` of the SessionTokenAuthenticator of {name}" 1260 ) 1261 self._reject_split_request_window_action( 1262 model.login_requester, f"`login_requester` of the SessionTokenAuthenticator of {name}" 1263 ) 1264 decoder = ( 1265 self._create_component_from_model(model=model.decoder, config=config) 1266 if model.decoder 1267 else JsonDecoder(parameters={}) 1268 ) 1269 login_requester = self._create_component_from_model( 1270 model=model.login_requester, 1271 config=config, 1272 name=f"{name}_login_requester", 1273 decoder=decoder, 1274 ) 1275 token_provider = SessionTokenProvider( 1276 login_requester=login_requester, 1277 session_token_path=model.session_token_path, 1278 expiration_duration=parse_duration(model.expiration_duration) 1279 if model.expiration_duration 1280 else None, 1281 parameters=model.parameters or {}, 1282 message_repository=self._message_repository, 1283 decoder=decoder, 1284 ) 1285 if model.request_authentication.type == "Bearer": 1286 return ModelToComponentFactory.create_bearer_authenticator( 1287 BearerAuthenticatorModel(type="BearerAuthenticator", api_token=""), # type: ignore # $parameters has a default value 1288 config, 1289 token_provider=token_provider, 1290 ) 1291 else: 1292 # Get the api_token template if specified, default to just the session token 1293 api_token_template = ( 1294 getattr(model.request_authentication, "api_token", None) or "{{ session_token }}" 1295 ) 1296 final_token_provider: TokenProvider = InterpolatedSessionTokenProvider( 1297 config=config, 1298 api_token=api_token_template, 1299 session_token_provider=token_provider, 1300 parameters=model.parameters or {}, 1301 ) 1302 return self.create_api_key_authenticator( 1303 ApiKeyAuthenticatorModel( 1304 type="ApiKeyAuthenticator", 1305 api_token="", 1306 inject_into=model.request_authentication.inject_into, 1307 ), # type: ignore # $parameters and headers default to None 1308 config=config, 1309 token_provider=final_token_provider, 1310 )
@staticmethod
def
create_basic_http_authenticator( model: airbyte_cdk.sources.declarative.models.declarative_component_schema.BasicHttpAuthenticator, config: Mapping[str, Any], **kwargs: Any) -> airbyte_cdk.BasicHttpAuthenticator:
1312 @staticmethod 1313 def create_basic_http_authenticator( 1314 model: BasicHttpAuthenticatorModel, config: Config, **kwargs: Any 1315 ) -> BasicHttpAuthenticator: 1316 return BasicHttpAuthenticator( 1317 password=model.password or "", 1318 username=model.username, 1319 config=config, 1320 parameters=model.parameters or {}, 1321 )
@staticmethod
def
create_bearer_authenticator( model: airbyte_cdk.sources.declarative.models.declarative_component_schema.BearerAuthenticator, config: Mapping[str, Any], token_provider: Optional[airbyte_cdk.sources.declarative.auth.token_provider.TokenProvider] = None, **kwargs: Any) -> airbyte_cdk.BearerAuthenticator:
1323 @staticmethod 1324 def create_bearer_authenticator( 1325 model: BearerAuthenticatorModel, 1326 config: Config, 1327 token_provider: Optional[TokenProvider] = None, 1328 **kwargs: Any, 1329 ) -> BearerAuthenticator: 1330 if token_provider is not None and model.api_token != "": 1331 raise ValueError( 1332 "If token_provider is set, api_token is ignored and has to be set to empty string." 1333 ) 1334 return BearerAuthenticator( 1335 token_provider=( 1336 token_provider 1337 if token_provider is not None 1338 else InterpolatedStringTokenProvider( 1339 api_token=model.api_token or "", 1340 config=config, 1341 parameters=model.parameters or {}, 1342 ) 1343 ), 1344 config=config, 1345 parameters=model.parameters or {}, 1346 )
@staticmethod
def
create_dynamic_stream_check_config( model: airbyte_cdk.sources.declarative.models.declarative_component_schema.DynamicStreamCheckConfig, config: Mapping[str, Any], **kwargs: Any) -> airbyte_cdk.sources.declarative.checks.DynamicStreamCheckConfig:
1348 @staticmethod 1349 def create_dynamic_stream_check_config( 1350 model: DynamicStreamCheckConfigModel, config: Config, **kwargs: Any 1351 ) -> DynamicStreamCheckConfig: 1352 return DynamicStreamCheckConfig( 1353 dynamic_stream_name=model.dynamic_stream_name, 1354 stream_count=model.stream_count, 1355 )
def
create_check_stream( self, model: airbyte_cdk.sources.declarative.models.declarative_component_schema.CheckStream, config: Mapping[str, Any], **kwargs: Any) -> airbyte_cdk.sources.declarative.checks.CheckStream:
1357 def create_check_stream( 1358 self, model: CheckStreamModel, config: Config, **kwargs: Any 1359 ) -> CheckStream: 1360 if model.dynamic_streams_check_configs is None and model.stream_names is None: 1361 raise ValueError( 1362 "Expected either stream_names or dynamic_streams_check_configs to be set for CheckStream" 1363 ) 1364 1365 dynamic_streams_check_configs = ( 1366 [ 1367 self._create_component_from_model(model=dynamic_stream_check_config, config=config) 1368 for dynamic_stream_check_config in model.dynamic_streams_check_configs 1369 ] 1370 if model.dynamic_streams_check_configs 1371 else [] 1372 ) 1373 1374 # `model.config_overrides` is deliberately not read here. The source applies it around the whole 1375 # check operation (`ConcurrentDeclarativeSource._config_overridden_for_check`), which is what makes 1376 # it work for every checker type rather than only this one. Do not wire it in a second time. 1377 return CheckStream( 1378 stream_names=model.stream_names or [], 1379 dynamic_streams_check_configs=dynamic_streams_check_configs, 1380 parameters={}, 1381 )
@staticmethod
def
create_check_dynamic_stream( model: airbyte_cdk.sources.declarative.models.declarative_component_schema.CheckDynamicStream, config: Mapping[str, Any], **kwargs: Any) -> airbyte_cdk.sources.declarative.checks.CheckDynamicStream:
1383 @staticmethod 1384 def create_check_dynamic_stream( 1385 model: CheckDynamicStreamModel, config: Config, **kwargs: Any 1386 ) -> CheckDynamicStream: 1387 assert model.use_check_availability is not None # for mypy 1388 1389 use_check_availability = model.use_check_availability 1390 1391 # See `create_check_stream`: `model.config_overrides` is applied by the source, not here. 1392 return CheckDynamicStream( 1393 stream_count=model.stream_count, 1394 use_check_availability=use_check_availability, 1395 parameters={}, 1396 )
def
create_composite_error_handler( self, model: airbyte_cdk.sources.declarative.models.declarative_component_schema.CompositeErrorHandler, config: Mapping[str, Any], **kwargs: Any) -> airbyte_cdk.sources.declarative.requesters.error_handlers.CompositeErrorHandler:
1398 def create_composite_error_handler( 1399 self, model: CompositeErrorHandlerModel, config: Config, **kwargs: Any 1400 ) -> CompositeErrorHandler: 1401 error_handlers = [ 1402 self._create_component_from_model(model=error_handler_model, config=config) 1403 for error_handler_model in model.error_handlers 1404 ] 1405 return CompositeErrorHandler( 1406 error_handlers=error_handlers, parameters=model.parameters or {} 1407 )
@staticmethod
def
create_concurrency_level( model: airbyte_cdk.sources.declarative.models.declarative_component_schema.ConcurrencyLevel, config: Mapping[str, Any], **kwargs: Any) -> airbyte_cdk.sources.declarative.concurrency_level.ConcurrencyLevel:
1409 @staticmethod 1410 def create_concurrency_level( 1411 model: ConcurrencyLevelModel, config: Config, **kwargs: Any 1412 ) -> ConcurrencyLevel: 1413 return ConcurrencyLevel( 1414 default_concurrency=model.default_concurrency, 1415 max_concurrency=model.max_concurrency, 1416 config=config, 1417 parameters={}, 1418 )
@staticmethod
def
apply_stream_state_migrations( stream_state_migrations: Optional[List[Any]], stream_state: MutableMapping[str, Any]) -> MutableMapping[str, Any]:
1420 @staticmethod 1421 def apply_stream_state_migrations( 1422 stream_state_migrations: List[Any] | None, stream_state: MutableMapping[str, Any] 1423 ) -> MutableMapping[str, Any]: 1424 if stream_state_migrations: 1425 for state_migration in stream_state_migrations: 1426 if state_migration.should_migrate(stream_state): 1427 # The state variable is expected to be mutable but the migrate method returns an immutable mapping. 1428 stream_state = dict(state_migration.migrate(stream_state)) 1429 return stream_state
def
create_concurrent_cursor_from_datetime_based_cursor( self, model_type: Type[pydantic.v1.main.BaseModel], component_definition: Mapping[str, Any], stream_name: str, stream_namespace: Optional[str], stream_state: MutableMapping[str, Any], config: Mapping[str, Any], message_repository: Optional[airbyte_cdk.MessageRepository] = None, runtime_lookback_window: Optional[datetime.timedelta] = None, **kwargs: Any) -> airbyte_cdk.ConcurrentCursor:
1431 def create_concurrent_cursor_from_datetime_based_cursor( 1432 self, 1433 model_type: Type[BaseModel], 1434 component_definition: ComponentDefinition, 1435 stream_name: str, 1436 stream_namespace: Optional[str], 1437 stream_state: MutableMapping[str, Any], 1438 config: Config, 1439 message_repository: Optional[MessageRepository] = None, 1440 runtime_lookback_window: Optional[datetime.timedelta] = None, 1441 **kwargs: Any, 1442 ) -> ConcurrentCursor: 1443 component_type = component_definition.get("type") 1444 if component_definition.get("type") != model_type.__name__: 1445 raise ValueError( 1446 f"Expected manifest component of type {model_type.__name__}, but received {component_type} instead" 1447 ) 1448 1449 # FIXME the interfaces of the concurrent cursor are kind of annoying as they take a `ComponentDefinition` instead of the actual model. This was done because the ConcurrentDeclarativeSource didn't have access to the models [here for example](https://github.com/airbytehq/airbyte-python-cdk/blob/f525803b3fec9329e4cc8478996a92bf884bfde9/airbyte_cdk/sources/declarative/concurrent_declarative_source.py#L354C54-L354C91). So now we have two cases: 1450 # * The ComponentDefinition comes from model.__dict__ in which case we have `parameters` 1451 # * The ComponentDefinition comes from the manifest as a dict in which case we have `$parameters` 1452 # We should change those interfaces to use the model once we clean up the code in CDS at which point the parameter propagation should happen as part of the ModelToComponentFactory. 1453 if "$parameters" not in component_definition and "parameters" in component_definition: 1454 component_definition["$parameters"] = component_definition.get("parameters") # type: ignore # This is a dict 1455 datetime_based_cursor_model = model_type.parse_obj(component_definition) 1456 1457 if not isinstance(datetime_based_cursor_model, DatetimeBasedCursorModel): 1458 raise ValueError( 1459 f"Expected {model_type.__name__} component, but received {datetime_based_cursor_model.__class__.__name__}" 1460 ) 1461 1462 model_parameters = datetime_based_cursor_model.parameters or {} 1463 1464 cursor_field = self._get_catalog_defined_cursor_field( 1465 stream_name=stream_name, 1466 allow_catalog_defined_cursor_field=datetime_based_cursor_model.allow_catalog_defined_cursor_field 1467 or False, 1468 ) 1469 1470 if not cursor_field: 1471 interpolated_cursor_field = InterpolatedString.create( 1472 datetime_based_cursor_model.cursor_field, 1473 parameters=model_parameters, 1474 ) 1475 cursor_field = CursorField( 1476 cursor_field_key=interpolated_cursor_field.eval(config=config), 1477 supports_catalog_defined_cursor_field=datetime_based_cursor_model.allow_catalog_defined_cursor_field 1478 or False, 1479 ) 1480 1481 interpolated_partition_field_start = InterpolatedString.create( 1482 datetime_based_cursor_model.partition_field_start or "start_time", 1483 parameters=model_parameters, 1484 ) 1485 interpolated_partition_field_end = InterpolatedString.create( 1486 datetime_based_cursor_model.partition_field_end or "end_time", 1487 parameters=model_parameters, 1488 ) 1489 1490 slice_boundary_fields = ( 1491 interpolated_partition_field_start.eval(config=config), 1492 interpolated_partition_field_end.eval(config=config), 1493 ) 1494 1495 datetime_format = datetime_based_cursor_model.datetime_format 1496 1497 cursor_granularity = ( 1498 parse_duration(datetime_based_cursor_model.cursor_granularity) 1499 if datetime_based_cursor_model.cursor_granularity 1500 else None 1501 ) 1502 1503 lookback_window = None 1504 interpolated_lookback_window = ( 1505 InterpolatedString.create( 1506 datetime_based_cursor_model.lookback_window, 1507 parameters=model_parameters, 1508 ) 1509 if datetime_based_cursor_model.lookback_window 1510 else None 1511 ) 1512 if interpolated_lookback_window: 1513 evaluated_lookback_window = interpolated_lookback_window.eval(config=config) 1514 if evaluated_lookback_window: 1515 lookback_window = parse_duration(evaluated_lookback_window) 1516 1517 connector_state_converter: DateTimeStreamStateConverter 1518 connector_state_converter = CustomFormatConcurrentStreamStateConverter( 1519 datetime_format=datetime_format, 1520 input_datetime_formats=datetime_based_cursor_model.cursor_datetime_formats, 1521 is_sequential_state=True, # ConcurrentPerPartitionCursor only works with sequential state 1522 cursor_granularity=cursor_granularity, 1523 ) 1524 1525 # Adjusts the stream state by applying the runtime lookback window. 1526 # This is used to ensure correct state handling in case of failed partitions. 1527 stream_state_value = stream_state.get(cursor_field.cursor_field_key) 1528 if runtime_lookback_window and stream_state_value: 1529 new_stream_state = ( 1530 connector_state_converter.parse_timestamp(stream_state_value) 1531 - runtime_lookback_window 1532 ) 1533 stream_state[cursor_field.cursor_field_key] = connector_state_converter.output_format( 1534 new_stream_state 1535 ) 1536 1537 start_date_runtime_value: Union[InterpolatedString, str, MinMaxDatetime] 1538 if isinstance(datetime_based_cursor_model.start_datetime, MinMaxDatetimeModel): 1539 start_date_runtime_value = self.create_min_max_datetime( 1540 model=datetime_based_cursor_model.start_datetime, config=config 1541 ) 1542 else: 1543 start_date_runtime_value = datetime_based_cursor_model.start_datetime 1544 1545 end_date_runtime_value: Optional[Union[InterpolatedString, str, MinMaxDatetime]] 1546 if isinstance(datetime_based_cursor_model.end_datetime, MinMaxDatetimeModel): 1547 end_date_runtime_value = self.create_min_max_datetime( 1548 model=datetime_based_cursor_model.end_datetime, config=config 1549 ) 1550 else: 1551 end_date_runtime_value = datetime_based_cursor_model.end_datetime 1552 1553 interpolated_start_date = MinMaxDatetime.create( 1554 interpolated_string_or_min_max_datetime=start_date_runtime_value, 1555 parameters=datetime_based_cursor_model.parameters, 1556 ) 1557 interpolated_end_date = ( 1558 None 1559 if not end_date_runtime_value 1560 else MinMaxDatetime.create( 1561 end_date_runtime_value, datetime_based_cursor_model.parameters 1562 ) 1563 ) 1564 1565 # If datetime format is not specified then start/end datetime should inherit it from the stream slicer 1566 if not interpolated_start_date.datetime_format: 1567 interpolated_start_date.datetime_format = datetime_format 1568 if interpolated_end_date and not interpolated_end_date.datetime_format: 1569 interpolated_end_date.datetime_format = datetime_format 1570 1571 start_date = interpolated_start_date.get_datetime(config=config) 1572 end_date_provider = ( 1573 partial(interpolated_end_date.get_datetime, config) 1574 if interpolated_end_date 1575 else connector_state_converter.get_end_provider() 1576 ) 1577 1578 if ( 1579 datetime_based_cursor_model.step and not datetime_based_cursor_model.cursor_granularity 1580 ) or ( 1581 not datetime_based_cursor_model.step and datetime_based_cursor_model.cursor_granularity 1582 ): 1583 raise ValueError( 1584 f"If step is defined, cursor_granularity should be as well and vice-versa. " 1585 f"Right now, step is `{datetime_based_cursor_model.step}` and cursor_granularity is `{datetime_based_cursor_model.cursor_granularity}`" 1586 ) 1587 1588 # When step is not defined, default to a step size from the starting date to the present moment 1589 step_length = datetime.timedelta.max 1590 interpolated_step = ( 1591 InterpolatedString.create( 1592 datetime_based_cursor_model.step, 1593 parameters=model_parameters, 1594 ) 1595 if datetime_based_cursor_model.step 1596 else None 1597 ) 1598 if interpolated_step: 1599 evaluated_step = interpolated_step.eval(config) 1600 if evaluated_step: 1601 step_length = parse_duration(evaluated_step) 1602 1603 clamping_strategy: ClampingStrategy = NoClamping() 1604 if datetime_based_cursor_model.clamping: 1605 # While it is undesirable to interpolate within the model factory (as opposed to at runtime), 1606 # it is still better than shifting interpolation low-code concept into the ConcurrentCursor runtime 1607 # object which we want to keep agnostic of being low-code 1608 target = InterpolatedString( 1609 string=datetime_based_cursor_model.clamping.target, 1610 parameters=model_parameters, 1611 ) 1612 evaluated_target = target.eval(config=config) 1613 match evaluated_target: 1614 case "DAY": 1615 clamping_strategy = DayClampingStrategy() 1616 end_date_provider = ClampingEndProvider( 1617 DayClampingStrategy(is_ceiling=False), 1618 end_date_provider, # type: ignore # Having issues w/ inspection for GapType and CursorValueType as shown in existing tests. Confirmed functionality is working in practice 1619 granularity=cursor_granularity or datetime.timedelta(seconds=1), 1620 ) 1621 case "WEEK": 1622 if ( 1623 not datetime_based_cursor_model.clamping.target_details 1624 or "weekday" not in datetime_based_cursor_model.clamping.target_details 1625 ): 1626 raise ValueError( 1627 "Given WEEK clamping, weekday needs to be provided as target_details" 1628 ) 1629 weekday = self._assemble_weekday( 1630 datetime_based_cursor_model.clamping.target_details["weekday"] 1631 ) 1632 clamping_strategy = WeekClampingStrategy(weekday) 1633 end_date_provider = ClampingEndProvider( 1634 WeekClampingStrategy(weekday, is_ceiling=False), 1635 end_date_provider, # type: ignore # Having issues w/ inspection for GapType and CursorValueType as shown in existing tests. Confirmed functionality is working in practice 1636 granularity=cursor_granularity or datetime.timedelta(days=1), 1637 ) 1638 case "MONTH": 1639 clamping_strategy = MonthClampingStrategy() 1640 end_date_provider = ClampingEndProvider( 1641 MonthClampingStrategy(is_ceiling=False), 1642 end_date_provider, # type: ignore # Having issues w/ inspection for GapType and CursorValueType as shown in existing tests. Confirmed functionality is working in practice 1643 granularity=cursor_granularity or datetime.timedelta(days=1), 1644 ) 1645 case _: 1646 raise ValueError( 1647 f"Invalid clamping target {evaluated_target}, expected DAY, WEEK, MONTH" 1648 ) 1649 1650 return ConcurrentCursor( 1651 stream_name=stream_name, 1652 stream_namespace=stream_namespace, 1653 stream_state=stream_state, 1654 message_repository=message_repository or self._message_repository, 1655 connector_state_manager=self._connector_state_manager, 1656 connector_state_converter=connector_state_converter, 1657 cursor_field=cursor_field, 1658 slice_boundary_fields=slice_boundary_fields, 1659 start=start_date, # type: ignore # Having issues w/ inspection for GapType and CursorValueType as shown in existing tests. Confirmed functionality is working in practice 1660 end_provider=end_date_provider, # type: ignore # Having issues w/ inspection for GapType and CursorValueType as shown in existing tests. Confirmed functionality is working in practice 1661 lookback_window=lookback_window, 1662 slice_range=step_length, 1663 cursor_granularity=cursor_granularity, 1664 clamping_strategy=clamping_strategy, 1665 is_compare_strictly=datetime_based_cursor_model.is_compare_strictly or False, 1666 )
def
create_concurrent_cursor_from_incrementing_count_cursor( self, model_type: Type[pydantic.v1.main.BaseModel], component_definition: Mapping[str, Any], stream_name: str, stream_namespace: Optional[str], stream_state: MutableMapping[str, Any], config: Mapping[str, Any], message_repository: Optional[airbyte_cdk.MessageRepository] = None, **kwargs: Any) -> airbyte_cdk.ConcurrentCursor:
1668 def create_concurrent_cursor_from_incrementing_count_cursor( 1669 self, 1670 model_type: Type[BaseModel], 1671 component_definition: ComponentDefinition, 1672 stream_name: str, 1673 stream_namespace: Optional[str], 1674 stream_state: MutableMapping[str, Any], 1675 config: Config, 1676 message_repository: Optional[MessageRepository] = None, 1677 **kwargs: Any, 1678 ) -> ConcurrentCursor: 1679 component_type = component_definition.get("type") 1680 if component_definition.get("type") != model_type.__name__: 1681 raise ValueError( 1682 f"Expected manifest component of type {model_type.__name__}, but received {component_type} instead" 1683 ) 1684 1685 incrementing_count_cursor_model = model_type.parse_obj(component_definition) 1686 1687 if not isinstance(incrementing_count_cursor_model, IncrementingCountCursorModel): 1688 raise ValueError( 1689 f"Expected {model_type.__name__} component, but received {incrementing_count_cursor_model.__class__.__name__}" 1690 ) 1691 1692 start_value: Union[int, str, None] = incrementing_count_cursor_model.start_value 1693 # Pydantic Union type coercion can convert int 0 to string '0' depending on Union order. 1694 # We need to handle both int and str representations of numeric values. 1695 # Evaluate the InterpolatedString and convert to int for the ConcurrentCursor. 1696 if start_value is not None: 1697 interpolated_start_value = InterpolatedString.create( 1698 str(start_value), # Ensure we pass a string to InterpolatedString.create 1699 parameters=incrementing_count_cursor_model.parameters or {}, 1700 ) 1701 evaluated_start_value: int = int(interpolated_start_value.eval(config=config)) 1702 else: 1703 evaluated_start_value = 0 1704 1705 cursor_field = self._get_catalog_defined_cursor_field( 1706 stream_name=stream_name, 1707 allow_catalog_defined_cursor_field=incrementing_count_cursor_model.allow_catalog_defined_cursor_field 1708 or False, 1709 ) 1710 1711 if not cursor_field: 1712 interpolated_cursor_field = InterpolatedString.create( 1713 incrementing_count_cursor_model.cursor_field, 1714 parameters=incrementing_count_cursor_model.parameters or {}, 1715 ) 1716 cursor_field = CursorField( 1717 cursor_field_key=interpolated_cursor_field.eval(config=config), 1718 supports_catalog_defined_cursor_field=incrementing_count_cursor_model.allow_catalog_defined_cursor_field 1719 or False, 1720 ) 1721 1722 connector_state_converter = IncrementingCountStreamStateConverter( 1723 is_sequential_state=True, # ConcurrentPerPartitionCursor only works with sequential state 1724 ) 1725 1726 return ConcurrentCursor( 1727 stream_name=stream_name, 1728 stream_namespace=stream_namespace, 1729 stream_state=stream_state, 1730 message_repository=message_repository or self._message_repository, 1731 connector_state_manager=self._connector_state_manager, 1732 connector_state_converter=connector_state_converter, 1733 cursor_field=cursor_field, 1734 slice_boundary_fields=None, 1735 start=evaluated_start_value, # type: ignore # Having issues w/ inspection for GapType and CursorValueType as shown in existing tests. Confirmed functionality is working in practice 1736 end_provider=connector_state_converter.get_end_provider(), # type: ignore # Having issues w/ inspection for GapType and CursorValueType as shown in existing tests. Confirmed functionality is working in practice 1737 )
def
create_concurrent_cursor_from_perpartition_cursor( self, state_manager: airbyte_cdk.ConnectorStateManager, model_type: Type[pydantic.v1.main.BaseModel], component_definition: Mapping[str, Any], stream_name: str, stream_namespace: Optional[str], config: Mapping[str, Any], stream_state: MutableMapping[str, Any], partition_router: airbyte_cdk.sources.declarative.partition_routers.PartitionRouter, attempt_to_create_cursor_if_not_provided: bool = False, **kwargs: Any) -> airbyte_cdk.sources.declarative.incremental.ConcurrentPerPartitionCursor:
1758 def create_concurrent_cursor_from_perpartition_cursor( 1759 self, 1760 state_manager: ConnectorStateManager, 1761 model_type: Type[BaseModel], 1762 component_definition: ComponentDefinition, 1763 stream_name: str, 1764 stream_namespace: Optional[str], 1765 config: Config, 1766 stream_state: MutableMapping[str, Any], 1767 partition_router: PartitionRouter, 1768 attempt_to_create_cursor_if_not_provided: bool = False, 1769 **kwargs: Any, 1770 ) -> ConcurrentPerPartitionCursor: 1771 component_type = component_definition.get("type") 1772 if component_definition.get("type") != model_type.__name__: 1773 raise ValueError( 1774 f"Expected manifest component of type {model_type.__name__}, but received {component_type} instead" 1775 ) 1776 1777 # FIXME the interfaces of the concurrent cursor are kind of annoying as they take a `ComponentDefinition` instead of the actual model. This was done because the ConcurrentDeclarativeSource didn't have access to the models [here for example](https://github.com/airbytehq/airbyte-python-cdk/blob/f525803b3fec9329e4cc8478996a92bf884bfde9/airbyte_cdk/sources/declarative/concurrent_declarative_source.py#L354C54-L354C91). So now we have two cases: 1778 # * The ComponentDefinition comes from model.__dict__ in which case we have `parameters` 1779 # * The ComponentDefinition comes from the manifest as a dict in which case we have `$parameters` 1780 # We should change those interfaces to use the model once we clean up the code in CDS at which point the parameter propagation should happen as part of the ModelToComponentFactory. 1781 if "$parameters" not in component_definition and "parameters" in component_definition: 1782 component_definition["$parameters"] = component_definition.get("parameters") # type: ignore # This is a dict 1783 datetime_based_cursor_model = model_type.parse_obj(component_definition) 1784 1785 if not isinstance(datetime_based_cursor_model, DatetimeBasedCursorModel): 1786 raise ValueError( 1787 f"Expected {model_type.__name__} component, but received {datetime_based_cursor_model.__class__.__name__}" 1788 ) 1789 1790 cursor_field = self._get_catalog_defined_cursor_field( 1791 stream_name=stream_name, 1792 allow_catalog_defined_cursor_field=datetime_based_cursor_model.allow_catalog_defined_cursor_field 1793 or False, 1794 ) 1795 1796 if not cursor_field: 1797 interpolated_cursor_field = InterpolatedString.create( 1798 datetime_based_cursor_model.cursor_field, 1799 # FIXME the interfaces of the concurrent cursor are kind of annoying as they take a `ComponentDefinition` instead of the actual model. This was done because the ConcurrentDeclarativeSource didn't have access to the models [here for example](https://github.com/airbytehq/airbyte-python-cdk/blob/f525803b3fec9329e4cc8478996a92bf884bfde9/airbyte_cdk/sources/declarative/concurrent_declarative_source.py#L354C54-L354C91). So now we have two cases: 1800 # * The ComponentDefinition comes from model.__dict__ in which case we have `parameters` 1801 # * The ComponentDefinition comes from the manifest as a dict in which case we have `$parameters` 1802 # We should change those interfaces to use the model once we clean up the code in CDS at which point the parameter propagation should happen as part of the ModelToComponentFactory. 1803 parameters=datetime_based_cursor_model.parameters or {}, 1804 ) 1805 cursor_field = CursorField( 1806 cursor_field_key=interpolated_cursor_field.eval(config=config), 1807 supports_catalog_defined_cursor_field=datetime_based_cursor_model.allow_catalog_defined_cursor_field 1808 or False, 1809 ) 1810 1811 datetime_format = datetime_based_cursor_model.datetime_format 1812 1813 cursor_granularity = ( 1814 parse_duration(datetime_based_cursor_model.cursor_granularity) 1815 if datetime_based_cursor_model.cursor_granularity 1816 else None 1817 ) 1818 1819 connector_state_converter: DateTimeStreamStateConverter 1820 connector_state_converter = CustomFormatConcurrentStreamStateConverter( 1821 datetime_format=datetime_format, 1822 input_datetime_formats=datetime_based_cursor_model.cursor_datetime_formats, 1823 is_sequential_state=True, # ConcurrentPerPartitionCursor only works with sequential state 1824 cursor_granularity=cursor_granularity, 1825 ) 1826 1827 # Create the cursor factory 1828 cursor_factory = ConcurrentCursorFactory( 1829 partial( 1830 self.create_concurrent_cursor_from_datetime_based_cursor, 1831 state_manager=state_manager, 1832 model_type=model_type, 1833 component_definition=component_definition, 1834 stream_name=stream_name, 1835 stream_namespace=stream_namespace, 1836 config=config, 1837 message_repository=NoopMessageRepository(), 1838 ) 1839 ) 1840 1841 # Per-partition state doesn't make sense for GroupingPartitionRouter, so force the global state 1842 use_global_cursor = isinstance( 1843 partition_router, GroupingPartitionRouter 1844 ) or component_definition.get("global_substream_cursor", False) 1845 1846 # Return the concurrent cursor and state converter 1847 return ConcurrentPerPartitionCursor( 1848 cursor_factory=cursor_factory, 1849 partition_router=partition_router, 1850 stream_name=stream_name, 1851 stream_namespace=stream_namespace, 1852 stream_state=stream_state, 1853 message_repository=self._message_repository, # type: ignore 1854 connector_state_manager=state_manager, 1855 connector_state_converter=connector_state_converter, 1856 cursor_field=cursor_field, 1857 use_global_cursor=use_global_cursor, 1858 attempt_to_create_cursor_if_not_provided=attempt_to_create_cursor_if_not_provided, 1859 )
@staticmethod
def
create_constant_backoff_strategy( model: airbyte_cdk.sources.declarative.models.declarative_component_schema.ConstantBackoffStrategy, config: Mapping[str, Any], **kwargs: Any) -> airbyte_cdk.sources.declarative.requesters.error_handlers.backoff_strategies.ConstantBackoffStrategy:
1861 @staticmethod 1862 def create_constant_backoff_strategy( 1863 model: ConstantBackoffStrategyModel, config: Config, **kwargs: Any 1864 ) -> ConstantBackoffStrategy: 1865 ModelToComponentFactory._validate_jitter_range(model.jitter_range_in_seconds) 1866 return ConstantBackoffStrategy( 1867 backoff_time_in_seconds=model.backoff_time_in_seconds, 1868 jitter_range_in_seconds=model.jitter_range_in_seconds, 1869 config=config, 1870 parameters=model.parameters or {}, 1871 )
def
create_cursor_pagination( self, model: airbyte_cdk.sources.declarative.models.declarative_component_schema.CursorPagination, config: Mapping[str, Any], decoder: Optional[airbyte_cdk.Decoder] = None, **kwargs: Any) -> airbyte_cdk.CursorPaginationStrategy:
1878 def create_cursor_pagination( 1879 self, 1880 model: CursorPaginationModel, 1881 config: Config, 1882 decoder: Optional[Decoder] = None, 1883 **kwargs: Any, 1884 ) -> CursorPaginationStrategy: 1885 # None when nested under a custom component, which cannot pass a decoder 1886 if decoder is None: 1887 decoder = JsonDecoder(parameters={}) 1888 if isinstance(decoder, PaginationDecoderDecorator): 1889 inner_decoder = decoder.decoder 1890 else: 1891 inner_decoder = decoder 1892 decoder = PaginationDecoderDecorator(decoder=decoder) 1893 1894 if self._is_supported_decoder_for_pagination(inner_decoder): 1895 decoder_to_use = decoder 1896 else: 1897 raise ValueError( 1898 self._UNSUPPORTED_DECODER_ERROR.format(decoder_type=type(inner_decoder)) 1899 ) 1900 1901 # Pydantic v1 Union type coercion can convert int to string depending on Union order. 1902 # If page_size is a string that represents an integer (not an interpolation), convert it back. 1903 page_size = model.page_size 1904 if isinstance(page_size, str) and page_size.isdigit(): 1905 page_size = int(page_size) 1906 1907 return CursorPaginationStrategy( 1908 cursor_value=model.cursor_value, 1909 decoder=decoder_to_use, 1910 page_size=page_size, 1911 stop_condition=model.stop_condition, 1912 config=config, 1913 parameters=model.parameters or {}, 1914 )
def
create_custom_component(self, model: Any, config: Mapping[str, Any], **kwargs: Any) -> Any:
1916 def create_custom_component(self, model: Any, config: Config, **kwargs: Any) -> Any: 1917 """ 1918 Generically creates a custom component based on the model type and a class_name reference to the custom Python class being 1919 instantiated. Only the model's additional properties that match the custom class definition are passed to the constructor 1920 :param model: The Pydantic model of the custom component being created 1921 :param config: The custom defined connector config 1922 :return: The declarative component built from the Pydantic model to be used at runtime 1923 """ 1924 # Instantiating a custom component means importing and executing arbitrary code referenced 1925 # by `class_name`. Manifests supplied by a caller, whether through the config or directly to 1926 # the manifest server, are untrusted input and could point `class_name` at any importable 1927 # callable, so they honor the same `AIRBYTE_ENABLE_UNSAFE_CODE` gate as injected 1928 # `components.py` code. Manifests bundled in a published connector image are trusted and may 1929 # always use their bundled custom components. 1930 manifest_is_untrusted = not self._custom_components_trusted or bool( 1931 config.get(INJECTED_MANIFEST) 1932 ) 1933 if manifest_is_untrusted and not custom_code_execution_permitted(): 1934 raise AirbyteCustomCodeNotPermittedError 1935 1936 custom_component_class = self._get_class_from_fully_qualified_class_name(model.class_name) 1937 component_fields = get_type_hints(custom_component_class) 1938 model_args = model.dict() 1939 model_args["config"] = config 1940 1941 # There are cases where a parent component will pass arguments to a child component via kwargs. When there are field collisions 1942 # we defer to these arguments over the component's definition 1943 for key, arg in kwargs.items(): 1944 model_args[key] = arg 1945 1946 # Pydantic is unable to parse a custom component's fields that are subcomponents into models because their fields and types are not 1947 # defined in the schema. The fields and types are defined within the Python class implementation. Pydantic can only parse down to 1948 # the custom component and this code performs a second parse to convert the sub-fields first into models, then declarative components 1949 for model_field, model_value in model_args.items(): 1950 # If a custom component field doesn't have a type set, we try to use the type hints to infer the type 1951 if ( 1952 isinstance(model_value, dict) 1953 and "type" not in model_value 1954 and model_field in component_fields 1955 ): 1956 derived_type = self._derive_component_type_from_type_hints( 1957 component_fields.get(model_field) 1958 ) 1959 if derived_type: 1960 model_value["type"] = derived_type 1961 1962 if self._is_component(model_value): 1963 model_args[model_field] = self._create_nested_component( 1964 model, 1965 model_field, 1966 model_value, 1967 config, 1968 **kwargs, 1969 ) 1970 elif isinstance(model_value, list): 1971 vals = [] 1972 for v in model_value: 1973 if isinstance(v, dict) and "type" not in v and model_field in component_fields: 1974 derived_type = self._derive_component_type_from_type_hints( 1975 component_fields.get(model_field) 1976 ) 1977 if derived_type: 1978 v["type"] = derived_type 1979 if self._is_component(v): 1980 vals.append( 1981 self._create_nested_component( 1982 model, 1983 model_field, 1984 v, 1985 config, 1986 **kwargs, 1987 ) 1988 ) 1989 else: 1990 vals.append(v) 1991 model_args[model_field] = vals 1992 1993 kwargs = { 1994 class_field: model_args[class_field] 1995 for class_field in component_fields.keys() 1996 if class_field in model_args 1997 } 1998 1999 if "api_budget" in component_fields and kwargs.get("api_budget") is None: 2000 kwargs["api_budget"] = self._api_budget 2001 2002 return custom_component_class(**kwargs)
Generically creates a custom component based on the model type and a class_name reference to the custom Python class being instantiated. Only the model's additional properties that match the custom class definition are passed to the constructor
Parameters
- model: The Pydantic model of the custom component being created
- config: The custom defined connector config
Returns
The declarative component built from the Pydantic model to be used at runtime
def
create_default_stream( self, model: airbyte_cdk.sources.declarative.models.declarative_component_schema.DeclarativeStream, config: Mapping[str, Any], is_parent: bool = False, **kwargs: Any) -> airbyte_cdk.sources.streams.concurrent.abstract_stream.AbstractStream:
2137 def create_default_stream( 2138 self, model: DeclarativeStreamModel, config: Config, is_parent: bool = False, **kwargs: Any 2139 ) -> AbstractStream: 2140 primary_key = model.primary_key.__root__ if model.primary_key else None 2141 self._validate_sync_token_stream(model) 2142 self._migrate_state(model, config) 2143 self._warn_on_ineffective_incremental_dependency(model) 2144 2145 partition_router = self._build_stream_slicer_from_partition_router( 2146 model.retriever, 2147 config, 2148 stream_name=model.name, 2149 **kwargs, 2150 ) 2151 concurrent_cursor = self._build_concurrent_cursor(model, partition_router, config) 2152 request_options_provider: Optional[RequestOptionsProvider] 2153 if model.incremental_sync and isinstance(model.incremental_sync, DatetimeBasedCursorModel): 2154 cursor_model: DatetimeBasedCursorModel = model.incremental_sync 2155 2156 end_time_option = ( 2157 self._create_component_from_model( 2158 cursor_model.end_time_option, config, parameters=cursor_model.parameters or {} 2159 ) 2160 if cursor_model.end_time_option 2161 else None 2162 ) 2163 start_time_option = ( 2164 self._create_component_from_model( 2165 cursor_model.start_time_option, config, parameters=cursor_model.parameters or {} 2166 ) 2167 if cursor_model.start_time_option 2168 else None 2169 ) 2170 2171 datetime_request_options_provider = DatetimeBasedRequestOptionsProvider( 2172 start_time_option=start_time_option, 2173 end_time_option=end_time_option, 2174 partition_field_start=cursor_model.partition_field_start, 2175 partition_field_end=cursor_model.partition_field_end, 2176 config=config, 2177 parameters=model.parameters or {}, 2178 ) 2179 request_options_provider = ( 2180 datetime_request_options_provider 2181 if not isinstance(concurrent_cursor, ConcurrentPerPartitionCursor) 2182 else PerPartitionRequestOptionsProvider( 2183 partition_router, datetime_request_options_provider 2184 ) 2185 ) 2186 elif isinstance(model.incremental_sync, SyncTokenCursorModel): 2187 sync_token_cursor_model = model.incremental_sync 2188 token_option = self._create_component_from_model( 2189 sync_token_cursor_model.token_option, 2190 config, 2191 parameters=sync_token_cursor_model.parameters or {}, 2192 ) 2193 sync_token_request_options_provider = SyncTokenRequestOptionsProvider( 2194 token_option=token_option, 2195 config=config, 2196 parameters=sync_token_cursor_model.parameters or {}, 2197 initial_token=sync_token_cursor_model.initial_token, 2198 ) 2199 request_options_provider = ( 2200 PerPartitionRequestOptionsProvider( 2201 partition_router, sync_token_request_options_provider 2202 ) 2203 if isinstance(concurrent_cursor, SyncTokenCursor) and concurrent_cursor.partitioned 2204 else sync_token_request_options_provider 2205 ) 2206 elif model.incremental_sync and isinstance( 2207 model.incremental_sync, IncrementingCountCursorModel 2208 ): 2209 if isinstance(concurrent_cursor, ConcurrentPerPartitionCursor): 2210 raise ValueError( 2211 "PerPartition does not support per partition states because switching to global state is time based" 2212 ) 2213 2214 cursor_model: IncrementingCountCursorModel = model.incremental_sync # type: ignore 2215 2216 start_time_option = ( 2217 self._create_component_from_model( 2218 cursor_model.start_value_option, # type: ignore # mypy still thinks cursor_model of type DatetimeBasedCursor 2219 config, 2220 parameters=cursor_model.parameters or {}, 2221 ) 2222 if cursor_model.start_value_option # type: ignore # mypy still thinks cursor_model of type DatetimeBasedCursor 2223 else None 2224 ) 2225 2226 # The concurrent engine defaults the start/end fields on the slice to "start" and "end", but 2227 # the default DatetimeBasedRequestOptionsProvider() sets them to start_time/end_time 2228 partition_field_start = "start" 2229 2230 request_options_provider = DatetimeBasedRequestOptionsProvider( 2231 start_time_option=start_time_option, 2232 partition_field_start=partition_field_start, 2233 config=config, 2234 parameters=model.parameters or {}, 2235 ) 2236 else: 2237 request_options_provider = None 2238 2239 transformations = [] 2240 if model.transformations: 2241 for transformation_model in model.transformations: 2242 transformations.append( 2243 self._create_component_from_model(model=transformation_model, config=config) 2244 ) 2245 file_uploader = None 2246 if model.file_uploader: 2247 file_uploader = self._create_component_from_model( 2248 model=model.file_uploader, config=config 2249 ) 2250 2251 stream_slicer: ConcurrentStreamSlicer = ( 2252 partition_router 2253 if isinstance(concurrent_cursor, FinalStateCursor) 2254 else concurrent_cursor 2255 ) 2256 2257 retriever = self._create_component_from_model( 2258 model=model.retriever, 2259 config=config, 2260 name=model.name, 2261 primary_key=primary_key, 2262 request_options_provider=request_options_provider, 2263 stream_slicer=stream_slicer, 2264 partition_router=partition_router, 2265 has_stop_condition_cursor=self._is_stop_condition_on_cursor(model), 2266 is_client_side_incremental_sync=self._is_client_side_filtering_enabled(model), 2267 cursor=concurrent_cursor, 2268 transformations=transformations, 2269 file_uploader=file_uploader, 2270 incremental_sync=model.incremental_sync, 2271 ) 2272 if isinstance(retriever, AsyncRetriever): 2273 stream_slicer = retriever.stream_slicer 2274 2275 schema_loader: SchemaLoader 2276 if model.schema_loader and isinstance(model.schema_loader, list): 2277 nested_schema_loaders = [ 2278 self._create_component_from_model(model=nested_schema_loader, config=config) 2279 for nested_schema_loader in model.schema_loader 2280 ] 2281 schema_loader = CompositeSchemaLoader( 2282 schema_loaders=nested_schema_loaders, parameters={} 2283 ) 2284 elif model.schema_loader: 2285 schema_loader = self._create_component_from_model( 2286 model=model.schema_loader, # type: ignore # If defined, schema_loader is guaranteed not to be a list and will be one of the existing base models 2287 config=config, 2288 ) 2289 else: 2290 options = model.parameters or {} 2291 if "name" not in options: 2292 options["name"] = model.name 2293 schema_loader = DefaultSchemaLoader(config=config, parameters=options) 2294 schema_loader = CachingSchemaLoaderDecorator(schema_loader) 2295 2296 stream_name = model.name or "" 2297 return DefaultStream( 2298 partition_generator=StreamSlicerPartitionGenerator( 2299 DeclarativePartitionFactory( 2300 stream_name, 2301 schema_loader, 2302 retriever, 2303 self._message_repository, 2304 ), 2305 stream_slicer, 2306 slice_limit=self._limit_slices_fetched, 2307 ), 2308 name=stream_name, 2309 json_schema=schema_loader.get_json_schema, 2310 primary_key=get_primary_key_from_stream(primary_key), 2311 cursor_field=( 2312 concurrent_cursor.cursor_field 2313 if hasattr(concurrent_cursor, "cursor_field") 2314 else None 2315 ), 2316 logger=logging.getLogger(f"airbyte.{stream_name}"), 2317 cursor=concurrent_cursor, 2318 supports_file_transfer=hasattr(model, "file_uploader") and bool(model.file_uploader), 2319 source_defined_incremental=isinstance(model.incremental_sync, SyncTokenCursorModel), 2320 )
def
create_sync_token_cursor( self, model: airbyte_cdk.sources.declarative.models.declarative_component_schema.SyncTokenCursor, *, config: Mapping[str, Any], stream_name: str, stream_state: Optional[Mapping[str, Any]], partition_router: airbyte_cdk.sources.declarative.partition_routers.PartitionRouter, retriever_model: Any) -> airbyte_cdk.sources.declarative.incremental.SyncTokenCursor:
2645 def create_sync_token_cursor( 2646 self, 2647 model: SyncTokenCursorModel, 2648 *, 2649 config: Config, 2650 stream_name: str, 2651 stream_state: Optional[Mapping[str, Any]], 2652 partition_router: PartitionRouter, 2653 retriever_model: Any, 2654 ) -> SyncTokenCursor: 2655 decoder_model = getattr(retriever_model, "decoder", None) 2656 decoder = ( 2657 self._create_component_from_model(model=decoder_model, config=config) 2658 if isinstance(decoder_model, BaseModel) 2659 else JsonDecoder(parameters={}) 2660 ) 2661 if self._decoder_streams_in_production(decoder): 2662 raise AirbyteTracedException( 2663 message=f"Stream {stream_name}: SyncTokenCursor is not supported with a streaming decoder.", 2664 internal_message=( 2665 f"Stream {stream_name} uses {type(decoder).__name__}, which streams the response " 2666 f"and can only read it once. SyncTokenCursor reads the token from the response " 2667 f"after the record selector has consumed it. Use a non-streaming decoder such " 2668 f"as JsonDecoder or XmlDecoder." 2669 ), 2670 failure_type=FailureType.config_error, 2671 ) 2672 return SyncTokenCursor( 2673 stream_name=stream_name, 2674 stream_namespace=None, 2675 stream_state=stream_state, 2676 message_repository=self._message_repository, 2677 connector_state_manager=self._connector_state_manager, 2678 partition_router=partition_router, 2679 token_path=model.token_path, 2680 decoder=decoder, 2681 config=config or {}, 2682 parameters=model.parameters or {}, 2683 completion_condition=model.completion_condition, 2684 cursor_field=( 2685 InterpolatedString.create( 2686 model.cursor_field, parameters=model.parameters or {} 2687 ).eval(config=config) 2688 if model.cursor_field 2689 else None 2690 ), 2691 )
def
create_default_error_handler( self, model: airbyte_cdk.sources.declarative.models.declarative_component_schema.DefaultErrorHandler, config: Mapping[str, Any], **kwargs: Any) -> airbyte_cdk.sources.declarative.requesters.error_handlers.DefaultErrorHandler:
2693 def create_default_error_handler( 2694 self, model: DefaultErrorHandlerModel, config: Config, **kwargs: Any 2695 ) -> DefaultErrorHandler: 2696 backoff_strategies = [] 2697 if model.backoff_strategies: 2698 for backoff_strategy_model in model.backoff_strategies: 2699 backoff_strategies.append( 2700 self._create_component_from_model(model=backoff_strategy_model, config=config) 2701 ) 2702 2703 response_filters = [] 2704 if model.response_filters: 2705 for response_filter_model in model.response_filters: 2706 response_filters.append( 2707 self._create_component_from_model(model=response_filter_model, config=config) 2708 ) 2709 response_filters.append( 2710 HttpResponseFilter(config=config, parameters=model.parameters or {}) 2711 ) 2712 2713 return DefaultErrorHandler( 2714 backoff_strategies=backoff_strategies, 2715 max_retries=model.max_retries, 2716 response_filters=response_filters, 2717 config=config, 2718 parameters=model.parameters or {}, 2719 )
def
create_default_paginator( self, model: airbyte_cdk.sources.declarative.models.declarative_component_schema.DefaultPaginator, config: Mapping[str, Any], *, url_base: str, extractor_model: Union[airbyte_cdk.sources.declarative.models.declarative_component_schema.CustomRecordExtractor, airbyte_cdk.sources.declarative.models.declarative_component_schema.DpathExtractor, airbyte_cdk.sources.declarative.models.declarative_component_schema.CombinedExtractor, NoneType] = None, decoder: Optional[airbyte_cdk.Decoder] = None, cursor_used_for_stop_condition: Optional[airbyte_cdk.Cursor] = None) -> Union[airbyte_cdk.DefaultPaginator, airbyte_cdk.sources.declarative.requesters.paginators.PaginatorTestReadDecorator]:
2721 def create_default_paginator( 2722 self, 2723 model: DefaultPaginatorModel, 2724 config: Config, 2725 *, 2726 url_base: str, 2727 extractor_model: Optional[ 2728 Union[CustomRecordExtractorModel, DpathExtractorModel, CombinedExtractorModel] 2729 ] = None, 2730 decoder: Optional[Decoder] = None, 2731 cursor_used_for_stop_condition: Optional[Cursor] = None, 2732 ) -> Union[DefaultPaginator, PaginatorTestReadDecorator]: 2733 if decoder: 2734 if self._is_supported_decoder_for_pagination(decoder): 2735 decoder_to_use = PaginationDecoderDecorator(decoder=decoder) 2736 else: 2737 raise ValueError(self._UNSUPPORTED_DECODER_ERROR.format(decoder_type=type(decoder))) 2738 else: 2739 decoder_to_use = PaginationDecoderDecorator(decoder=JsonDecoder(parameters={})) 2740 page_size_option = ( 2741 self._create_component_from_model(model=model.page_size_option, config=config) 2742 if model.page_size_option 2743 else None 2744 ) 2745 page_token_option = ( 2746 self._create_component_from_model(model=model.page_token_option, config=config) 2747 if model.page_token_option 2748 else None 2749 ) 2750 pagination_strategy = self._create_component_from_model( 2751 model=model.pagination_strategy, 2752 config=config, 2753 decoder=decoder_to_use, 2754 extractor_model=extractor_model, 2755 ) 2756 if cursor_used_for_stop_condition: 2757 pagination_strategy = StopConditionPaginationStrategyDecorator( 2758 pagination_strategy, CursorStopCondition(cursor_used_for_stop_condition) 2759 ) 2760 paginator = DefaultPaginator( 2761 decoder=decoder_to_use, 2762 page_size_option=page_size_option, 2763 page_token_option=page_token_option, 2764 pagination_strategy=pagination_strategy, 2765 url_base=url_base, 2766 config=config, 2767 parameters=model.parameters or {}, 2768 ) 2769 if self._limit_pages_fetched_per_slice: 2770 return PaginatorTestReadDecorator(paginator, self._limit_pages_fetched_per_slice) 2771 return paginator
def
create_combined_extractor( self, model: airbyte_cdk.sources.declarative.models.declarative_component_schema.CombinedExtractor, config: Mapping[str, Any], decoder: Optional[airbyte_cdk.Decoder] = None, **kwargs: Any) -> airbyte_cdk.CombinedExtractor:
2837 def create_combined_extractor( 2838 self, 2839 model: CombinedExtractorModel, 2840 config: Config, 2841 decoder: Optional[Decoder] = None, 2842 **kwargs: Any, 2843 ) -> CombinedExtractor: 2844 self._reject_combined_extractor_over_streaming_decoder(decoder) 2845 extractors = [ 2846 self._create_component_from_model(model=sub_extractor, config=config, decoder=decoder) 2847 for sub_extractor in model.extractors 2848 ] 2849 return CombinedExtractor( 2850 extractors=extractors, 2851 mode=CombineMode(model.mode.value) if model.mode else CombineMode.union, 2852 skip_empty_records=bool(model.skip_empty_records), 2853 parameters=model.parameters or {}, 2854 )
def
create_dpath_extractor( self, model: airbyte_cdk.sources.declarative.models.declarative_component_schema.DpathExtractor, config: Mapping[str, Any], decoder: Optional[airbyte_cdk.Decoder] = None, **kwargs: Any) -> airbyte_cdk.DpathExtractor:
2856 def create_dpath_extractor( 2857 self, 2858 model: DpathExtractorModel, 2859 config: Config, 2860 decoder: Optional[Decoder] = None, 2861 **kwargs: Any, 2862 ) -> DpathExtractor: 2863 if decoder: 2864 decoder_to_use = decoder 2865 else: 2866 decoder_to_use = JsonDecoder(parameters={}) 2867 model_field_path: List[Union[InterpolatedString, str]] = [x for x in model.field_path] 2868 2869 record_expander = None 2870 if model.record_expander: 2871 record_expander = self._create_component_from_model( 2872 model=model.record_expander, 2873 config=config, 2874 ) 2875 2876 return DpathExtractor( 2877 decoder=decoder_to_use, 2878 field_path=model_field_path, 2879 config=config, 2880 parameters=model.parameters or {}, 2881 record_expander=record_expander, 2882 )
def
create_record_expander( self, model: airbyte_cdk.sources.declarative.models.declarative_component_schema.RecordExpander, config: Mapping[str, Any], **kwargs: Any) -> airbyte_cdk.sources.declarative.expanders.RecordExpander:
2884 def create_record_expander( 2885 self, 2886 model: RecordExpanderModel, 2887 config: Config, 2888 **kwargs: Any, 2889 ) -> RecordExpander: 2890 truncated_list_retriever = None 2891 suppress_incomplete_fetch_warning = False 2892 if model.truncated_list_retriever: 2893 retriever_model = model.truncated_list_retriever 2894 name = "record_expander_truncated_list" 2895 # `CustomRetriever` allows extra fields, so read from the dumped model to cover both types. 2896 retriever_fields = retriever_model.dict() 2897 for unsupported_option in ("partition_router", "pagination_reset"): 2898 if retriever_fields.get(unsupported_option): 2899 raise ValueError( 2900 f"`{unsupported_option}` is not supported on `truncated_list_retriever`." 2901 ) 2902 log_formatter = lambda response: format_http_message( 2903 response, 2904 f"Record expander '{name}' request", 2905 "Request performed in order to fetch the complete nested list of a truncated record.", 2906 name, 2907 is_auxiliary=True, 2908 ) 2909 # `name`/`primary_key`/`transformations` are also what a `CustomRetriever` forwards to its 2910 # nested `requester`/`record_selector`, so they are passed for both retriever types. 2911 truncated_list_retriever = self._create_component_from_model( 2912 model=retriever_model, 2913 config=config, 2914 name=name, 2915 primary_key=None, 2916 transformations=[], 2917 log_formatter=log_formatter, 2918 ) 2919 # Only a capped paginator makes a shortfall expected; `NoPagination` and retrievers 2920 # without a paginator are never capped. 2921 suppress_incomplete_fetch_warning = isinstance( 2922 truncated_list_retriever, SimpleRetriever 2923 ) and isinstance(truncated_list_retriever.paginator, PaginatorTestReadDecorator) 2924 return RecordExpander( 2925 expand_records_from_field=model.expand_records_from_field, 2926 config=config, 2927 parameters=model.parameters or {}, 2928 remain_original_record=model.remain_original_record or False, 2929 parent_fields=[ 2930 self._create_component_from_model(model=parent_field, config=config) 2931 for parent_field in model.parent_fields 2932 ] 2933 if model.parent_fields 2934 else None, 2935 merge_parent=model.merge_parent or False, 2936 on_no_records=OnNoRecords(model.on_no_records.value) 2937 if model.on_no_records 2938 else OnNoRecords.skip, 2939 truncation_indicator_path=model.truncation_indicator_path, 2940 truncated_list_retriever=truncated_list_retriever, 2941 message_repository=self._message_repository, 2942 suppress_incomplete_fetch_warning=suppress_incomplete_fetch_warning, 2943 )
@staticmethod
def
create_parent_field_path( model: airbyte_cdk.sources.declarative.models.declarative_component_schema.ParentFieldPath, config: Mapping[str, Any], **kwargs: Any) -> airbyte_cdk.sources.declarative.expanders.ParentFieldPath:
2945 @staticmethod 2946 def create_parent_field_path( 2947 model: ParentFieldPathModel, 2948 config: Config, 2949 **kwargs: Any, 2950 ) -> ParentFieldPath: 2951 return ParentFieldPath( 2952 parent_path=model.parent_path, 2953 record_path=model.record_path, 2954 config=config, 2955 parameters=model.parameters or {}, 2956 )
@staticmethod
def
create_response_to_file_extractor( model: airbyte_cdk.sources.declarative.models.declarative_component_schema.ResponseToFileExtractor, **kwargs: Any) -> airbyte_cdk.sources.declarative.extractors.ResponseToFileExtractor:
2958 @staticmethod 2959 def create_response_to_file_extractor( 2960 model: ResponseToFileExtractorModel, 2961 **kwargs: Any, 2962 ) -> ResponseToFileExtractor: 2963 return ResponseToFileExtractor( 2964 parameters=model.parameters or {}, 2965 preserve_na_values=model.preserve_na_values or False, 2966 )
@staticmethod
def
create_exponential_backoff_strategy( model: airbyte_cdk.sources.declarative.models.declarative_component_schema.ExponentialBackoffStrategy, config: Mapping[str, Any]) -> airbyte_cdk.sources.declarative.requesters.error_handlers.backoff_strategies.ExponentialBackoffStrategy:
2968 @staticmethod 2969 def create_exponential_backoff_strategy( 2970 model: ExponentialBackoffStrategyModel, config: Config 2971 ) -> ExponentialBackoffStrategy: 2972 ModelToComponentFactory._validate_jitter_range(model.jitter_range_in_seconds) 2973 return ExponentialBackoffStrategy( 2974 factor=model.factor or 5, 2975 jitter_range_in_seconds=model.jitter_range_in_seconds, 2976 parameters=model.parameters or {}, 2977 config=config, 2978 )
@staticmethod
def
create_group_by_key( model: airbyte_cdk.sources.declarative.models.declarative_component_schema.GroupByKeyMergeStrategy, config: Mapping[str, Any]) -> airbyte_cdk.sources.declarative.requesters.query_properties.strategies.GroupByKey:
def
create_http_requester( self, model: airbyte_cdk.sources.declarative.models.declarative_component_schema.HttpRequester, config: Mapping[str, Any], decoder: airbyte_cdk.Decoder = JsonDecoder(), query_properties_key: Optional[str] = None, use_cache: Optional[bool] = None, *, name: str) -> airbyte_cdk.HttpRequester:
2984 def create_http_requester( 2985 self, 2986 model: HttpRequesterModel, 2987 config: Config, 2988 decoder: Decoder = JsonDecoder(parameters={}), 2989 query_properties_key: Optional[str] = None, 2990 use_cache: Optional[bool] = None, 2991 *, 2992 name: str, 2993 ) -> HttpRequester: 2994 authenticator = ( 2995 self._create_component_from_model( 2996 model=model.authenticator, 2997 config=config, 2998 url_base=model.url or model.url_base, 2999 name=name, 3000 decoder=decoder, 3001 ) 3002 if model.authenticator 3003 else None 3004 ) 3005 error_handler = ( 3006 self._create_component_from_model(model=model.error_handler, config=config) 3007 if model.error_handler 3008 else DefaultErrorHandler( 3009 backoff_strategies=[], 3010 response_filters=[], 3011 config=config, 3012 parameters=model.parameters or {}, 3013 ) 3014 ) 3015 3016 api_budget = self._api_budget 3017 3018 request_options_provider = InterpolatedRequestOptionsProvider( 3019 request_body=model.request_body, 3020 request_body_data=model.request_body_data, 3021 request_body_json=model.request_body_json, 3022 request_headers=model.request_headers, 3023 request_parameters=model.request_parameters, # type: ignore # QueryProperties have been removed in `create_simple_retriever` 3024 query_properties_key=query_properties_key, 3025 config=config, 3026 parameters=model.parameters or {}, 3027 ) 3028 3029 assert model.use_cache is not None # for mypy 3030 assert model.http_method is not None # for mypy 3031 3032 should_use_cache = (model.use_cache or bool(use_cache)) and not self._disable_cache 3033 3034 return HttpRequester( 3035 name=name, 3036 url=model.url, 3037 url_base=model.url_base, 3038 path=model.path, 3039 authenticator=authenticator, 3040 error_handler=error_handler, 3041 api_budget=api_budget, 3042 http_method=HttpMethod[model.http_method.value], 3043 request_options_provider=request_options_provider, 3044 config=config, 3045 disable_retries=self._disable_retries, 3046 parameters=model.parameters or {}, 3047 message_repository=self._message_repository, 3048 use_cache=should_use_cache, 3049 connect_timeout_in_seconds=model.connect_timeout_in_seconds, 3050 read_timeout_in_seconds=model.read_timeout_in_seconds, 3051 use_tcp_keepalive=model.use_tcp_keepalive or False, 3052 decoder=decoder, 3053 stream_response=decoder.is_stream_response() if decoder else False, 3054 )
@staticmethod
def
create_http_response_filter( model: airbyte_cdk.sources.declarative.models.declarative_component_schema.HttpResponseFilter, config: Mapping[str, Any], **kwargs: Any) -> airbyte_cdk.sources.declarative.requesters.error_handlers.HttpResponseFilter:
3056 @staticmethod 3057 def create_http_response_filter( 3058 model: HttpResponseFilterModel, config: Config, **kwargs: Any 3059 ) -> HttpResponseFilter: 3060 if model.action: 3061 action = ResponseAction(model.action.value) 3062 else: 3063 action = None 3064 3065 failure_type = FailureType(model.failure_type.value) if model.failure_type else None 3066 3067 http_codes = ( 3068 set(model.http_codes) if model.http_codes else set() 3069 ) # JSON schema notation has no set data type. The schema enforces an array of unique elements 3070 3071 return HttpResponseFilter( 3072 action=action, 3073 failure_type=failure_type, 3074 error_message=model.error_message or "", 3075 error_message_contains=model.error_message_contains or "", 3076 http_codes=http_codes, 3077 predicate=model.predicate or "", 3078 config=config, 3079 parameters=model.parameters or {}, 3080 )
@staticmethod
def
create_inline_schema_loader( model: airbyte_cdk.sources.declarative.models.declarative_component_schema.InlineSchemaLoader, config: Mapping[str, Any], **kwargs: Any) -> airbyte_cdk.sources.declarative.schema.InlineSchemaLoader:
def
create_complex_field_type( self, model: airbyte_cdk.sources.declarative.models.declarative_component_schema.ComplexFieldType, config: Mapping[str, Any], **kwargs: Any) -> airbyte_cdk.sources.declarative.schema.ComplexFieldType:
3088 def create_complex_field_type( 3089 self, model: ComplexFieldTypeModel, config: Config, **kwargs: Any 3090 ) -> ComplexFieldType: 3091 items = ( 3092 self._create_component_from_model(model=model.items, config=config) 3093 if isinstance(model.items, ComplexFieldTypeModel) 3094 else model.items 3095 ) 3096 3097 return ComplexFieldType(field_type=model.field_type, items=items)
def
create_types_map( self, model: airbyte_cdk.sources.declarative.models.declarative_component_schema.TypesMap, config: Mapping[str, Any], **kwargs: Any) -> airbyte_cdk.sources.declarative.schema.TypesMap:
3099 def create_types_map(self, model: TypesMapModel, config: Config, **kwargs: Any) -> TypesMap: 3100 target_type = ( 3101 self._create_component_from_model(model=model.target_type, config=config) 3102 if isinstance(model.target_type, ComplexFieldTypeModel) 3103 else model.target_type 3104 ) 3105 3106 return TypesMap( 3107 target_type=target_type, 3108 current_type=model.current_type, 3109 condition=model.condition if model.condition is not None else "True", 3110 )
def
create_schema_type_identifier( self, model: airbyte_cdk.sources.declarative.models.declarative_component_schema.SchemaTypeIdentifier, config: Mapping[str, Any], **kwargs: Any) -> airbyte_cdk.sources.declarative.schema.SchemaTypeIdentifier:
3112 def create_schema_type_identifier( 3113 self, model: SchemaTypeIdentifierModel, config: Config, **kwargs: Any 3114 ) -> SchemaTypeIdentifier: 3115 types_mapping = [] 3116 if model.types_mapping: 3117 types_mapping.extend( 3118 [ 3119 self._create_component_from_model(types_map, config=config) 3120 for types_map in model.types_mapping 3121 ] 3122 ) 3123 model_schema_pointer: List[Union[InterpolatedString, str]] = ( 3124 [x for x in model.schema_pointer] if model.schema_pointer else [] 3125 ) 3126 model_key_pointer: List[Union[InterpolatedString, str]] = [x for x in model.key_pointer] 3127 model_type_pointer: Optional[List[Union[InterpolatedString, str]]] = ( 3128 [x for x in model.type_pointer] if model.type_pointer else None 3129 ) 3130 3131 return SchemaTypeIdentifier( 3132 schema_pointer=model_schema_pointer, 3133 key_pointer=model_key_pointer, 3134 type_pointer=model_type_pointer, 3135 types_mapping=types_mapping, 3136 parameters=model.parameters or {}, 3137 )
def
create_dynamic_schema_loader( self, model: airbyte_cdk.sources.declarative.models.declarative_component_schema.DynamicSchemaLoader, config: Mapping[str, Any], **kwargs: Any) -> airbyte_cdk.sources.declarative.schema.DynamicSchemaLoader:
3139 def create_dynamic_schema_loader( 3140 self, model: DynamicSchemaLoaderModel, config: Config, **kwargs: Any 3141 ) -> DynamicSchemaLoader: 3142 schema_transformations = [] 3143 if model.schema_transformations: 3144 for transformation_model in model.schema_transformations: 3145 schema_transformations.append( 3146 self._create_component_from_model(model=transformation_model, config=config) 3147 ) 3148 name = "dynamic_properties" 3149 partition_router = self._build_stream_slicer_from_partition_router(model.retriever, config) 3150 retriever_kwargs: Dict[str, Any] = {} 3151 if isinstance(model.retriever, AsyncRetrieverModel): 3152 if model.retriever.partition_router: 3153 raise ValueError( 3154 "DynamicSchemaLoader does not support an AsyncRetriever with a partition_router: only the first record is read, so jobs for other partitions would be orphaned. Remove the partition_router from the schema loader's AsyncRetriever." 3155 ) 3156 # create_async_retriever requires stream_slicer; passing it to a CustomRetriever would override its own 3157 retriever_kwargs["stream_slicer"] = partition_router 3158 retriever = self._create_component_from_model( 3159 model=model.retriever, 3160 config=config, 3161 name=name, 3162 primary_key=None, 3163 partition_router=partition_router, 3164 **retriever_kwargs, 3165 transformations=[], 3166 use_cache=True, 3167 log_formatter=( 3168 lambda response: format_http_message( 3169 response, 3170 f"Schema loader '{name}' request", 3171 f"Request performed in order to extract schema.", 3172 name, 3173 is_auxiliary=True, 3174 ) 3175 ), 3176 ) 3177 schema_type_identifier = self._create_component_from_model( 3178 model.schema_type_identifier, config=config, parameters=model.parameters or {} 3179 ) 3180 schema_filter = ( 3181 self._create_component_from_model( 3182 model.schema_filter, config=config, parameters=model.parameters or {} 3183 ) 3184 if model.schema_filter is not None 3185 else None 3186 ) 3187 3188 return DynamicSchemaLoader( 3189 retriever=retriever, 3190 config=config, 3191 schema_transformations=schema_transformations, 3192 schema_filter=schema_filter, 3193 schema_type_identifier=schema_type_identifier, 3194 parameters=model.parameters or {}, 3195 )
@staticmethod
def
create_json_decoder( model: airbyte_cdk.sources.declarative.models.declarative_component_schema.JsonDecoder, config: Mapping[str, Any], **kwargs: Any) -> airbyte_cdk.Decoder:
def
create_csv_decoder( self, model: airbyte_cdk.sources.declarative.models.declarative_component_schema.CsvDecoder, config: Mapping[str, Any], **kwargs: Any) -> airbyte_cdk.Decoder:
def
create_jsonl_decoder( self, model: airbyte_cdk.sources.declarative.models.declarative_component_schema.JsonlDecoder, config: Mapping[str, Any], **kwargs: Any) -> airbyte_cdk.Decoder:
def
create_json_items_decoder( self, model: airbyte_cdk.sources.declarative.models.declarative_component_schema.JsonItemsDecoder, config: Mapping[str, Any], **kwargs: Any) -> airbyte_cdk.Decoder:
3215 def create_json_items_decoder( 3216 self, model: JsonItemsDecoderModel, config: Config, **kwargs: Any 3217 ) -> Decoder: 3218 return CompositeRawDecoder( 3219 parser=ModelToComponentFactory._get_parser(model, config), 3220 stream_response=False if self._emit_connector_builder_messages else True, 3221 )
def
create_gzip_decoder( self, model: airbyte_cdk.sources.declarative.models.declarative_component_schema.GzipDecoder, config: Mapping[str, Any], **kwargs: Any) -> airbyte_cdk.Decoder:
3223 def create_gzip_decoder( 3224 self, model: GzipDecoderModel, config: Config, **kwargs: Any 3225 ) -> Decoder: 3226 _compressed_response_types = { 3227 "gzip", 3228 "x-gzip", 3229 "gzip, deflate", 3230 "x-gzip, deflate", 3231 "application/zip", 3232 "application/gzip", 3233 "application/x-gzip", 3234 "application/x-zip-compressed", 3235 } 3236 3237 gzip_parser: GzipParser = ModelToComponentFactory._get_parser(model, config) # type: ignore # based on the model, we know this will be a GzipParser 3238 3239 if self._emit_connector_builder_messages: 3240 return CompositeRawDecoder(gzip_parser, False) 3241 3242 transport_gzip_parser = GzipParser(inner_parser=gzip_parser) 3243 return CompositeRawDecoder.by_headers( 3244 [ 3245 ({"Content-Encoding"}, {"gzip"}, transport_gzip_parser), 3246 ({"Content-Type"}, _compressed_response_types, gzip_parser), 3247 ], 3248 stream_response=True, 3249 fallback_parser=gzip_parser, 3250 )
@staticmethod
def
create_iterable_decoder( model: airbyte_cdk.sources.declarative.models.declarative_component_schema.IterableDecoder, config: Mapping[str, Any], **kwargs: Any) -> airbyte_cdk.sources.declarative.decoders.IterableDecoder:
@staticmethod
def
create_xml_decoder( model: airbyte_cdk.sources.declarative.models.declarative_component_schema.XmlDecoder, config: Mapping[str, Any], **kwargs: Any) -> airbyte_cdk.sources.declarative.decoders.XmlDecoder:
def
create_zipfile_decoder( self, model: airbyte_cdk.sources.declarative.models.declarative_component_schema.ZipfileDecoder, config: Mapping[str, Any], **kwargs: Any) -> airbyte_cdk.sources.declarative.decoders.ZipfileDecoder:
@staticmethod
def
create_json_file_schema_loader( model: airbyte_cdk.sources.declarative.models.declarative_component_schema.JsonFileSchemaLoader, config: Mapping[str, Any], **kwargs: Any) -> airbyte_cdk.JsonFileSchemaLoader:
def
create_jwt_authenticator( self, model: airbyte_cdk.sources.declarative.models.declarative_component_schema.JwtAuthenticator, config: Mapping[str, Any], **kwargs: Any) -> airbyte_cdk.sources.declarative.auth.JwtAuthenticator:
3304 def create_jwt_authenticator( 3305 self, model: JwtAuthenticatorModel, config: Config, **kwargs: Any 3306 ) -> JwtAuthenticator: 3307 jwt_headers = model.jwt_headers or JwtHeadersModel(kid=None, typ="JWT", cty=None) 3308 jwt_payload = model.jwt_payload or JwtPayloadModel(iss=None, sub=None, aud=None) 3309 request_option = ( 3310 self._create_component_from_model(model.request_option, config) 3311 if model.request_option 3312 else None 3313 ) 3314 return JwtAuthenticator( 3315 config=config, 3316 parameters=model.parameters or {}, 3317 algorithm=JwtAlgorithm(model.algorithm.value), 3318 secret_key=model.secret_key, 3319 base64_encode_secret_key=model.base64_encode_secret_key, 3320 token_duration=model.token_duration, 3321 header_prefix=model.header_prefix, 3322 kid=jwt_headers.kid, 3323 typ=jwt_headers.typ, 3324 cty=jwt_headers.cty, 3325 iss=jwt_payload.iss, 3326 sub=jwt_payload.sub, 3327 aud=jwt_payload.aud, 3328 additional_jwt_headers=model.additional_jwt_headers, 3329 additional_jwt_payload=model.additional_jwt_payload, 3330 passphrase=model.passphrase, 3331 request_option=request_option, 3332 )
def
create_list_partition_router( self, model: airbyte_cdk.sources.declarative.models.declarative_component_schema.ListPartitionRouter, config: Mapping[str, Any], **kwargs: Any) -> airbyte_cdk.sources.declarative.partition_routers.ListPartitionRouter:
3334 def create_list_partition_router( 3335 self, model: ListPartitionRouterModel, config: Config, **kwargs: Any 3336 ) -> ListPartitionRouter: 3337 request_option = ( 3338 self._create_component_from_model(model.request_option, config) 3339 if model.request_option 3340 else None 3341 ) 3342 return ListPartitionRouter( 3343 cursor_field=model.cursor_field, 3344 request_option=request_option, 3345 values=model.values, 3346 config=config, 3347 parameters=model.parameters or {}, 3348 )
@staticmethod
def
create_min_max_datetime( model: airbyte_cdk.sources.declarative.models.declarative_component_schema.MinMaxDatetime, config: Mapping[str, Any], **kwargs: Any) -> airbyte_cdk.MinMaxDatetime:
3350 @staticmethod 3351 def create_min_max_datetime( 3352 model: MinMaxDatetimeModel, config: Config, **kwargs: Any 3353 ) -> MinMaxDatetime: 3354 return MinMaxDatetime( 3355 datetime=model.datetime, 3356 datetime_format=model.datetime_format or "", 3357 max_datetime=model.max_datetime or "", 3358 min_datetime=model.min_datetime or "", 3359 parameters=model.parameters or {}, 3360 )
@staticmethod
def
create_no_auth( model: airbyte_cdk.sources.declarative.models.declarative_component_schema.NoAuth, config: Mapping[str, Any], **kwargs: Any) -> airbyte_cdk.NoAuth:
@staticmethod
def
create_no_pagination( model: airbyte_cdk.sources.declarative.models.declarative_component_schema.NoPagination, config: Mapping[str, Any], **kwargs: Any) -> airbyte_cdk.sources.declarative.requesters.paginators.NoPagination:
def
create_oauth_authenticator( self, model: airbyte_cdk.sources.declarative.models.declarative_component_schema.OAuthAuthenticator, config: Mapping[str, Any], **kwargs: Any) -> airbyte_cdk.DeclarativeOauth2Authenticator:
3372 def create_oauth_authenticator( 3373 self, model: OAuthAuthenticatorModel, config: Config, **kwargs: Any 3374 ) -> DeclarativeOauth2Authenticator: 3375 profile_assertion = ( 3376 self._create_component_from_model(model.profile_assertion, config=config) 3377 if model.profile_assertion 3378 else None 3379 ) 3380 3381 refresh_token_error_status_codes, refresh_token_error_key, refresh_token_error_values = ( 3382 self._get_refresh_token_error_information(model) 3383 ) 3384 if model.refresh_token_updater: 3385 # ignore type error because fixing it would have a lot of dependencies, revisit later 3386 return DeclarativeSingleUseRefreshTokenOauth2Authenticator( # type: ignore 3387 config, 3388 InterpolatedString.create( 3389 model.token_refresh_endpoint, # type: ignore 3390 parameters=model.parameters or {}, 3391 ).eval(config), 3392 access_token_name=InterpolatedString.create( 3393 model.access_token_name or "access_token", parameters=model.parameters or {} 3394 ).eval(config), 3395 refresh_token_name=model.refresh_token_updater.refresh_token_name, 3396 expires_in_name=InterpolatedString.create( 3397 model.expires_in_name or "expires_in", parameters=model.parameters or {} 3398 ).eval(config), 3399 client_id_name=InterpolatedString.create( 3400 model.client_id_name or "client_id", parameters=model.parameters or {} 3401 ).eval(config), 3402 client_id=InterpolatedString.create( 3403 model.client_id, parameters=model.parameters or {} 3404 ).eval(config) 3405 if model.client_id 3406 else model.client_id, 3407 client_secret_name=InterpolatedString.create( 3408 model.client_secret_name or "client_secret", parameters=model.parameters or {} 3409 ).eval(config), 3410 client_secret=InterpolatedString.create( 3411 model.client_secret, parameters=model.parameters or {} 3412 ).eval(config) 3413 if model.client_secret 3414 else model.client_secret, 3415 access_token_config_path=model.refresh_token_updater.access_token_config_path, 3416 refresh_token_config_path=model.refresh_token_updater.refresh_token_config_path, 3417 token_expiry_date_config_path=model.refresh_token_updater.token_expiry_date_config_path, 3418 grant_type_name=InterpolatedString.create( 3419 model.grant_type_name or "grant_type", parameters=model.parameters or {} 3420 ).eval(config), 3421 grant_type=InterpolatedString.create( 3422 model.grant_type or "refresh_token", parameters=model.parameters or {} 3423 ).eval(config), 3424 refresh_request_body=InterpolatedMapping( 3425 model.refresh_request_body or {}, parameters=model.parameters or {} 3426 ).eval(config), 3427 refresh_request_headers=InterpolatedMapping( 3428 model.refresh_request_headers or {}, parameters=model.parameters or {} 3429 ).eval(config), 3430 send_refresh_request_as_query_params=bool( 3431 model.send_refresh_request_as_query_params 3432 ), 3433 scopes=model.scopes, 3434 token_expiry_date_format=model.token_expiry_date_format, 3435 token_expiry_is_time_of_expiration=bool(model.token_expiry_date_format), 3436 message_repository=self._message_repository, 3437 emit_control_message_to_message_repository=self._emit_connector_builder_messages, 3438 refresh_token_error_status_codes=refresh_token_error_status_codes, 3439 refresh_token_error_key=refresh_token_error_key, 3440 refresh_token_error_values=refresh_token_error_values, 3441 refresh_token_error_messages=self._get_refresh_token_error_messages(model), 3442 ) 3443 # ignore type error because fixing it would have a lot of dependencies, revisit later 3444 return DeclarativeOauth2Authenticator( # type: ignore 3445 access_token_name=model.access_token_name or "access_token", 3446 access_token_value=model.access_token_value, 3447 client_id_name=model.client_id_name or "client_id", 3448 client_id=model.client_id, 3449 client_secret_name=model.client_secret_name or "client_secret", 3450 client_secret=model.client_secret, 3451 expires_in_name=model.expires_in_name or "expires_in", 3452 grant_type_name=model.grant_type_name or "grant_type", 3453 grant_type=model.grant_type or "refresh_token", 3454 refresh_request_body=model.refresh_request_body, 3455 refresh_request_headers=model.refresh_request_headers, 3456 send_refresh_request_as_query_params=bool(model.send_refresh_request_as_query_params), 3457 refresh_token_name=model.refresh_token_name or "refresh_token", 3458 refresh_token=model.refresh_token, 3459 scopes=model.scopes, 3460 token_expiry_date=model.token_expiry_date, 3461 token_expiry_date_format=model.token_expiry_date_format, 3462 token_expiry_is_time_of_expiration=bool(model.token_expiry_date_format), 3463 token_refresh_endpoint=model.token_refresh_endpoint, 3464 config=config, 3465 parameters=model.parameters or {}, 3466 message_repository=self._message_repository, 3467 profile_assertion=profile_assertion, 3468 use_profile_assertion=model.use_profile_assertion, 3469 refresh_token_error_status_codes=refresh_token_error_status_codes, 3470 refresh_token_error_key=refresh_token_error_key, 3471 refresh_token_error_values=refresh_token_error_values, 3472 refresh_token_error_messages=self._get_refresh_token_error_messages(model), 3473 )
def
create_offset_increment( self, model: airbyte_cdk.sources.declarative.models.declarative_component_schema.OffsetIncrement, config: Mapping[str, Any], decoder: Optional[airbyte_cdk.Decoder] = None, extractor_model: Union[airbyte_cdk.sources.declarative.models.declarative_component_schema.CustomRecordExtractor, airbyte_cdk.sources.declarative.models.declarative_component_schema.DpathExtractor, airbyte_cdk.sources.declarative.models.declarative_component_schema.CombinedExtractor, NoneType] = None, **kwargs: Any) -> airbyte_cdk.OffsetIncrement:
3587 def create_offset_increment( 3588 self, 3589 model: OffsetIncrementModel, 3590 config: Config, 3591 decoder: Optional[Decoder] = None, 3592 extractor_model: Optional[ 3593 Union[CustomRecordExtractorModel, DpathExtractorModel, CombinedExtractorModel] 3594 ] = None, 3595 **kwargs: Any, 3596 ) -> OffsetIncrement: 3597 # None when nested under a custom component, which cannot pass a decoder 3598 if decoder is None: 3599 decoder = JsonDecoder(parameters={}) 3600 if isinstance(decoder, PaginationDecoderDecorator): 3601 inner_decoder = decoder.decoder 3602 else: 3603 inner_decoder = decoder 3604 decoder = PaginationDecoderDecorator(decoder=decoder) 3605 3606 if self._is_supported_decoder_for_pagination(inner_decoder): 3607 decoder_to_use = decoder 3608 else: 3609 raise ValueError( 3610 self._UNSUPPORTED_DECODER_ERROR.format(decoder_type=type(inner_decoder)) 3611 ) 3612 3613 self._reject_union_combined_extractor_for_offset_increment(extractor_model) 3614 3615 # Pydantic v1 Union type coercion can convert int to string depending on Union order. 3616 # If page_size is a string that represents an integer (not an interpolation), convert it back. 3617 page_size = model.page_size 3618 if isinstance(page_size, str) and page_size.isdigit(): 3619 page_size = int(page_size) 3620 3621 return OffsetIncrement( 3622 page_size=page_size, 3623 config=config, 3624 decoder=decoder_to_use, 3625 inject_on_first_request=model.inject_on_first_request or False, 3626 parameters=model.parameters or {}, 3627 )
@staticmethod
def
create_page_increment( model: airbyte_cdk.sources.declarative.models.declarative_component_schema.PageIncrement, config: Mapping[str, Any], **kwargs: Any) -> airbyte_cdk.PageIncrement:
3629 @staticmethod 3630 def create_page_increment( 3631 model: PageIncrementModel, config: Config, **kwargs: Any 3632 ) -> PageIncrement: 3633 # Pydantic v1 Union type coercion can convert int to string depending on Union order. 3634 # If page_size is a string that represents an integer (not an interpolation), convert it back. 3635 page_size = model.page_size 3636 if isinstance(page_size, str) and page_size.isdigit(): 3637 page_size = int(page_size) 3638 3639 return PageIncrement( 3640 page_size=page_size, 3641 config=config, 3642 start_from_page=model.start_from_page or 0, 3643 inject_on_first_request=model.inject_on_first_request or False, 3644 parameters=model.parameters or {}, 3645 )
def
create_parent_stream_config( self, model: airbyte_cdk.sources.declarative.models.declarative_component_schema.ParentStreamConfig, config: Mapping[str, Any], *, stream_name: str, **kwargs: Any) -> airbyte_cdk.ParentStreamConfig:
3647 def create_parent_stream_config( 3648 self, model: ParentStreamConfigModel, config: Config, *, stream_name: str, **kwargs: Any 3649 ) -> ParentStreamConfig: 3650 declarative_stream = self._create_component_from_model( 3651 model.stream, 3652 config=config, 3653 is_parent=True, 3654 **kwargs, 3655 ) 3656 request_option = ( 3657 self._create_component_from_model(model.request_option, config=config) 3658 if model.request_option 3659 else None 3660 ) 3661 3662 if model.lazy_read_pointer and any("*" in pointer for pointer in model.lazy_read_pointer): 3663 raise ValueError( 3664 "The '*' wildcard in 'lazy_read_pointer' is not supported — only direct paths are allowed." 3665 ) 3666 3667 model_lazy_read_pointer: List[Union[InterpolatedString, str]] = ( 3668 [x for x in model.lazy_read_pointer] if model.lazy_read_pointer else [] 3669 ) 3670 3671 return ParentStreamConfig( 3672 parent_key=model.parent_key, 3673 request_option=request_option, 3674 stream=declarative_stream, 3675 partition_field=model.partition_field, 3676 config=config, 3677 incremental_dependency=model.incremental_dependency or False, 3678 parameters=model.parameters or {}, 3679 extra_fields=model.extra_fields, 3680 lazy_read_pointer=model_lazy_read_pointer, 3681 )
def
create_properties_from_endpoint( self, model: airbyte_cdk.sources.declarative.models.declarative_component_schema.PropertiesFromEndpoint, config: Mapping[str, Any], **kwargs: Any) -> airbyte_cdk.sources.declarative.requesters.query_properties.PropertiesFromEndpoint:
3683 def create_properties_from_endpoint( 3684 self, model: PropertiesFromEndpointModel, config: Config, **kwargs: Any 3685 ) -> PropertiesFromEndpoint: 3686 retriever = self._create_component_from_model( 3687 model=model.retriever, 3688 config=config, 3689 name="dynamic_properties", 3690 primary_key=None, 3691 stream_slicer=None, 3692 transformations=[], 3693 use_cache=True, # Enable caching on the HttpRequester/HttpClient because the properties endpoint will be called for every slice being processed, and it is highly unlikely for the response to different 3694 ) 3695 return PropertiesFromEndpoint( 3696 property_field_path=model.property_field_path, 3697 retriever=retriever, 3698 config=config, 3699 parameters=model.parameters or {}, 3700 )
def
create_property_chunking( self, model: airbyte_cdk.sources.declarative.models.declarative_component_schema.PropertyChunking, config: Mapping[str, Any], **kwargs: Any) -> airbyte_cdk.sources.declarative.requesters.query_properties.PropertyChunking:
3702 def create_property_chunking( 3703 self, model: PropertyChunkingModel, config: Config, **kwargs: Any 3704 ) -> PropertyChunking: 3705 record_merge_strategy = ( 3706 self._create_component_from_model( 3707 model=model.record_merge_strategy, config=config, **kwargs 3708 ) 3709 if model.record_merge_strategy 3710 else None 3711 ) 3712 3713 property_limit_type: PropertyLimitType 3714 match model.property_limit_type: 3715 case PropertyLimitTypeModel.property_count: 3716 property_limit_type = PropertyLimitType.property_count 3717 case PropertyLimitTypeModel.characters: 3718 property_limit_type = PropertyLimitType.characters 3719 case _: 3720 raise ValueError(f"Invalid PropertyLimitType {property_limit_type}") 3721 3722 return PropertyChunking( 3723 property_limit_type=property_limit_type, 3724 property_limit=model.property_limit, 3725 record_merge_strategy=record_merge_strategy, 3726 config=config, 3727 parameters=model.parameters or {}, 3728 )
def
create_query_properties( self, model: airbyte_cdk.sources.declarative.models.declarative_component_schema.QueryProperties, config: Mapping[str, Any], *, stream_name: str, **kwargs: Any) -> airbyte_cdk.sources.declarative.requesters.query_properties.QueryProperties:
3730 def create_query_properties( 3731 self, model: QueryPropertiesModel, config: Config, *, stream_name: str, **kwargs: Any 3732 ) -> QueryProperties: 3733 if isinstance(model.property_list, list): 3734 property_list = model.property_list 3735 else: 3736 property_list = self._create_component_from_model( 3737 model=model.property_list, config=config, **kwargs 3738 ) 3739 3740 property_chunking = ( 3741 self._create_component_from_model( 3742 model=model.property_chunking, config=config, **kwargs 3743 ) 3744 if model.property_chunking 3745 else None 3746 ) 3747 3748 property_selector = ( 3749 self._create_component_from_model( 3750 model=model.property_selector, config=config, stream_name=stream_name, **kwargs 3751 ) 3752 if model.property_selector 3753 else None 3754 ) 3755 3756 return QueryProperties( 3757 property_list=property_list, 3758 always_include_properties=model.always_include_properties, 3759 property_chunking=property_chunking, 3760 property_selector=property_selector, 3761 config=config, 3762 parameters=model.parameters or {}, 3763 )
def
create_json_schema_property_selector( self, model: airbyte_cdk.sources.declarative.models.declarative_component_schema.JsonSchemaPropertySelector, config: Mapping[str, Any], *, stream_name: str, **kwargs: Any) -> airbyte_cdk.sources.declarative.requesters.query_properties.property_selector.JsonSchemaPropertySelector:
3765 def create_json_schema_property_selector( 3766 self, 3767 model: JsonSchemaPropertySelectorModel, 3768 config: Config, 3769 *, 3770 stream_name: str, 3771 **kwargs: Any, 3772 ) -> JsonSchemaPropertySelector: 3773 configured_stream = self._stream_name_to_configured_stream.get(stream_name) 3774 3775 transformations = [] 3776 if model.transformations: 3777 for transformation_model in model.transformations: 3778 transformations.append( 3779 self._create_component_from_model(model=transformation_model, config=config) 3780 ) 3781 3782 return JsonSchemaPropertySelector( 3783 configured_stream=configured_stream, 3784 properties_transformations=transformations, 3785 config=config, 3786 parameters=model.parameters or {}, 3787 )
@staticmethod
def
create_record_filter( model: airbyte_cdk.sources.declarative.models.declarative_component_schema.RecordFilter, config: Mapping[str, Any], **kwargs: Any) -> airbyte_cdk.RecordFilter:
@staticmethod
def
create_request_path( model: airbyte_cdk.sources.declarative.models.declarative_component_schema.RequestPath, config: Mapping[str, Any], **kwargs: Any) -> airbyte_cdk.sources.declarative.requesters.request_path.RequestPath:
@staticmethod
def
create_request_option( model: airbyte_cdk.sources.declarative.models.declarative_component_schema.RequestOption, config: Mapping[str, Any], **kwargs: Any) -> airbyte_cdk.RequestOption:
3801 @staticmethod 3802 def create_request_option( 3803 model: RequestOptionModel, config: Config, **kwargs: Any 3804 ) -> RequestOption: 3805 inject_into = RequestOptionType(model.inject_into.value) 3806 field_path: Optional[List[Union[InterpolatedString, str]]] = ( 3807 [ 3808 InterpolatedString.create(segment, parameters=kwargs.get("parameters", {})) 3809 for segment in model.field_path 3810 ] 3811 if model.field_path 3812 else None 3813 ) 3814 field_name = ( 3815 InterpolatedString.create(model.field_name, parameters=kwargs.get("parameters", {})) 3816 if model.field_name 3817 else None 3818 ) 3819 return RequestOption( 3820 field_name=field_name, 3821 field_path=field_path, 3822 inject_into=inject_into, 3823 parameters=kwargs.get("parameters", {}), 3824 )
def
create_record_selector( self, model: airbyte_cdk.sources.declarative.models.declarative_component_schema.RecordSelector, config: Mapping[str, Any], *, name: str, transformations: Optional[List[airbyte_cdk.RecordTransformation]] = None, decoder: airbyte_cdk.Decoder | None = None, client_side_incremental_sync_cursor: Optional[airbyte_cdk.Cursor] = None, is_client_side_incremental_sync: bool = False, file_uploader: Optional[airbyte_cdk.sources.declarative.retrievers.file_uploader.DefaultFileUploader] = None, **kwargs: Any) -> airbyte_cdk.RecordSelector:
3826 def create_record_selector( 3827 self, 3828 model: RecordSelectorModel, 3829 config: Config, 3830 *, 3831 name: str, 3832 transformations: List[RecordTransformation] | None = None, 3833 decoder: Decoder | None = None, 3834 client_side_incremental_sync_cursor: Optional[Cursor] = None, 3835 is_client_side_incremental_sync: bool = False, 3836 file_uploader: Optional[DefaultFileUploader] = None, 3837 **kwargs: Any, 3838 ) -> RecordSelector: 3839 extractor = self._create_component_from_model( 3840 model=model.extractor, decoder=decoder, config=config 3841 ) 3842 self._count_dropped_empty_records(extractor) 3843 record_filter = ( 3844 self._create_component_from_model(model.record_filter, config=config) 3845 if model.record_filter 3846 else None 3847 ) 3848 3849 # A client-side incremental stream transforms before filtering by default. That default belongs to the flag, 3850 # not to the component that ends up doing the cursor comparison: a data feed does it in the retriever and 3851 # receives no cursor here, but its `record_filter` condition must keep running after the transformations. 3852 default_transform_before_filtering = bool( 3853 client_side_incremental_sync_cursor or is_client_side_incremental_sync 3854 ) 3855 transform_before_filtering = ( 3856 default_transform_before_filtering 3857 if model.transform_before_filtering is None 3858 else model.transform_before_filtering 3859 ) 3860 if client_side_incremental_sync_cursor: 3861 record_filter = ClientSideIncrementalRecordFilterDecorator( 3862 config=config, 3863 parameters=model.parameters, 3864 condition=model.record_filter.condition 3865 if (model.record_filter and hasattr(model.record_filter, "condition")) 3866 else None, 3867 cursor=client_side_incremental_sync_cursor, 3868 ) 3869 3870 if model.schema_normalization is None: 3871 # default to no schema normalization if not set 3872 model.schema_normalization = SchemaNormalizationModel.None_ 3873 3874 schema_normalization = ( 3875 TypeTransformer(SCHEMA_TRANSFORMER_TYPE_MAPPING[model.schema_normalization]) 3876 if isinstance(model.schema_normalization, SchemaNormalizationModel) 3877 else self._create_component_from_model(model.schema_normalization, config=config) # type: ignore[arg-type] # custom normalization model expected here 3878 ) 3879 3880 return RecordSelector( 3881 extractor=extractor, 3882 name=name, 3883 config=config, 3884 record_filter=record_filter, 3885 transformations=transformations or [], 3886 file_uploader=file_uploader, 3887 schema_normalization=schema_normalization, 3888 parameters=model.parameters or {}, 3889 transform_before_filtering=transform_before_filtering, 3890 )
@staticmethod
def
create_remove_fields( model: airbyte_cdk.sources.declarative.models.declarative_component_schema.RemoveFields, config: Mapping[str, Any], **kwargs: Any) -> airbyte_cdk.sources.declarative.transformations.RemoveFields:
def
create_selective_authenticator( self, model: airbyte_cdk.sources.declarative.models.declarative_component_schema.SelectiveAuthenticator, config: Mapping[str, Any], *, name: Optional[str] = None, url_base: Optional[str] = None, **kwargs: Any) -> airbyte_cdk.DeclarativeAuthenticator:
3900 def create_selective_authenticator( 3901 self, 3902 model: SelectiveAuthenticatorModel, 3903 config: Config, 3904 *, 3905 name: Optional[str] = None, 3906 url_base: Optional[str] = None, 3907 **kwargs: Any, 3908 ) -> DeclarativeAuthenticator: 3909 # Keyword-only so that _create_nested_component also fills them under a custom component, 3910 # from the parent's kwargs or $parameters. None is not forwarded, so a missing required one 3911 # keeps its hint. 3912 nested_kwargs = { 3913 key: value 3914 for key, value in {"name": name, "url_base": url_base}.items() 3915 if value is not None 3916 } 3917 authenticators = { 3918 key: self._create_component_from_model( 3919 model=auth, config=config, **nested_kwargs, **kwargs 3920 ) 3921 for key, auth in model.authenticators.items() 3922 } 3923 # SelectiveAuthenticator will return instance of DeclarativeAuthenticator or raise ValueError error 3924 return SelectiveAuthenticator( # type: ignore[abstract] 3925 config=config, 3926 authenticators=authenticators, 3927 authenticator_selection_path=model.authenticator_selection_path, 3928 **kwargs, 3929 )
@staticmethod
def
create_legacy_session_token_authenticator( model: airbyte_cdk.sources.declarative.models.declarative_component_schema.LegacySessionTokenAuthenticator, config: Mapping[str, Any], *, url_base: str, **kwargs: Any) -> airbyte_cdk.sources.declarative.auth.token.LegacySessionTokenAuthenticator:
3931 @staticmethod 3932 def create_legacy_session_token_authenticator( 3933 model: LegacySessionTokenAuthenticatorModel, config: Config, *, url_base: str, **kwargs: Any 3934 ) -> LegacySessionTokenAuthenticator: 3935 return LegacySessionTokenAuthenticator( 3936 api_url=url_base, 3937 header=model.header, 3938 login_url=model.login_url, 3939 password=model.password or "", 3940 session_token=model.session_token or "", 3941 session_token_response_key=model.session_token_response_key or "", 3942 username=model.username or "", 3943 validate_session_url=model.validate_session_url, 3944 config=config, 3945 parameters=model.parameters or {}, 3946 )
def
create_simple_retriever( self, model: airbyte_cdk.sources.declarative.models.declarative_component_schema.SimpleRetriever, config: Mapping[str, Any], *, name: str, primary_key: Union[str, List[str], List[List[str]], NoneType], request_options_provider: Optional[airbyte_cdk.sources.declarative.requesters.request_options.RequestOptionsProvider] = None, cursor: Optional[airbyte_cdk.Cursor] = None, has_stop_condition_cursor: bool = False, is_client_side_incremental_sync: bool = False, transformations: List[airbyte_cdk.RecordTransformation], file_uploader: Optional[airbyte_cdk.sources.declarative.retrievers.file_uploader.DefaultFileUploader] = None, incremental_sync: Union[airbyte_cdk.sources.declarative.models.declarative_component_schema.IncrementingCountCursor, airbyte_cdk.sources.declarative.models.declarative_component_schema.DatetimeBasedCursor, airbyte_cdk.sources.declarative.models.declarative_component_schema.SyncTokenCursor, NoneType] = None, use_cache: Optional[bool] = None, log_formatter: Optional[Callable[[requests.models.Response], Any]] = None, partition_router: Optional[airbyte_cdk.sources.declarative.partition_routers.PartitionRouter] = None, **kwargs: Any) -> airbyte_cdk.SimpleRetriever:
3948 def create_simple_retriever( 3949 self, 3950 model: SimpleRetrieverModel, 3951 config: Config, 3952 *, 3953 name: str, 3954 primary_key: Optional[Union[str, List[str], List[List[str]]]], 3955 request_options_provider: Optional[RequestOptionsProvider] = None, 3956 cursor: Optional[Cursor] = None, 3957 has_stop_condition_cursor: bool = False, 3958 is_client_side_incremental_sync: bool = False, 3959 transformations: List[RecordTransformation], 3960 file_uploader: Optional[DefaultFileUploader] = None, 3961 incremental_sync: Optional[ 3962 Union[ 3963 IncrementingCountCursorModel, 3964 DatetimeBasedCursorModel, 3965 SyncTokenCursorModel, 3966 ] 3967 ] = None, 3968 use_cache: Optional[bool] = None, 3969 log_formatter: Optional[Callable[[Response], Any]] = None, 3970 partition_router: Optional[PartitionRouter] = None, 3971 **kwargs: Any, 3972 ) -> SimpleRetriever: 3973 def _get_url(req: Requester) -> str: 3974 """ 3975 Closure to get the URL from the requester. This is used to get the URL in the case of a lazy retriever. 3976 This is needed because the URL is not set until the requester is created. 3977 """ 3978 3979 _url: str = ( 3980 model.requester.url 3981 if hasattr(model.requester, "url") and model.requester.url is not None 3982 else req.get_url(stream_state=None, stream_slice=None, next_page_token=None) 3983 ) 3984 _url_base: str = ( 3985 model.requester.url_base 3986 if hasattr(model.requester, "url_base") and model.requester.url_base is not None 3987 else req.get_url_base(stream_state=None, stream_slice=None, next_page_token=None) 3988 ) 3989 3990 return _url or _url_base 3991 3992 if cursor is None: 3993 cursor = FinalStateCursor(name, None, self._message_repository) 3994 3995 # A data feed drops the records the cursor considers already synced in the retriever, which sits downstream of 3996 # the paginator. Letting the record selector drop them as well would be redundant and would hide them from the 3997 # pagination stop condition, so a data feed never delegates that filtering to the record selector, whether 3998 # `is_client_side_incremental` is set or not. The `condition` from `record_filter` is intentionally left out of 3999 # the post-pagination filter and stays in the record selector, which preserves the existing behaviour: the 4000 # selector runs inside the page loop, so the records the condition rejects never reach the paginator's 4001 # accounting. Moving it downstream would start counting them. 4002 post_pagination_filter = ( 4003 ClientSideIncrementalRecordFilterDecorator( 4004 config=config, 4005 parameters=model.parameters or {}, 4006 condition=None, 4007 cursor=cursor, 4008 ) 4009 if has_stop_condition_cursor 4010 else None 4011 ) 4012 client_side_incremental_cursor = ( 4013 cursor if is_client_side_incremental_sync and not post_pagination_filter else None 4014 ) 4015 if post_pagination_filter and is_client_side_incremental_sync: 4016 LOGGER.warning( 4017 f"Stream {name}: `is_client_side_incremental` adds no record filtering when `is_data_feed` is set, " 4018 "as a data feed already filters on the cursor value. It still makes the record selector apply the " 4019 "transformations before the `record_filter` condition." 4020 ) 4021 4022 decoder = ( 4023 cursor.decoder 4024 if isinstance(cursor, SyncTokenCursor) 4025 else ( 4026 self._create_component_from_model(model=model.decoder, config=config) 4027 if model.decoder 4028 else JsonDecoder(parameters={}) 4029 ) 4030 ) 4031 record_selector = self._create_component_from_model( 4032 model=model.record_selector, 4033 name=name, 4034 config=config, 4035 decoder=decoder, 4036 transformations=transformations, 4037 client_side_incremental_sync_cursor=client_side_incremental_cursor, 4038 is_client_side_incremental_sync=is_client_side_incremental_sync, 4039 file_uploader=file_uploader, 4040 ) 4041 4042 query_properties: Optional[QueryProperties] = None 4043 query_properties_key: Optional[str] = None 4044 self._ensure_query_properties_to_model(model.requester) 4045 if self._has_query_properties_in_request_parameters(model.requester): 4046 # It is better to be explicit about an error if PropertiesFromEndpoint is defined in multiple 4047 # places instead of default to request_parameters which isn't clearly documented 4048 if ( 4049 hasattr(model.requester, "fetch_properties_from_endpoint") 4050 and model.requester.fetch_properties_from_endpoint 4051 ): 4052 raise ValueError( 4053 f"PropertiesFromEndpoint should only be specified once per stream, but found in {model.requester.type}.fetch_properties_from_endpoint and {model.requester.type}.request_parameters" 4054 ) 4055 4056 query_properties_definitions = [] 4057 for key, request_parameter in model.requester.request_parameters.items(): # type: ignore # request_parameters is already validated to be a Mapping using _has_query_properties_in_request_parameters() 4058 if isinstance(request_parameter, QueryPropertiesModel): 4059 query_properties_key = key 4060 query_properties_definitions.append(request_parameter) 4061 4062 if len(query_properties_definitions) > 1: 4063 raise ValueError( 4064 f"request_parameters only supports defining one QueryProperties field, but found {len(query_properties_definitions)} usages" 4065 ) 4066 4067 if len(query_properties_definitions) == 1: 4068 query_properties = self._create_component_from_model( 4069 model=query_properties_definitions[0], stream_name=name, config=config 4070 ) 4071 4072 # Removes QueryProperties components from the interpolated mappings because it has been designed 4073 # to be used by the SimpleRetriever and will be resolved from the provider from the slice directly 4074 # instead of through jinja interpolation 4075 if hasattr(model.requester, "request_parameters") and isinstance( 4076 model.requester.request_parameters, Mapping 4077 ): 4078 model.requester.request_parameters = self._remove_query_properties( 4079 model.requester.request_parameters 4080 ) 4081 elif ( 4082 hasattr(model.requester, "fetch_properties_from_endpoint") 4083 and model.requester.fetch_properties_from_endpoint 4084 ): 4085 # todo: Deprecate this condition once dependent connectors migrate to query_properties 4086 query_properties_definition = QueryPropertiesModel( 4087 type="QueryProperties", 4088 property_list=model.requester.fetch_properties_from_endpoint, 4089 always_include_properties=None, 4090 property_chunking=None, 4091 ) # type: ignore # $parameters has a default value 4092 4093 query_properties = self.create_query_properties( 4094 model=query_properties_definition, 4095 stream_name=name, 4096 config=config, 4097 ) 4098 elif hasattr(model.requester, "query_properties") and model.requester.query_properties: 4099 query_properties = self.create_query_properties( 4100 model=model.requester.query_properties, 4101 stream_name=name, 4102 config=config, 4103 ) 4104 4105 requester = self._create_component_from_model( 4106 model=model.requester, 4107 decoder=decoder, 4108 name=name, 4109 query_properties_key=query_properties_key, 4110 use_cache=use_cache, 4111 config=config, 4112 ) 4113 4114 if not request_options_provider: 4115 request_options_provider = DefaultRequestOptionsProvider(parameters={}) 4116 if isinstance(request_options_provider, DefaultRequestOptionsProvider) and isinstance( 4117 partition_router, PartitionRouter 4118 ): 4119 request_options_provider = partition_router 4120 4121 paginator = ( 4122 self._create_component_from_model( 4123 model=model.paginator, 4124 config=config, 4125 url_base=_get_url(requester), 4126 extractor_model=model.record_selector.extractor, 4127 decoder=decoder, 4128 cursor_used_for_stop_condition=cursor if has_stop_condition_cursor else None, 4129 ) 4130 if model.paginator 4131 else NoPagination(parameters={}) 4132 ) 4133 4134 ignore_stream_slicer_parameters_on_paginated_requests = ( 4135 model.ignore_stream_slicer_parameters_on_paginated_requests or False 4136 ) 4137 4138 reads_parent_stream_lazily = bool( 4139 model.partition_router 4140 and isinstance(model.partition_router, SubstreamPartitionRouterModel) 4141 and any( 4142 parent_stream_config.lazy_read_pointer 4143 for parent_stream_config in model.partition_router.parent_stream_configs 4144 ) 4145 ) 4146 if reads_parent_stream_lazily and ( 4147 model.page_size_reduction 4148 or self._uses_reduce_page_size_action(getattr(model.requester, "error_handler", None)) 4149 ): 4150 # Checked outside of the LazySimpleRetriever branch below, which only applies on the first 4151 # sync of a stream: gating it on the absence of state would accept the same manifest from the 4152 # second sync onwards. LazySimpleRetriever paginates the parent's embedded pages, so there is 4153 # no page of its own to re-issue with a smaller page size. 4154 raise ValueError( 4155 f"`page_size_reduction` and the REDUCE_PAGE_SIZE response action are not supported when " 4156 f"reading a parent stream lazily. Remove either the page size reduction or the parent " 4157 f"stream's `lazy_read_pointer` for stream {name}." 4158 ) 4159 4160 if reads_parent_stream_lazily and ( 4161 model.request_window_splitting 4162 or self._uses_split_request_window_action( 4163 getattr(model.requester, "error_handler", None) 4164 ) 4165 ): 4166 # Same reasoning as the page_size_reduction check above: LazySimpleRetriever reads records 4167 # embedded in the parent's own pages rather than requesting a window of its own, so there is 4168 # nothing here for a split child window to be read from. 4169 raise ValueError( 4170 f"`request_window_splitting` and the SPLIT_REQUEST_WINDOW response action are not " 4171 f"supported when reading a parent stream lazily. Remove either the request window " 4172 f"splitting or the parent stream's `lazy_read_pointer` for stream {name}." 4173 ) 4174 4175 if reads_parent_stream_lazily and not bool( 4176 self._connector_state_manager.get_stream_state(name, None) 4177 ): 4178 if incremental_sync: 4179 if incremental_sync.type != "DatetimeBasedCursor": 4180 raise ValueError( 4181 f"LazySimpleRetriever only supports DatetimeBasedCursor. Found: {incremental_sync.type}." 4182 ) 4183 4184 elif incremental_sync.step or incremental_sync.cursor_granularity: 4185 raise ValueError( 4186 f"Found more that one slice per parent. LazySimpleRetriever only supports single slice read for stream - {name}." 4187 ) 4188 4189 if model.decoder and model.decoder.type != "JsonDecoder": 4190 raise ValueError( 4191 f"LazySimpleRetriever only supports JsonDecoder. Found: {model.decoder.type}." 4192 ) 4193 4194 return LazySimpleRetriever( 4195 name=name, 4196 paginator=paginator, 4197 primary_key=primary_key, 4198 requester=requester, 4199 record_selector=record_selector, 4200 stream_slicer=_NO_STREAM_SLICING, 4201 request_option_provider=request_options_provider, 4202 config=config, 4203 ignore_stream_slicer_parameters_on_paginated_requests=ignore_stream_slicer_parameters_on_paginated_requests, 4204 post_pagination_filter=post_pagination_filter, 4205 parameters=model.parameters or {}, 4206 ) 4207 4208 if ( 4209 model.record_selector.record_filter 4210 and model.pagination_reset 4211 and model.pagination_reset.limits 4212 ): 4213 raise ValueError("PaginationResetLimits are not supported while having record filter.") 4214 4215 request_window_splitting = self._create_request_window_splitting( 4216 model, name, cursor, incremental_sync, query_properties, file_uploader 4217 ) 4218 return SimpleRetriever( 4219 name=name, 4220 paginator=paginator, 4221 primary_key=primary_key, 4222 requester=requester, 4223 record_selector=record_selector, 4224 stream_slicer=_NO_STREAM_SLICING, 4225 request_option_provider=request_options_provider, 4226 config=config, 4227 ignore_stream_slicer_parameters_on_paginated_requests=ignore_stream_slicer_parameters_on_paginated_requests, 4228 additional_query_properties=query_properties, 4229 log_formatter=self._get_log_formatter(log_formatter, name), 4230 pagination_tracker_factory=self._create_pagination_tracker_factory( 4231 model.pagination_reset, cursor 4232 ), 4233 page_size_reduction=self._create_page_size_reduction( 4234 model, name, query_properties, file_uploader 4235 ), 4236 request_window_splitting=request_window_splitting, 4237 request_window_splitter=( 4238 cursor.split_request_window 4239 if request_window_splitting and hasattr(cursor, "split_request_window") 4240 else None 4241 ), 4242 post_pagination_filter=post_pagination_filter, 4243 page_observer=( 4244 cursor.on_page_completed if isinstance(cursor, SyncTokenCursor) else None 4245 ), 4246 parameters=model.parameters or {}, 4247 )
def
create_state_delegating_stream( self, model: airbyte_cdk.sources.declarative.models.declarative_component_schema.StateDelegatingStream, config: Mapping[str, Any], **kwargs: Any) -> airbyte_cdk.sources.streams.concurrent.default_stream.DefaultStream:
4828 def create_state_delegating_stream( 4829 self, 4830 model: StateDelegatingStreamModel, 4831 config: Config, 4832 **kwargs: Any, 4833 ) -> DefaultStream: 4834 if ( 4835 model.full_refresh_stream.name != model.name 4836 or model.name != model.incremental_stream.name 4837 ): 4838 raise ValueError( 4839 f"state_delegating_stream, full_refresh_stream name and incremental_stream must have equal names. Instead has {model.name}, {model.full_refresh_stream.name} and {model.incremental_stream.name}." 4840 ) 4841 4842 # Resolve api_retention_period with config context (supports Jinja2 interpolation) 4843 resolved_retention_period: Optional[str] = None 4844 if model.api_retention_period: 4845 interpolated_retention = InterpolatedString.create( 4846 model.api_retention_period, parameters=model.parameters or {} 4847 ) 4848 resolved_value = interpolated_retention.eval(config=config) 4849 if resolved_value: 4850 resolved_retention_period = str(resolved_value) 4851 4852 if resolved_retention_period: 4853 for stream_model in (model.full_refresh_stream, model.incremental_stream): 4854 if isinstance(stream_model.incremental_sync, IncrementingCountCursorModel): 4855 raise ValueError( 4856 f"Stream '{model.name}' uses IncrementingCountCursor which is not supported " 4857 f"with api_retention_period. IncrementingCountCursor does not use datetime-based " 4858 f"cursors, so cursor age validation cannot be performed." 4859 ) 4860 if isinstance(stream_model.incremental_sync, SyncTokenCursorModel): 4861 raise ValueError( 4862 f"Stream '{model.name}' uses SyncTokenCursor which is not supported " 4863 f"with api_retention_period. A sync token has no datetime cursor, so " 4864 f"cursor age validation cannot be performed." 4865 ) 4866 4867 stream_state = self._connector_state_manager.get_stream_state(model.name, None) 4868 4869 if not stream_state: 4870 return self._create_component_from_model( # type: ignore[no-any-return] 4871 model.full_refresh_stream, config=config, **kwargs 4872 ) 4873 4874 incremental_stream: DefaultStream = self._create_component_from_model( 4875 model.incremental_stream, config=config, **kwargs 4876 ) # type: ignore[assignment] 4877 4878 # Only run cursor age validation for streams that are in the configured 4879 # catalog (or when no catalog was provided, e.g. during discover / connector 4880 # builder). Streams not selected by the user but instantiated as parent-stream 4881 # dependencies must not go through this path because it emits state messages 4882 # that the destination does not know about, causing "Stream not found" crashes. 4883 stream_is_in_catalog = ( 4884 not self._stream_name_to_configured_stream # no catalog → validate by default 4885 or model.name in self._stream_name_to_configured_stream 4886 ) 4887 if resolved_retention_period and stream_is_in_catalog: 4888 full_refresh_stream: DefaultStream = self._create_component_from_model( 4889 model.full_refresh_stream, config=config, **kwargs 4890 ) # type: ignore[assignment] 4891 if self._is_cursor_older_than_retention_period( 4892 stream_state, 4893 full_refresh_stream.cursor, 4894 incremental_stream.cursor, 4895 resolved_retention_period, 4896 model.name, 4897 ): 4898 # Clear state BEFORE constructing the full_refresh_stream so that 4899 # its cursor starts from start_date instead of the stale cursor. 4900 self._connector_state_manager.update_state_for_stream(model.name, None, {}) 4901 state_message = self._connector_state_manager.create_state_message(model.name, None) 4902 self._message_repository.emit_message(state_message) 4903 return self._create_component_from_model( # type: ignore[no-any-return] 4904 model.full_refresh_stream, config=config, **kwargs 4905 ) 4906 4907 return incremental_stream
def
create_async_retriever( self, model: airbyte_cdk.sources.declarative.models.declarative_component_schema.AsyncRetriever, config: Mapping[str, Any], *, name: str, primary_key: Union[str, List[str], List[List[str]], NoneType], stream_slicer: Optional[airbyte_cdk.sources.declarative.stream_slicers.StreamSlicer], client_side_incremental_sync: Optional[Dict[str, Any]] = None, transformations: List[airbyte_cdk.RecordTransformation], **kwargs: Any) -> airbyte_cdk.sources.declarative.retrievers.AsyncRetriever:
5006 def create_async_retriever( 5007 self, 5008 model: AsyncRetrieverModel, 5009 config: Config, 5010 *, 5011 name: str, 5012 primary_key: Optional[ 5013 Union[str, List[str], List[List[str]]] 5014 ], # this seems to be needed to match create_simple_retriever 5015 stream_slicer: Optional[StreamSlicer], 5016 client_side_incremental_sync: Optional[Dict[str, Any]] = None, 5017 transformations: List[RecordTransformation], 5018 **kwargs: Any, 5019 ) -> AsyncRetriever: 5020 if model.download_target_requester and not model.download_target_extractor: 5021 raise ValueError( 5022 f"`download_target_extractor` required if using a `download_target_requester`" 5023 ) 5024 5025 for requester_field in ( 5026 "creation_requester", 5027 "polling_requester", 5028 "download_requester", 5029 "download_target_requester", 5030 "abort_requester", 5031 "delete_requester", 5032 ): 5033 self._reject_reduce_page_size_action( 5034 getattr(model, requester_field, None), 5035 f"`{requester_field}` of the AsyncRetriever of stream {name}", 5036 ) 5037 self._reject_split_request_window_action( 5038 getattr(model, requester_field, None), 5039 f"`{requester_field}` of the AsyncRetriever of stream {name}", 5040 ) 5041 5042 def _get_download_retriever( 5043 requester: Requester, extractor: RecordExtractor, _decoder: Decoder 5044 ) -> SimpleRetriever: 5045 self._count_dropped_empty_records(extractor) 5046 # We create a record selector for the download retriever 5047 # with no schema normalization and no transformations, neither record filter 5048 # as all this occurs in the record_selector of the AsyncRetriever 5049 record_selector = RecordSelector( 5050 extractor=extractor, 5051 name=name, 5052 record_filter=None, 5053 transformations=[], 5054 schema_normalization=TypeTransformer(TransformConfig.NoTransform), 5055 config=config, 5056 parameters={}, 5057 ) 5058 paginator = ( 5059 self._create_component_from_model( 5060 model=model.download_paginator, 5061 decoder=_decoder, 5062 config=config, 5063 url_base="", 5064 # Only read by the `union` rejection of `OffsetIncrement`. 5065 extractor_model=( 5066 model.download_extractor 5067 if isinstance(model.download_extractor, CombinedExtractorModel) 5068 else None 5069 ), 5070 ) 5071 if model.download_paginator 5072 else NoPagination(parameters={}) 5073 ) 5074 5075 return SimpleRetriever( 5076 requester=requester, 5077 record_selector=record_selector, 5078 primary_key=None, 5079 name=name, 5080 paginator=paginator, 5081 config=config, 5082 parameters={}, 5083 log_formatter=self._get_log_formatter(None, name), 5084 ) 5085 5086 def _get_job_timeout() -> datetime.timedelta: 5087 user_defined_timeout: Optional[int] = ( 5088 int( 5089 InterpolatedString.create( 5090 str(model.polling_job_timeout), 5091 parameters={}, 5092 ).eval(config) 5093 ) 5094 if model.polling_job_timeout 5095 else None 5096 ) 5097 5098 # check for user defined timeout during the test read or 15 minutes 5099 test_read_timeout = datetime.timedelta(minutes=user_defined_timeout or 15) 5100 # default value for non-connector builder is 60 minutes. 5101 default_sync_timeout = datetime.timedelta(minutes=user_defined_timeout or 60) 5102 5103 return ( 5104 test_read_timeout if self._emit_connector_builder_messages else default_sync_timeout 5105 ) 5106 5107 decoder = ( 5108 self._create_component_from_model(model=model.decoder, config=config) 5109 if model.decoder 5110 else JsonDecoder(parameters={}) 5111 ) 5112 record_selector = self._create_component_from_model( 5113 model=model.record_selector, 5114 config=config, 5115 decoder=decoder, 5116 name=name, 5117 transformations=transformations, 5118 client_side_incremental_sync=client_side_incremental_sync, 5119 ) 5120 5121 stream_slicer = stream_slicer or SinglePartitionRouter(parameters={}) 5122 if self._should_limit_slices_fetched(): 5123 stream_slicer = cast( 5124 StreamSlicer, 5125 StreamSlicerTestReadDecorator( 5126 wrapped_slicer=stream_slicer, 5127 maximum_number_of_slices=self._limit_slices_fetched or 5, 5128 ), 5129 ) 5130 5131 creation_requester = self._create_component_from_model( 5132 model=model.creation_requester, 5133 decoder=decoder, 5134 config=config, 5135 name=f"job creation - {name}", 5136 ) 5137 polling_requester = self._create_component_from_model( 5138 model=model.polling_requester, 5139 decoder=decoder, 5140 config=config, 5141 name=f"job polling - {name}", 5142 ) 5143 job_download_components_name = f"job download - {name}" 5144 download_decoder = ( 5145 self._create_component_from_model(model=model.download_decoder, config=config) 5146 if model.download_decoder 5147 else JsonDecoder(parameters={}) 5148 ) 5149 download_extractor = ( 5150 self._create_component_from_model( 5151 model=model.download_extractor, 5152 config=config, 5153 decoder=download_decoder, 5154 parameters=model.parameters, 5155 ) 5156 if model.download_extractor 5157 else DpathExtractor( 5158 [], 5159 config=config, 5160 decoder=download_decoder, 5161 parameters=model.parameters or {}, 5162 ) 5163 ) 5164 download_requester = self._create_component_from_model( 5165 model=model.download_requester, 5166 decoder=download_decoder, 5167 config=config, 5168 name=job_download_components_name, 5169 ) 5170 download_retriever = _get_download_retriever( 5171 download_requester, download_extractor, download_decoder 5172 ) 5173 abort_requester = ( 5174 self._create_component_from_model( 5175 model=model.abort_requester, 5176 decoder=decoder, 5177 config=config, 5178 name=f"job abort - {name}", 5179 ) 5180 if model.abort_requester 5181 else None 5182 ) 5183 delete_requester = ( 5184 self._create_component_from_model( 5185 model=model.delete_requester, 5186 decoder=decoder, 5187 config=config, 5188 name=f"job delete - {name}", 5189 ) 5190 if model.delete_requester 5191 else None 5192 ) 5193 download_target_requester = ( 5194 self._create_component_from_model( 5195 model=model.download_target_requester, 5196 decoder=decoder, 5197 config=config, 5198 name=f"job extract_url - {name}", 5199 ) 5200 if model.download_target_requester 5201 else None 5202 ) 5203 status_extractor = self._create_component_from_model( 5204 model=model.status_extractor, decoder=decoder, config=config, name=name 5205 ) 5206 download_target_extractor = ( 5207 self._create_component_from_model( 5208 model=model.download_target_extractor, 5209 decoder=decoder, 5210 config=config, 5211 name=name, 5212 ) 5213 if model.download_target_extractor 5214 else None 5215 ) 5216 5217 job_repository: AsyncJobRepository = AsyncHttpJobRepository( 5218 creation_requester=creation_requester, 5219 polling_requester=polling_requester, 5220 download_retriever=download_retriever, 5221 download_target_requester=download_target_requester, 5222 abort_requester=abort_requester, 5223 delete_requester=delete_requester, 5224 status_extractor=status_extractor, 5225 status_mapping=self._create_async_job_status_mapping(model.status_mapping, config), 5226 download_target_extractor=download_target_extractor, 5227 job_timeout=_get_job_timeout(), 5228 ) 5229 5230 failed_retry_wait_time_in_seconds: Optional[int] = ( 5231 int( 5232 InterpolatedString.create( 5233 str(model.failed_retry_wait_time_in_seconds), 5234 parameters={}, 5235 ).eval(config) 5236 ) 5237 if model.failed_retry_wait_time_in_seconds 5238 else None 5239 ) 5240 5241 async_job_partition_router = AsyncJobPartitionRouter( 5242 job_orchestrator_factory=lambda stream_slices: AsyncJobOrchestrator( 5243 job_repository, 5244 stream_slices, 5245 self._job_tracker, 5246 self._message_repository, 5247 # FIXME work would need to be done here in order to detect if a stream as a parent stream that is bulk 5248 has_bulk_parent=False, 5249 # set the `job_max_retry` to 1 for the `Connector Builder`` use-case. 5250 # `None` == default retry is set to 3 attempts, under the hood. 5251 job_max_retry=1 if self._emit_connector_builder_messages else None, 5252 failed_retry_wait_time_in_seconds=failed_retry_wait_time_in_seconds, 5253 ), 5254 stream_slicer=stream_slicer, 5255 config=config, 5256 parameters=model.parameters or {}, 5257 ) 5258 5259 return AsyncRetriever( 5260 record_selector=record_selector, 5261 stream_slicer=async_job_partition_router, 5262 config=config, 5263 parameters=model.parameters or {}, 5264 )
def
create_spec( self, model: airbyte_cdk.sources.declarative.models.declarative_component_schema.Spec, config: Mapping[str, Any], **kwargs: Any) -> airbyte_cdk.sources.declarative.spec.Spec:
5266 def create_spec(self, model: SpecModel, config: Config, **kwargs: Any) -> Spec: 5267 config_migrations = [ 5268 self._create_component_from_model(migration, config) 5269 for migration in ( 5270 model.config_normalization_rules.config_migrations 5271 if ( 5272 model.config_normalization_rules 5273 and model.config_normalization_rules.config_migrations 5274 ) 5275 else [] 5276 ) 5277 ] 5278 config_transformations = [ 5279 self._create_component_from_model(transformation, config) 5280 for transformation in ( 5281 model.config_normalization_rules.transformations 5282 if ( 5283 model.config_normalization_rules 5284 and model.config_normalization_rules.transformations 5285 ) 5286 else [] 5287 ) 5288 ] 5289 config_validations = [ 5290 self._create_component_from_model(validation, config) 5291 for validation in ( 5292 model.config_normalization_rules.validations 5293 if ( 5294 model.config_normalization_rules 5295 and model.config_normalization_rules.validations 5296 ) 5297 else [] 5298 ) 5299 ] 5300 5301 return Spec( 5302 connection_specification=model.connection_specification, 5303 documentation_url=model.documentation_url, 5304 advanced_auth=model.advanced_auth, 5305 parameters={}, 5306 config_migrations=config_migrations, 5307 config_transformations=config_transformations, 5308 config_validations=config_validations, 5309 )
def
create_substream_partition_router( self, model: airbyte_cdk.sources.declarative.models.declarative_component_schema.SubstreamPartitionRouter, config: Mapping[str, Any], *, stream_name: str, **kwargs: Any) -> airbyte_cdk.SubstreamPartitionRouter:
5311 def create_substream_partition_router( 5312 self, 5313 model: SubstreamPartitionRouterModel, 5314 config: Config, 5315 *, 5316 stream_name: str, 5317 **kwargs: Any, 5318 ) -> SubstreamPartitionRouter: 5319 parent_stream_configs = [] 5320 if model.parent_stream_configs: 5321 parent_stream_configs.extend( 5322 [ 5323 self.create_parent_stream_config_with_substream_wrapper( 5324 model=parent_stream_config, config=config, stream_name=stream_name, **kwargs 5325 ) 5326 for parent_stream_config in model.parent_stream_configs 5327 ] 5328 ) 5329 5330 return SubstreamPartitionRouter( 5331 parent_stream_configs=parent_stream_configs, 5332 parameters=model.parameters or {}, 5333 config=config, 5334 )
def
create_parent_stream_config_with_substream_wrapper( self, model: airbyte_cdk.sources.declarative.models.declarative_component_schema.ParentStreamConfig, config: Mapping[str, Any], *, stream_name: str, **kwargs: Any) -> Any:
5336 def create_parent_stream_config_with_substream_wrapper( 5337 self, model: ParentStreamConfigModel, config: Config, *, stream_name: str, **kwargs: Any 5338 ) -> Any: 5339 child_state = self._connector_state_manager.get_stream_state(stream_name, None) 5340 if NO_CURSOR_STATE_KEY in child_state: 5341 # Full refresh streams checkpoint a `{NO_CURSOR_STATE_KEY: true}` sentinel. When such a 5342 # stream is later converted to incremental with an incremental_dependency parent, 5343 # `_instantiate_parent_stream_state_manager` would treat the sentinel's boolean as a legacy 5344 # cursor value and re-key it under the parent's cursor field, crashing cursor initialization. 5345 child_state = { 5346 key: value for key, value in child_state.items() if key != NO_CURSOR_STATE_KEY 5347 } 5348 5349 parent_state: Optional[Mapping[str, Any]] = ( 5350 child_state if model.incremental_dependency and child_state else None 5351 ) 5352 connector_state_manager = self._instantiate_parent_stream_state_manager( 5353 child_state, config, model, parent_state 5354 ) 5355 5356 substream_factory = ModelToComponentFactory( 5357 custom_components_trusted=self._custom_components_trusted, 5358 connector_state_manager=connector_state_manager, 5359 limit_pages_fetched_per_slice=self._limit_pages_fetched_per_slice, 5360 limit_slices_fetched=self._limit_slices_fetched, 5361 emit_connector_builder_messages=self._emit_connector_builder_messages, 5362 disable_retries=self._disable_retries, 5363 disable_cache=self._disable_cache, 5364 message_repository=StateFilteringMessageRepository( 5365 LogAppenderMessageRepositoryDecorator( 5366 { 5367 "airbyte_cdk": {"stream": {"is_substream": True}}, 5368 "http": {"is_auxiliary": True}, 5369 }, 5370 self._message_repository, 5371 self._evaluate_log_level(self._emit_connector_builder_messages), 5372 ), 5373 ), 5374 api_budget=self._api_budget, 5375 # Share the authenticator registry so parent and child streams draw from the 5376 # same token quota counters 5377 rate_limited_authenticators=self._rate_limited_authenticators, 5378 ) 5379 5380 return substream_factory.create_parent_stream_config( 5381 model=model, config=config, stream_name=stream_name, **kwargs 5382 )
@staticmethod
def
create_wait_time_from_header( model: airbyte_cdk.sources.declarative.models.declarative_component_schema.WaitTimeFromHeader, config: Mapping[str, Any], **kwargs: Any) -> airbyte_cdk.sources.declarative.requesters.error_handlers.backoff_strategies.WaitTimeFromHeaderBackoffStrategy:
5447 @staticmethod 5448 def create_wait_time_from_header( 5449 model: WaitTimeFromHeaderModel, config: Config, **kwargs: Any 5450 ) -> WaitTimeFromHeaderBackoffStrategy: 5451 return WaitTimeFromHeaderBackoffStrategy( 5452 header=model.header, 5453 parameters=model.parameters or {}, 5454 config=config, 5455 regex=model.regex, 5456 max_waiting_time_in_seconds=model.max_waiting_time_in_seconds, 5457 )
@staticmethod
def
create_wait_time_from_response( model: airbyte_cdk.sources.declarative.models.declarative_component_schema.WaitTimeFromResponse, config: Mapping[str, Any], **kwargs: Any) -> airbyte_cdk.sources.declarative.requesters.error_handlers.backoff_strategies.WaitTimeFromResponseBackoffStrategy:
5459 @staticmethod 5460 def create_wait_time_from_response( 5461 model: WaitTimeFromResponseModel, config: Config, **kwargs: Any 5462 ) -> WaitTimeFromResponseBackoffStrategy: 5463 return WaitTimeFromResponseBackoffStrategy( 5464 wait_time_in_seconds=model.wait_time_in_seconds, 5465 parameters=model.parameters or {}, 5466 config=config, 5467 max_waiting_time_in_seconds=model.max_waiting_time_in_seconds, 5468 )
@staticmethod
def
create_wait_until_time_from_header( model: airbyte_cdk.sources.declarative.models.declarative_component_schema.WaitUntilTimeFromHeader, config: Mapping[str, Any], **kwargs: Any) -> airbyte_cdk.sources.declarative.requesters.error_handlers.backoff_strategies.WaitUntilTimeFromHeaderBackoffStrategy:
5470 @staticmethod 5471 def create_wait_until_time_from_header( 5472 model: WaitUntilTimeFromHeaderModel, config: Config, **kwargs: Any 5473 ) -> WaitUntilTimeFromHeaderBackoffStrategy: 5474 return WaitUntilTimeFromHeaderBackoffStrategy( 5475 header=model.header, 5476 parameters=model.parameters or {}, 5477 config=config, 5478 min_wait=model.min_wait, 5479 regex=model.regex, 5480 max_waiting_time_in_seconds=model.max_waiting_time_in_seconds, 5481 )
@staticmethod
def
create_components_mapping_definition( model: airbyte_cdk.sources.declarative.models.declarative_component_schema.ComponentMappingDefinition, config: Mapping[str, Any], **kwargs: Any) -> airbyte_cdk.sources.declarative.resolvers.ComponentMappingDefinition:
5489 @staticmethod 5490 def create_components_mapping_definition( 5491 model: ComponentMappingDefinitionModel, config: Config, **kwargs: Any 5492 ) -> ComponentMappingDefinition: 5493 interpolated_value = InterpolatedString.create( 5494 model.value, parameters=model.parameters or {} 5495 ) 5496 field_path = [ 5497 InterpolatedString.create(path, parameters=model.parameters or {}) 5498 for path in model.field_path 5499 ] 5500 return ComponentMappingDefinition( 5501 field_path=field_path, # type: ignore[arg-type] # field_path can be str and InterpolatedString 5502 value=interpolated_value, 5503 value_type=ModelToComponentFactory._json_schema_type_name_to_type(model.value_type), 5504 create_or_update=model.create_or_update, 5505 condition=model.condition, 5506 parameters=model.parameters or {}, 5507 )
def
create_http_components_resolver( self, model: airbyte_cdk.sources.declarative.models.declarative_component_schema.HttpComponentsResolver, config: Mapping[str, Any], stream_name: Optional[str] = None) -> Any:
5509 def create_http_components_resolver( 5510 self, model: HttpComponentsResolverModel, config: Config, stream_name: Optional[str] = None 5511 ) -> Any: 5512 partition_router = self._build_stream_slicer_from_partition_router(model.retriever, config) 5513 retriever = self._create_component_from_model( 5514 model=model.retriever, 5515 config=config, 5516 name=f"{stream_name if stream_name else '__http_components_resolver'}", 5517 primary_key=None, 5518 stream_slicer=partition_router, 5519 partition_router=partition_router, 5520 transformations=[], 5521 ) 5522 5523 components_mapping = [] 5524 for component_mapping_definition_model in model.components_mapping: 5525 if component_mapping_definition_model.condition: 5526 raise ValueError("`condition` is only supported for `ConfigComponentsResolver`") 5527 components_mapping.append( 5528 self._create_component_from_model( 5529 model=component_mapping_definition_model, 5530 value_type=ModelToComponentFactory._json_schema_type_name_to_type( 5531 component_mapping_definition_model.value_type 5532 ), 5533 config=config, 5534 ) 5535 ) 5536 5537 return HttpComponentsResolver( 5538 retriever=retriever, 5539 # AsyncRetriever reads records only from the job slices its own slicer yields 5540 stream_slicer=retriever.stream_slicer 5541 if isinstance(retriever, AsyncRetriever) 5542 else partition_router, 5543 config=config, 5544 components_mapping=components_mapping, 5545 parameters=model.parameters or {}, 5546 )
@staticmethod
def
create_stream_config( model: airbyte_cdk.sources.declarative.models.declarative_component_schema.StreamConfig, config: Mapping[str, Any], **kwargs: Any) -> airbyte_cdk.sources.declarative.resolvers.StreamConfig:
5548 @staticmethod 5549 def create_stream_config( 5550 model: StreamConfigModel, config: Config, **kwargs: Any 5551 ) -> StreamConfig: 5552 model_configs_pointer: List[Union[InterpolatedString, str]] = ( 5553 [x for x in model.configs_pointer] if model.configs_pointer else [] 5554 ) 5555 5556 return StreamConfig( 5557 configs_pointer=model_configs_pointer, 5558 default_values=model.default_values, 5559 parameters=model.parameters or {}, 5560 )
def
create_config_components_resolver( self, model: airbyte_cdk.sources.declarative.models.declarative_component_schema.ConfigComponentsResolver, config: Mapping[str, Any]) -> Any:
5562 def create_config_components_resolver( 5563 self, 5564 model: ConfigComponentsResolverModel, 5565 config: Config, 5566 ) -> Any: 5567 model_stream_configs = ( 5568 model.stream_config if isinstance(model.stream_config, list) else [model.stream_config] 5569 ) 5570 5571 stream_configs = [ 5572 self._create_component_from_model( 5573 stream_config, config=config, parameters=model.parameters or {} 5574 ) 5575 for stream_config in model_stream_configs 5576 ] 5577 5578 components_mapping = [ 5579 self._create_component_from_model( 5580 model=components_mapping_definition_model, 5581 value_type=ModelToComponentFactory._json_schema_type_name_to_type( 5582 components_mapping_definition_model.value_type 5583 ), 5584 config=config, 5585 parameters=model.parameters, 5586 ) 5587 for components_mapping_definition_model in model.components_mapping 5588 ] 5589 5590 return ConfigComponentsResolver( 5591 stream_configs=stream_configs, 5592 config=config, 5593 components_mapping=components_mapping, 5594 parameters=model.parameters or {}, 5595 )
def
create_parametrized_components_resolver( self, model: airbyte_cdk.sources.declarative.models.declarative_component_schema.ParametrizedComponentsResolver, config: Mapping[str, Any]) -> airbyte_cdk.sources.declarative.resolvers.ParametrizedComponentsResolver:
5597 def create_parametrized_components_resolver( 5598 self, 5599 model: ParametrizedComponentsResolverModel, 5600 config: Config, 5601 ) -> ParametrizedComponentsResolver: 5602 stream_parameters = StreamParametersDefinition( 5603 list_of_parameters_for_stream=model.stream_parameters.list_of_parameters_for_stream 5604 ) 5605 5606 components_mapping = [] 5607 for components_mapping_definition_model in model.components_mapping: 5608 if components_mapping_definition_model.condition: 5609 raise ValueError("`condition` is only supported for `ConfigComponentsResolver`") 5610 components_mapping.append( 5611 self._create_component_from_model( 5612 model=components_mapping_definition_model, 5613 value_type=ModelToComponentFactory._json_schema_type_name_to_type( 5614 components_mapping_definition_model.value_type 5615 ), 5616 config=config, 5617 ) 5618 ) 5619 return ParametrizedComponentsResolver( 5620 stream_parameters=stream_parameters, 5621 config=config, 5622 components_mapping=components_mapping, 5623 parameters=model.parameters or {}, 5624 )
def
create_http_api_budget( self, model: airbyte_cdk.sources.declarative.models.declarative_component_schema.HTTPAPIBudget, config: Mapping[str, Any], **kwargs: Any) -> airbyte_cdk.HttpAPIBudget:
5648 def create_http_api_budget( 5649 self, model: HTTPAPIBudgetModel, config: Config, **kwargs: Any 5650 ) -> HttpAPIBudget: 5651 policies = [ 5652 self._create_component_from_model(model=policy, config=config) 5653 for policy in model.policies 5654 ] 5655 5656 return HttpAPIBudget( 5657 policies=policies, 5658 ratelimit_reset_header=model.ratelimit_reset_header or "ratelimit-reset", 5659 ratelimit_remaining_header=model.ratelimit_remaining_header or "ratelimit-remaining", 5660 status_codes_for_ratelimit_hit=model.status_codes_for_ratelimit_hit or [429], 5661 )
def
create_fixed_window_call_rate_policy( self, model: airbyte_cdk.sources.declarative.models.declarative_component_schema.FixedWindowCallRatePolicy, config: Mapping[str, Any], **kwargs: Any) -> airbyte_cdk.sources.streams.call_rate.FixedWindowCallRatePolicy:
5663 def create_fixed_window_call_rate_policy( 5664 self, model: FixedWindowCallRatePolicyModel, config: Config, **kwargs: Any 5665 ) -> FixedWindowCallRatePolicy: 5666 matchers = [ 5667 self._create_component_from_model(model=matcher, config=config) 5668 for matcher in model.matchers 5669 ] 5670 5671 # Set the initial reset timestamp to 10 days from now. 5672 # This value will be updated by the first request. 5673 return FixedWindowCallRatePolicy( 5674 next_reset_ts=datetime.datetime.now() + datetime.timedelta(days=10), 5675 period=parse_duration(model.period), 5676 call_limit=model.call_limit, 5677 matchers=matchers, 5678 )
def
create_file_uploader( self, model: airbyte_cdk.sources.declarative.models.declarative_component_schema.FileUploader, config: Mapping[str, Any], **kwargs: Any) -> airbyte_cdk.sources.declarative.retrievers.file_uploader.FileUploader:
5680 def create_file_uploader( 5681 self, model: FileUploaderModel, config: Config, **kwargs: Any 5682 ) -> FileUploader: 5683 self._reject_reduce_page_size_action(model.requester, "requester of a `file_uploader`") 5684 self._reject_split_request_window_action(model.requester, "requester of a `file_uploader`") 5685 name = "File Uploader" 5686 requester = self._create_component_from_model( 5687 model=model.requester, 5688 config=config, 5689 name=name, 5690 **kwargs, 5691 ) 5692 download_target_extractor = self._create_component_from_model( 5693 model=model.download_target_extractor, 5694 config=config, 5695 name=name, 5696 **kwargs, 5697 ) 5698 emit_connector_builder_messages = self._emit_connector_builder_messages 5699 file_uploader = DefaultFileUploader( 5700 requester=requester, 5701 download_target_extractor=download_target_extractor, 5702 config=config, 5703 file_writer=NoopFileWriter() 5704 if emit_connector_builder_messages 5705 else LocalFileSystemFileWriter(), 5706 parameters=model.parameters or {}, 5707 filename_extractor=model.filename_extractor if model.filename_extractor else None, 5708 ) 5709 5710 return ( 5711 ConnectorBuilderFileUploader(file_uploader) 5712 if emit_connector_builder_messages 5713 else file_uploader 5714 )
def
create_moving_window_call_rate_policy( self, model: airbyte_cdk.sources.declarative.models.declarative_component_schema.MovingWindowCallRatePolicy, config: Mapping[str, Any], **kwargs: Any) -> airbyte_cdk.MovingWindowCallRatePolicy:
5716 def create_moving_window_call_rate_policy( 5717 self, model: MovingWindowCallRatePolicyModel, config: Config, **kwargs: Any 5718 ) -> MovingWindowCallRatePolicy: 5719 rates = [ 5720 self._create_component_from_model(model=rate, config=config) for rate in model.rates 5721 ] 5722 matchers = [ 5723 self._create_component_from_model(model=matcher, config=config) 5724 for matcher in model.matchers 5725 ] 5726 return MovingWindowCallRatePolicy( 5727 rates=rates, 5728 matchers=matchers, 5729 )
def
create_unlimited_call_rate_policy( self, model: airbyte_cdk.sources.declarative.models.declarative_component_schema.UnlimitedCallRatePolicy, config: Mapping[str, Any], **kwargs: Any) -> airbyte_cdk.sources.streams.call_rate.UnlimitedCallRatePolicy:
5731 def create_unlimited_call_rate_policy( 5732 self, model: UnlimitedCallRatePolicyModel, config: Config, **kwargs: Any 5733 ) -> UnlimitedCallRatePolicy: 5734 matchers = [ 5735 self._create_component_from_model(model=matcher, config=config) 5736 for matcher in model.matchers 5737 ] 5738 5739 return UnlimitedCallRatePolicy( 5740 matchers=matchers, 5741 )
def
create_rate( self, model: airbyte_cdk.sources.declarative.models.declarative_component_schema.Rate, config: Mapping[str, Any], **kwargs: Any) -> airbyte_cdk.Rate:
def
create_http_request_matcher( self, model: airbyte_cdk.sources.declarative.models.declarative_component_schema.HttpRequestRegexMatcher, config: Mapping[str, Any], **kwargs: Any) -> airbyte_cdk.sources.streams.call_rate.HttpRequestRegexMatcher:
5750 def create_http_request_matcher( 5751 self, model: HttpRequestRegexMatcherModel, config: Config, **kwargs: Any 5752 ) -> HttpRequestRegexMatcher: 5753 weight = model.weight 5754 if weight is not None: 5755 if isinstance(weight, str): 5756 weight = int(InterpolatedString.create(weight, parameters={}).eval(config)) 5757 else: 5758 weight = int(weight) 5759 if weight < 1: 5760 raise ValueError(f"weight must be >= 1, got {weight}") 5761 return HttpRequestRegexMatcher( 5762 method=model.method, 5763 url_base=model.url_base, 5764 url_path_pattern=model.url_path_pattern, 5765 params=model.params, 5766 headers=model.headers, 5767 weight=weight, 5768 )
def
create_rate_limited_multiple_token_authenticator( self, model: airbyte_cdk.sources.declarative.models.declarative_component_schema.RateLimitedMultipleTokenAuthenticator, config: Mapping[str, Any], **kwargs: Any) -> airbyte_cdk.sources.declarative.auth.RateLimitedMultipleTokenAuthenticator:
5770 def create_rate_limited_multiple_token_authenticator( 5771 self, 5772 model: RateLimitedMultipleTokenAuthenticatorModel, 5773 config: Config, 5774 **kwargs: Any, 5775 ) -> RateLimitedMultipleTokenAuthenticator: 5776 if isinstance(model.tokens, str): 5777 tokens_value = InterpolatedString.create(model.tokens, parameters={}).eval(config) 5778 delimiter = model.token_delimiter or "," 5779 tokens = [ 5780 token.strip() for token in str(tokens_value).split(delimiter) if token.strip() 5781 ] 5782 else: 5783 tokens = [ 5784 token_value 5785 for token in model.tokens 5786 if ( 5787 token_value := str( 5788 InterpolatedString.create(token, parameters={}).eval(config) 5789 ).strip() 5790 ) 5791 ] 5792 5793 quota_specs = [ 5794 { 5795 "name": quota_model.name, 5796 "remaining_path": quota_model.remaining_path, 5797 "reset_path": quota_model.reset_path, 5798 "limit_path": quota_model.limit_path, 5799 "remaining_header": quota_model.remaining_header, 5800 "reset_header": quota_model.reset_header, 5801 "limit_header": quota_model.limit_header, 5802 # Normalize the same way as the runtime TokenQuota below, so an omitted field 5803 # and an explicit `[]` key identically and keep sharing one set of counters. 5804 "exhaustion_status_codes": quota_model.exhaustion_status_codes or [], 5805 "matchers": [ 5806 { 5807 "method": matcher_model.method, 5808 "url_base": matcher_model.url_base, 5809 "url_path_pattern": matcher_model.url_path_pattern, 5810 "params": matcher_model.params, 5811 "headers": matcher_model.headers, 5812 "weight": matcher_model.weight, 5813 } 5814 for matcher_model in quota_model.matchers or [] 5815 ], 5816 } 5817 for quota_model in model.quotas 5818 ] 5819 5820 quota_status_url = str( 5821 InterpolatedString.create(model.quota_status_source.url, parameters={}).eval(config) 5822 ) 5823 quota_status_http_method = ( 5824 model.quota_status_source.http_method.value 5825 if model.quota_status_source.http_method 5826 else "GET" 5827 ) 5828 quota_status_headers = { 5829 key: str(InterpolatedString.create(value, parameters={}).eval(config)) 5830 for key, value in (model.quota_status_source.request_headers or {}).items() 5831 } 5832 # Normalize the same way as the quota specs above, so an omitted field and an explicit 5833 # `[]` key identically and keep sharing one set of counters. Deduplicated as well as 5834 # sorted, because the runtime turns this into a set: without it `[404]` and `[404, 404]` 5835 # would key differently and stop sharing counters while behaving identically. 5836 quota_status_unavailable_status_codes = sorted( 5837 set(model.quota_status_source.unavailable_status_codes or []) 5838 ) 5839 auth_method = model.auth_method or "Bearer" 5840 header = model.header or "Authorization" 5841 max_wait_time_str = str( 5842 InterpolatedString.create(model.max_wait_time or "PT2H", parameters={}).eval(config) 5843 ) 5844 max_wait_time = parse_duration(max_wait_time_str) 5845 if not isinstance(max_wait_time, datetime.timedelta): 5846 raise ValueError( 5847 f"max_wait_time must be a fixed-length ISO 8601 duration (e.g. 'PT2H'); " 5848 f"calendar-unit durations like '{max_wait_time_str}' are not supported" 5849 ) 5850 budget_reserve_fraction = ( 5851 model.budget_reserve_fraction if model.budget_reserve_fraction is not None else 0.1 5852 ) 5853 budget_min_reserve = ( 5854 model.budget_min_reserve if model.budget_min_reserve is not None else 50 5855 ) 5856 5857 # Reuse the same instance for identical definitions so that all streams share the 5858 # same token quota counters (similar to how api_budget is shared). The key is built 5859 # from the resolved constructor arguments rather than the raw model so that 5860 # stream-specific `$parameters` propagated onto the model (and its nested components) 5861 # cannot break instance sharing. 5862 cache_key = json.dumps( 5863 { 5864 "tokens": tokens, 5865 "quotas": quota_specs, 5866 "quota_status_url": quota_status_url, 5867 "quota_status_http_method": quota_status_http_method, 5868 "quota_status_headers": quota_status_headers, 5869 "quota_status_unavailable_status_codes": quota_status_unavailable_status_codes, 5870 "auth_method": auth_method, 5871 "header": header, 5872 "max_wait_time": max_wait_time.total_seconds(), 5873 "budget_reserve_fraction": budget_reserve_fraction, 5874 "budget_min_reserve": budget_min_reserve, 5875 }, 5876 sort_keys=True, 5877 ) 5878 if cache_key in self._rate_limited_authenticators: 5879 return self._rate_limited_authenticators[cache_key] 5880 5881 quotas = [ 5882 TokenQuota( 5883 name=quota_model.name, 5884 remaining_path=quota_model.remaining_path, 5885 reset_path=quota_model.reset_path, 5886 limit_path=quota_model.limit_path, 5887 remaining_header=quota_model.remaining_header, 5888 reset_header=quota_model.reset_header, 5889 limit_header=quota_model.limit_header, 5890 exhaustion_status_codes=quota_model.exhaustion_status_codes or [], 5891 matchers=[ 5892 self.create_http_request_matcher(matcher_model, config) 5893 for matcher_model in quota_model.matchers or [] 5894 ], 5895 ) 5896 for quota_model in model.quotas 5897 ] 5898 5899 authenticator = RateLimitedMultipleTokenAuthenticator( 5900 tokens=tokens, 5901 quotas=quotas, 5902 quota_status_url=quota_status_url, 5903 quota_status_http_method=quota_status_http_method, 5904 quota_status_headers=quota_status_headers, 5905 quota_status_unavailable_status_codes=quota_status_unavailable_status_codes, 5906 auth_method=auth_method, 5907 header=header, 5908 max_wait_time=max_wait_time, 5909 budget_reserve_fraction=budget_reserve_fraction, 5910 budget_min_reserve=budget_min_reserve, 5911 ) 5912 self._rate_limited_authenticators[cache_key] = authenticator 5913 return authenticator
def
set_api_budget( self, component_definition: Mapping[str, Any], config: Mapping[str, Any]) -> None:
def
create_grouping_partition_router( self, model: airbyte_cdk.sources.declarative.models.declarative_component_schema.GroupingPartitionRouter, config: Mapping[str, Any], *, stream_name: str, **kwargs: Any) -> airbyte_cdk.sources.declarative.partition_routers.GroupingPartitionRouter:
5920 def create_grouping_partition_router( 5921 self, 5922 model: GroupingPartitionRouterModel, 5923 config: Config, 5924 *, 5925 stream_name: str, 5926 **kwargs: Any, 5927 ) -> GroupingPartitionRouter: 5928 underlying_router = self._create_component_from_model( 5929 model=model.underlying_partition_router, 5930 config=config, 5931 stream_name=stream_name, 5932 **kwargs, 5933 ) 5934 if model.group_size < 1: 5935 raise ValueError(f"Group size must be greater than 0, got {model.group_size}") 5936 5937 # Request options in underlying partition routers are not supported for GroupingPartitionRouter 5938 # because they are specific to individual partitions and cannot be aggregated or handled 5939 # when grouping, potentially leading to incorrect API calls. Any request customization 5940 # should be managed at the stream level through the requester's configuration. 5941 if isinstance(underlying_router, SubstreamPartitionRouter): 5942 if any( 5943 parent_config.request_option 5944 for parent_config in underlying_router.parent_stream_configs 5945 ): 5946 raise ValueError("Request options are not supported for GroupingPartitionRouter.") 5947 5948 if isinstance(underlying_router, ListPartitionRouter): 5949 if underlying_router.request_option: 5950 raise ValueError("Request options are not supported for GroupingPartitionRouter.") 5951 5952 return GroupingPartitionRouter( 5953 group_size=model.group_size, 5954 underlying_partition_router=underlying_router, 5955 deduplicate=model.deduplicate if model.deduplicate is not None else True, 5956 config=config, 5957 )
def
create_union_partition_router( self, model: airbyte_cdk.sources.declarative.models.declarative_component_schema.UnionPartitionRouter, config: Mapping[str, Any], *, stream_name: str, **kwargs: Any) -> airbyte_cdk.UnionPartitionRouter:
5959 def create_union_partition_router( 5960 self, 5961 model: UnionPartitionRouterModel, 5962 config: Config, 5963 *, 5964 stream_name: str, 5965 **kwargs: Any, 5966 ) -> UnionPartitionRouter: 5967 # The schema enforces minItems: 2 for manifests; this guard covers construction paths 5968 # that bypass JSON-schema validation (the generated model carries no min_items constraint). 5969 if len(model.partition_routers) < 2: 5970 raise ValueError( 5971 f"UnionPartitionRouter for stream {stream_name} needs at least 2 child partition routers" 5972 ) 5973 5974 partition_routers = [ 5975 self._create_component_from_model( 5976 model=child, 5977 config=config, 5978 stream_name=stream_name, 5979 **kwargs, 5980 ) 5981 for child in model.partition_routers 5982 ] 5983 5984 # partition_field depends only on config/parameters, so it is evaluated once at build 5985 # time; the runtime component always receives a plain string. 5986 partition_field = InterpolatedString.create( 5987 model.partition_field, parameters=model.parameters or {} 5988 ).eval(config) 5989 5990 # Fail fast at build time when a built-in child router is statically known to emit a 5991 # partition field different from the union's. CustomPartitionRouter children are opaque 5992 # and can only be validated at runtime. 5993 for child_model in model.partition_routers: 5994 child_partition_fields: List[str] = [] 5995 if isinstance(child_model, ListPartitionRouterModel): 5996 child_partition_fields.append( 5997 InterpolatedString.create( 5998 child_model.cursor_field, parameters=child_model.parameters or {} 5999 ).eval(config) 6000 ) 6001 elif isinstance(child_model, SubstreamPartitionRouterModel): 6002 for parent_stream_config in child_model.parent_stream_configs: 6003 child_partition_fields.append( 6004 InterpolatedString.create( 6005 parent_stream_config.partition_field, 6006 parameters=parent_stream_config.parameters 6007 or child_model.parameters 6008 or {}, 6009 ).eval(config) 6010 ) 6011 elif isinstance(child_model, UnionPartitionRouterModel): 6012 child_partition_fields.append( 6013 InterpolatedString.create( 6014 child_model.partition_field, parameters=child_model.parameters or {} 6015 ).eval(config) 6016 ) 6017 for child_partition_field in child_partition_fields: 6018 if child_partition_field != partition_field: 6019 raise ValueError( 6020 f"UnionPartitionRouter expects all child partition routers to emit the " 6021 f"partition field '{partition_field}', but a " 6022 f"{child_model.type} child emits '{child_partition_field}'." 6023 ) 6024 6025 # A union slice comes from exactly one child partition router, so request options 6026 # declared on children cannot be applied consistently to requests built from the 6027 # normalized union slices. Partition values should be consumed via interpolation 6028 # (e.g. stream_partition) instead. Note that this validation only covers built-in 6029 # router types; CustomPartitionRouter children are opaque, so any request options 6030 # they implement internally cannot be detected or rejected here. 6031 for router in partition_routers: 6032 if isinstance(router, SubstreamPartitionRouter): 6033 if any( 6034 parent_config.request_option for parent_config in router.parent_stream_configs 6035 ): 6036 raise ValueError("Request options are not supported for UnionPartitionRouter.") 6037 if isinstance(router, ListPartitionRouter) and router.request_option: 6038 raise ValueError("Request options are not supported for UnionPartitionRouter.") 6039 6040 return UnionPartitionRouter( 6041 partition_routers=partition_routers, 6042 partition_field=partition_field, 6043 parameters=model.parameters or {}, 6044 )