airbyte_cdk.sources.declarative.parsers.model_to_component_factory
1# 2# Copyright (c) 2025 Airbyte, Inc., all rights reserved. 3# 4 5from __future__ import annotations 6 7import datetime 8import importlib 9import inspect 10import json 11import logging 12import re 13from functools import partial 14from typing import ( 15 TYPE_CHECKING, 16 Any, 17 Callable, 18 Dict, 19 List, 20 Mapping, 21 MutableMapping, 22 Optional, 23 Tuple, 24 Type, 25 Union, 26 cast, 27 get_args, 28 get_origin, 29 get_type_hints, 30) 31 32if TYPE_CHECKING: 33 from airbyte_cdk.legacy.sources.declarative.incremental.datetime_based_cursor import ( 34 DatetimeBasedCursor, 35 ) 36 37from airbyte_protocol_dataclasses.models import ConfiguredAirbyteStream 38from isodate import parse_duration 39from pydantic.v1 import BaseModel 40from requests import Response 41 42from airbyte_cdk.connector_builder.models import ( 43 LogMessage as ConnectorBuilderLogMessage, 44) 45from airbyte_cdk.models import ( 46 AirbyteStateBlob, 47 AirbyteStateMessage, 48 AirbyteStateType, 49 AirbyteStreamState, 50 ConfiguredAirbyteCatalog, 51 FailureType, 52 Level, 53 StreamDescriptor, 54) 55from airbyte_cdk.sources.connector_state_manager import ConnectorStateManager 56from airbyte_cdk.sources.declarative.async_job.job_orchestrator import AsyncJobOrchestrator 57from airbyte_cdk.sources.declarative.async_job.job_tracker import JobTracker 58from airbyte_cdk.sources.declarative.async_job.repository import AsyncJobRepository 59from airbyte_cdk.sources.declarative.async_job.status import AsyncJobStatus 60from airbyte_cdk.sources.declarative.auth import DeclarativeOauth2Authenticator, JwtAuthenticator 61from airbyte_cdk.sources.declarative.auth.declarative_authenticator import ( 62 DeclarativeAuthenticator, 63 NoAuth, 64) 65from airbyte_cdk.sources.declarative.auth.jwt import JwtAlgorithm 66from airbyte_cdk.sources.declarative.auth.oauth import ( 67 DeclarativeSingleUseRefreshTokenOauth2Authenticator, 68) 69from airbyte_cdk.sources.declarative.auth.rate_limited_multiple_token import ( 70 RateLimitedMultipleTokenAuthenticator, 71 TokenQuota, 72) 73from airbyte_cdk.sources.declarative.auth.selective_authenticator import SelectiveAuthenticator 74from airbyte_cdk.sources.declarative.auth.token import ( 75 ApiKeyAuthenticator, 76 BasicHttpAuthenticator, 77 BearerAuthenticator, 78 LegacySessionTokenAuthenticator, 79) 80from airbyte_cdk.sources.declarative.auth.token_provider import ( 81 InterpolatedSessionTokenProvider, 82 InterpolatedStringTokenProvider, 83 SessionTokenProvider, 84 TokenProvider, 85) 86from airbyte_cdk.sources.declarative.checks import ( 87 CheckDynamicStream, 88 CheckStream, 89 DynamicStreamCheckConfig, 90) 91from airbyte_cdk.sources.declarative.concurrency_level import ConcurrencyLevel 92from airbyte_cdk.sources.declarative.datetime.min_max_datetime import MinMaxDatetime 93from airbyte_cdk.sources.declarative.decoders import ( 94 Decoder, 95 IterableDecoder, 96 JsonDecoder, 97 PaginationDecoderDecorator, 98 XmlDecoder, 99 ZipfileDecoder, 100) 101from airbyte_cdk.sources.declarative.decoders.composite_raw_decoder import ( 102 CompositeRawDecoder, 103 CsvParser, 104 GzipParser, 105 JsonItemsParser, 106 JsonLineParser, 107 JsonParser, 108 Parser, 109) 110from airbyte_cdk.sources.declarative.expanders.record_expander import ( 111 OnNoRecords, 112 RecordExpander, 113) 114from airbyte_cdk.sources.declarative.extractors import ( 115 DpathExtractor, 116 RecordFilter, 117 RecordSelector, 118 ResponseToFileExtractor, 119) 120from airbyte_cdk.sources.declarative.extractors.record_extractor import RecordExtractor 121from airbyte_cdk.sources.declarative.extractors.record_filter import ( 122 ClientSideIncrementalRecordFilterDecorator, 123) 124from airbyte_cdk.sources.declarative.incremental import ( 125 ConcurrentCursorFactory, 126 ConcurrentPerPartitionCursor, 127) 128from airbyte_cdk.sources.declarative.interpolation import InterpolatedString 129from airbyte_cdk.sources.declarative.interpolation.interpolated_mapping import InterpolatedMapping 130from airbyte_cdk.sources.declarative.migrations.legacy_to_per_partition_state_migration import ( 131 LegacyToPerPartitionStateMigration, 132) 133from airbyte_cdk.sources.declarative.models import ( 134 CustomStateMigration, 135 PaginationResetLimits, 136) 137from airbyte_cdk.sources.declarative.models.base_model_with_deprecations import ( 138 DEPRECATION_LOGS_TAG, 139 BaseModelWithDeprecations, 140) 141from airbyte_cdk.sources.declarative.models.declarative_component_schema import ( 142 Action1 as PaginationResetActionModel, 143) 144from airbyte_cdk.sources.declarative.models.declarative_component_schema import ( 145 AddedFieldDefinition as AddedFieldDefinitionModel, 146) 147from airbyte_cdk.sources.declarative.models.declarative_component_schema import ( 148 AddFields as AddFieldsModel, 149) 150from airbyte_cdk.sources.declarative.models.declarative_component_schema import ( 151 ApiKeyAuthenticator as ApiKeyAuthenticatorModel, 152) 153from airbyte_cdk.sources.declarative.models.declarative_component_schema import ( 154 AsyncJobStatusMap as AsyncJobStatusMapModel, 155) 156from airbyte_cdk.sources.declarative.models.declarative_component_schema import ( 157 AsyncRetriever as AsyncRetrieverModel, 158) 159from airbyte_cdk.sources.declarative.models.declarative_component_schema import ( 160 BasicHttpAuthenticator as BasicHttpAuthenticatorModel, 161) 162from airbyte_cdk.sources.declarative.models.declarative_component_schema import ( 163 BearerAuthenticator as BearerAuthenticatorModel, 164) 165from airbyte_cdk.sources.declarative.models.declarative_component_schema import ( 166 CheckDynamicStream as CheckDynamicStreamModel, 167) 168from airbyte_cdk.sources.declarative.models.declarative_component_schema import ( 169 CheckStream as CheckStreamModel, 170) 171from airbyte_cdk.sources.declarative.models.declarative_component_schema import ( 172 ComplexFieldType as ComplexFieldTypeModel, 173) 174from airbyte_cdk.sources.declarative.models.declarative_component_schema import ( 175 ComponentMappingDefinition as ComponentMappingDefinitionModel, 176) 177from airbyte_cdk.sources.declarative.models.declarative_component_schema import ( 178 CompositeErrorHandler as CompositeErrorHandlerModel, 179) 180from airbyte_cdk.sources.declarative.models.declarative_component_schema import ( 181 ConcurrencyLevel as ConcurrencyLevelModel, 182) 183from airbyte_cdk.sources.declarative.models.declarative_component_schema import ( 184 ConfigAddFields as ConfigAddFieldsModel, 185) 186from airbyte_cdk.sources.declarative.models.declarative_component_schema import ( 187 ConfigComponentsResolver as ConfigComponentsResolverModel, 188) 189from airbyte_cdk.sources.declarative.models.declarative_component_schema import ( 190 ConfigMigration as ConfigMigrationModel, 191) 192from airbyte_cdk.sources.declarative.models.declarative_component_schema import ( 193 ConfigRemapField as ConfigRemapFieldModel, 194) 195from airbyte_cdk.sources.declarative.models.declarative_component_schema import ( 196 ConfigRemoveFields as ConfigRemoveFieldsModel, 197) 198from airbyte_cdk.sources.declarative.models.declarative_component_schema import ( 199 ConstantBackoffStrategy as ConstantBackoffStrategyModel, 200) 201from airbyte_cdk.sources.declarative.models.declarative_component_schema import ( 202 CsvDecoder as CsvDecoderModel, 203) 204from airbyte_cdk.sources.declarative.models.declarative_component_schema import ( 205 CursorPagination as CursorPaginationModel, 206) 207from airbyte_cdk.sources.declarative.models.declarative_component_schema import ( 208 CustomAuthenticator as CustomAuthenticatorModel, 209) 210from airbyte_cdk.sources.declarative.models.declarative_component_schema import ( 211 CustomBackoffStrategy as CustomBackoffStrategyModel, 212) 213from airbyte_cdk.sources.declarative.models.declarative_component_schema import ( 214 CustomConfigTransformation as CustomConfigTransformationModel, 215) 216from airbyte_cdk.sources.declarative.models.declarative_component_schema import ( 217 CustomDecoder as CustomDecoderModel, 218) 219from airbyte_cdk.sources.declarative.models.declarative_component_schema import ( 220 CustomErrorHandler as CustomErrorHandlerModel, 221) 222from airbyte_cdk.sources.declarative.models.declarative_component_schema import ( 223 CustomPaginationStrategy as CustomPaginationStrategyModel, 224) 225from airbyte_cdk.sources.declarative.models.declarative_component_schema import ( 226 CustomPartitionRouter as CustomPartitionRouterModel, 227) 228from airbyte_cdk.sources.declarative.models.declarative_component_schema import ( 229 CustomRecordExtractor as CustomRecordExtractorModel, 230) 231from airbyte_cdk.sources.declarative.models.declarative_component_schema import ( 232 CustomRecordFilter as CustomRecordFilterModel, 233) 234from airbyte_cdk.sources.declarative.models.declarative_component_schema import ( 235 CustomRequester as CustomRequesterModel, 236) 237from airbyte_cdk.sources.declarative.models.declarative_component_schema import ( 238 CustomRetriever as CustomRetrieverModel, 239) 240from airbyte_cdk.sources.declarative.models.declarative_component_schema import ( 241 CustomSchemaLoader as CustomSchemaLoader, 242) 243from airbyte_cdk.sources.declarative.models.declarative_component_schema import ( 244 CustomSchemaNormalization as CustomSchemaNormalizationModel, 245) 246from airbyte_cdk.sources.declarative.models.declarative_component_schema import ( 247 CustomTransformation as CustomTransformationModel, 248) 249from airbyte_cdk.sources.declarative.models.declarative_component_schema import ( 250 CustomValidationStrategy as CustomValidationStrategyModel, 251) 252from airbyte_cdk.sources.declarative.models.declarative_component_schema import ( 253 DatetimeBasedCursor as DatetimeBasedCursorModel, 254) 255from airbyte_cdk.sources.declarative.models.declarative_component_schema import ( 256 DeclarativeStream as DeclarativeStreamModel, 257) 258from airbyte_cdk.sources.declarative.models.declarative_component_schema import ( 259 DefaultErrorHandler as DefaultErrorHandlerModel, 260) 261from airbyte_cdk.sources.declarative.models.declarative_component_schema import ( 262 DefaultPaginator as DefaultPaginatorModel, 263) 264from airbyte_cdk.sources.declarative.models.declarative_component_schema import ( 265 DpathExtractor as DpathExtractorModel, 266) 267from airbyte_cdk.sources.declarative.models.declarative_component_schema import ( 268 DpathFlattenFields as DpathFlattenFieldsModel, 269) 270from airbyte_cdk.sources.declarative.models.declarative_component_schema import ( 271 DpathValidator as DpathValidatorModel, 272) 273from airbyte_cdk.sources.declarative.models.declarative_component_schema import ( 274 DynamicSchemaLoader as DynamicSchemaLoaderModel, 275) 276from airbyte_cdk.sources.declarative.models.declarative_component_schema import ( 277 DynamicStreamCheckConfig as DynamicStreamCheckConfigModel, 278) 279from airbyte_cdk.sources.declarative.models.declarative_component_schema import ( 280 ExponentialBackoffStrategy as ExponentialBackoffStrategyModel, 281) 282from airbyte_cdk.sources.declarative.models.declarative_component_schema import ( 283 FileUploader as FileUploaderModel, 284) 285from airbyte_cdk.sources.declarative.models.declarative_component_schema import ( 286 FixedWindowCallRatePolicy as FixedWindowCallRatePolicyModel, 287) 288from airbyte_cdk.sources.declarative.models.declarative_component_schema import ( 289 FlattenFields as FlattenFieldsModel, 290) 291from airbyte_cdk.sources.declarative.models.declarative_component_schema import ( 292 GroupByKeyMergeStrategy as GroupByKeyMergeStrategyModel, 293) 294from airbyte_cdk.sources.declarative.models.declarative_component_schema import ( 295 GroupingPartitionRouter as GroupingPartitionRouterModel, 296) 297from airbyte_cdk.sources.declarative.models.declarative_component_schema import ( 298 GzipDecoder as GzipDecoderModel, 299) 300from airbyte_cdk.sources.declarative.models.declarative_component_schema import ( 301 HTTPAPIBudget as HTTPAPIBudgetModel, 302) 303from airbyte_cdk.sources.declarative.models.declarative_component_schema import ( 304 HttpComponentsResolver as HttpComponentsResolverModel, 305) 306from airbyte_cdk.sources.declarative.models.declarative_component_schema import ( 307 HttpRequester as HttpRequesterModel, 308) 309from airbyte_cdk.sources.declarative.models.declarative_component_schema import ( 310 HttpRequestRegexMatcher as HttpRequestRegexMatcherModel, 311) 312from airbyte_cdk.sources.declarative.models.declarative_component_schema import ( 313 HttpResponseFilter as HttpResponseFilterModel, 314) 315from airbyte_cdk.sources.declarative.models.declarative_component_schema import ( 316 IncrementingCountCursor as IncrementingCountCursorModel, 317) 318from airbyte_cdk.sources.declarative.models.declarative_component_schema import ( 319 InlineSchemaLoader as InlineSchemaLoaderModel, 320) 321from airbyte_cdk.sources.declarative.models.declarative_component_schema import ( 322 IterableDecoder as IterableDecoderModel, 323) 324from airbyte_cdk.sources.declarative.models.declarative_component_schema import ( 325 JsonDecoder as JsonDecoderModel, 326) 327from airbyte_cdk.sources.declarative.models.declarative_component_schema import ( 328 JsonFileSchemaLoader as JsonFileSchemaLoaderModel, 329) 330from airbyte_cdk.sources.declarative.models.declarative_component_schema import ( 331 JsonItemsDecoder as JsonItemsDecoderModel, 332) 333from airbyte_cdk.sources.declarative.models.declarative_component_schema import ( 334 JsonlDecoder as JsonlDecoderModel, 335) 336from airbyte_cdk.sources.declarative.models.declarative_component_schema import ( 337 JsonSchemaPropertySelector as JsonSchemaPropertySelectorModel, 338) 339from airbyte_cdk.sources.declarative.models.declarative_component_schema import ( 340 JwtAuthenticator as JwtAuthenticatorModel, 341) 342from airbyte_cdk.sources.declarative.models.declarative_component_schema import ( 343 JwtHeaders as JwtHeadersModel, 344) 345from airbyte_cdk.sources.declarative.models.declarative_component_schema import ( 346 JwtPayload as JwtPayloadModel, 347) 348from airbyte_cdk.sources.declarative.models.declarative_component_schema import ( 349 KeysReplace as KeysReplaceModel, 350) 351from airbyte_cdk.sources.declarative.models.declarative_component_schema import ( 352 KeysToLower as KeysToLowerModel, 353) 354from airbyte_cdk.sources.declarative.models.declarative_component_schema import ( 355 KeysToSnakeCase as KeysToSnakeCaseModel, 356) 357from airbyte_cdk.sources.declarative.models.declarative_component_schema import ( 358 LegacySessionTokenAuthenticator as LegacySessionTokenAuthenticatorModel, 359) 360from airbyte_cdk.sources.declarative.models.declarative_component_schema import ( 361 LegacyToPerPartitionStateMigration as LegacyToPerPartitionStateMigrationModel, 362) 363from airbyte_cdk.sources.declarative.models.declarative_component_schema import ( 364 ListPartitionRouter as ListPartitionRouterModel, 365) 366from airbyte_cdk.sources.declarative.models.declarative_component_schema import ( 367 MinMaxDatetime as MinMaxDatetimeModel, 368) 369from airbyte_cdk.sources.declarative.models.declarative_component_schema import ( 370 MovingWindowCallRatePolicy as MovingWindowCallRatePolicyModel, 371) 372from airbyte_cdk.sources.declarative.models.declarative_component_schema import ( 373 NoAuth as NoAuthModel, 374) 375from airbyte_cdk.sources.declarative.models.declarative_component_schema import ( 376 NoPagination as NoPaginationModel, 377) 378from airbyte_cdk.sources.declarative.models.declarative_component_schema import ( 379 OAuthAuthenticator as OAuthAuthenticatorModel, 380) 381from airbyte_cdk.sources.declarative.models.declarative_component_schema import ( 382 OffsetIncrement as OffsetIncrementModel, 383) 384from airbyte_cdk.sources.declarative.models.declarative_component_schema import ( 385 PageIncrement as PageIncrementModel, 386) 387from airbyte_cdk.sources.declarative.models.declarative_component_schema import ( 388 PaginationReset as PaginationResetModel, 389) 390from airbyte_cdk.sources.declarative.models.declarative_component_schema import ( 391 ParametrizedComponentsResolver as ParametrizedComponentsResolverModel, 392) 393from airbyte_cdk.sources.declarative.models.declarative_component_schema import ( 394 ParentStreamConfig as ParentStreamConfigModel, 395) 396from airbyte_cdk.sources.declarative.models.declarative_component_schema import ( 397 PredicateValidator as PredicateValidatorModel, 398) 399from airbyte_cdk.sources.declarative.models.declarative_component_schema import ( 400 PropertiesFromEndpoint as PropertiesFromEndpointModel, 401) 402from airbyte_cdk.sources.declarative.models.declarative_component_schema import ( 403 PropertyChunking as PropertyChunkingModel, 404) 405from airbyte_cdk.sources.declarative.models.declarative_component_schema import ( 406 PropertyLimitType as PropertyLimitTypeModel, 407) 408from airbyte_cdk.sources.declarative.models.declarative_component_schema import ( 409 QueryProperties as QueryPropertiesModel, 410) 411from airbyte_cdk.sources.declarative.models.declarative_component_schema import ( 412 Rate as RateModel, 413) 414from airbyte_cdk.sources.declarative.models.declarative_component_schema import ( 415 RateLimitedMultipleTokenAuthenticator as RateLimitedMultipleTokenAuthenticatorModel, 416) 417from airbyte_cdk.sources.declarative.models.declarative_component_schema import ( 418 RecordExpander as RecordExpanderModel, 419) 420from airbyte_cdk.sources.declarative.models.declarative_component_schema import ( 421 RecordFilter as RecordFilterModel, 422) 423from airbyte_cdk.sources.declarative.models.declarative_component_schema import ( 424 RecordSelector as RecordSelectorModel, 425) 426from airbyte_cdk.sources.declarative.models.declarative_component_schema import ( 427 RefreshTokenUpdater as RefreshTokenUpdaterModel, 428) 429from airbyte_cdk.sources.declarative.models.declarative_component_schema import ( 430 RemoveFields as RemoveFieldsModel, 431) 432from airbyte_cdk.sources.declarative.models.declarative_component_schema import ( 433 RequestOption as RequestOptionModel, 434) 435from airbyte_cdk.sources.declarative.models.declarative_component_schema import ( 436 RequestPath as RequestPathModel, 437) 438from airbyte_cdk.sources.declarative.models.declarative_component_schema import ( 439 ResponseToFileExtractor as ResponseToFileExtractorModel, 440) 441from airbyte_cdk.sources.declarative.models.declarative_component_schema import ( 442 SchemaNormalization as SchemaNormalizationModel, 443) 444from airbyte_cdk.sources.declarative.models.declarative_component_schema import ( 445 SchemaTypeIdentifier as SchemaTypeIdentifierModel, 446) 447from airbyte_cdk.sources.declarative.models.declarative_component_schema import ( 448 SelectiveAuthenticator as SelectiveAuthenticatorModel, 449) 450from airbyte_cdk.sources.declarative.models.declarative_component_schema import ( 451 SessionTokenAuthenticator as SessionTokenAuthenticatorModel, 452) 453from airbyte_cdk.sources.declarative.models.declarative_component_schema import ( 454 SimpleRetriever as SimpleRetrieverModel, 455) 456from airbyte_cdk.sources.declarative.models.declarative_component_schema import Spec as SpecModel 457from airbyte_cdk.sources.declarative.models.declarative_component_schema import ( 458 StateDelegatingStream as StateDelegatingStreamModel, 459) 460from airbyte_cdk.sources.declarative.models.declarative_component_schema import ( 461 StreamConfig as StreamConfigModel, 462) 463from airbyte_cdk.sources.declarative.models.declarative_component_schema import ( 464 SubstreamPartitionRouter as SubstreamPartitionRouterModel, 465) 466from airbyte_cdk.sources.declarative.models.declarative_component_schema import ( 467 TypesMap as TypesMapModel, 468) 469from airbyte_cdk.sources.declarative.models.declarative_component_schema import ( 470 UnlimitedCallRatePolicy as UnlimitedCallRatePolicyModel, 471) 472from airbyte_cdk.sources.declarative.models.declarative_component_schema import ( 473 ValidateAdheresToSchema as ValidateAdheresToSchemaModel, 474) 475from airbyte_cdk.sources.declarative.models.declarative_component_schema import ValueType 476from airbyte_cdk.sources.declarative.models.declarative_component_schema import ( 477 WaitTimeFromHeader as WaitTimeFromHeaderModel, 478) 479from airbyte_cdk.sources.declarative.models.declarative_component_schema import ( 480 WaitUntilTimeFromHeader as WaitUntilTimeFromHeaderModel, 481) 482from airbyte_cdk.sources.declarative.models.declarative_component_schema import ( 483 XmlDecoder as XmlDecoderModel, 484) 485from airbyte_cdk.sources.declarative.models.declarative_component_schema import ( 486 ZipfileDecoder as ZipfileDecoderModel, 487) 488from airbyte_cdk.sources.declarative.parsers.custom_code_compiler import ( 489 INJECTED_MANIFEST, 490 AirbyteCustomCodeNotPermittedError, 491 custom_code_execution_permitted, 492) 493from airbyte_cdk.sources.declarative.partition_routers import ( 494 CartesianProductStreamSlicer, 495 GroupingPartitionRouter, 496 ListPartitionRouter, 497 PartitionRouter, 498 SinglePartitionRouter, 499 SubstreamPartitionRouter, 500) 501from airbyte_cdk.sources.declarative.partition_routers.async_job_partition_router import ( 502 AsyncJobPartitionRouter, 503) 504from airbyte_cdk.sources.declarative.partition_routers.substream_partition_router import ( 505 ParentStreamConfig, 506) 507from airbyte_cdk.sources.declarative.requesters import HttpRequester, RequestOption 508from airbyte_cdk.sources.declarative.requesters.error_handlers import ( 509 CompositeErrorHandler, 510 DefaultErrorHandler, 511 HttpResponseFilter, 512) 513from airbyte_cdk.sources.declarative.requesters.error_handlers.backoff_strategies import ( 514 ConstantBackoffStrategy, 515 ExponentialBackoffStrategy, 516 WaitTimeFromHeaderBackoffStrategy, 517 WaitUntilTimeFromHeaderBackoffStrategy, 518) 519from airbyte_cdk.sources.declarative.requesters.http_job_repository import AsyncHttpJobRepository 520from airbyte_cdk.sources.declarative.requesters.paginators import ( 521 DefaultPaginator, 522 NoPagination, 523 PaginatorTestReadDecorator, 524) 525from airbyte_cdk.sources.declarative.requesters.paginators.strategies import ( 526 CursorPaginationStrategy, 527 CursorStopCondition, 528 OffsetIncrement, 529 PageIncrement, 530 StopConditionPaginationStrategyDecorator, 531) 532from airbyte_cdk.sources.declarative.requesters.query_properties import ( 533 PropertiesFromEndpoint, 534 PropertyChunking, 535 QueryProperties, 536) 537from airbyte_cdk.sources.declarative.requesters.query_properties.property_chunking import ( 538 PropertyLimitType, 539) 540from airbyte_cdk.sources.declarative.requesters.query_properties.property_selector import ( 541 JsonSchemaPropertySelector, 542) 543from airbyte_cdk.sources.declarative.requesters.query_properties.strategies import ( 544 GroupByKey, 545) 546from airbyte_cdk.sources.declarative.requesters.request_option import RequestOptionType 547from airbyte_cdk.sources.declarative.requesters.request_options import ( 548 DatetimeBasedRequestOptionsProvider, 549 DefaultRequestOptionsProvider, 550 InterpolatedRequestOptionsProvider, 551 RequestOptionsProvider, 552) 553from airbyte_cdk.sources.declarative.requesters.request_options.per_partition_request_option_provider import ( 554 PerPartitionRequestOptionsProvider, 555) 556from airbyte_cdk.sources.declarative.requesters.request_path import RequestPath 557from airbyte_cdk.sources.declarative.requesters.requester import HttpMethod, Requester 558from airbyte_cdk.sources.declarative.resolvers import ( 559 ComponentMappingDefinition, 560 ConfigComponentsResolver, 561 HttpComponentsResolver, 562 ParametrizedComponentsResolver, 563 StreamConfig, 564 StreamParametersDefinition, 565) 566from airbyte_cdk.sources.declarative.retrievers import ( 567 AsyncRetriever, 568 LazySimpleRetriever, 569 SimpleRetriever, 570) 571from airbyte_cdk.sources.declarative.retrievers.file_uploader import ( 572 ConnectorBuilderFileUploader, 573 DefaultFileUploader, 574 FileUploader, 575 LocalFileSystemFileWriter, 576 NoopFileWriter, 577) 578from airbyte_cdk.sources.declarative.retrievers.pagination_tracker import PaginationTracker 579from airbyte_cdk.sources.declarative.schema import ( 580 ComplexFieldType, 581 DefaultSchemaLoader, 582 DynamicSchemaLoader, 583 InlineSchemaLoader, 584 JsonFileSchemaLoader, 585 SchemaLoader, 586 SchemaTypeIdentifier, 587 TypesMap, 588) 589from airbyte_cdk.sources.declarative.schema.caching_schema_loader_decorator import ( 590 CachingSchemaLoaderDecorator, 591) 592from airbyte_cdk.sources.declarative.schema.composite_schema_loader import CompositeSchemaLoader 593from airbyte_cdk.sources.declarative.spec import ConfigMigration, Spec 594from airbyte_cdk.sources.declarative.stream_slicers import ( 595 StreamSlicer, 596 StreamSlicerTestReadDecorator, 597) 598from airbyte_cdk.sources.declarative.stream_slicers.declarative_partition_generator import ( 599 DeclarativePartitionFactory, 600 StreamSlicerPartitionGenerator, 601) 602from airbyte_cdk.sources.declarative.transformations import ( 603 AddFields, 604 RecordTransformation, 605 RemoveFields, 606) 607from airbyte_cdk.sources.declarative.transformations.add_fields import AddedFieldDefinition 608from airbyte_cdk.sources.declarative.transformations.config_transformations import ( 609 ConfigAddFields, 610 ConfigRemapField, 611 ConfigRemoveFields, 612) 613from airbyte_cdk.sources.declarative.transformations.config_transformations.config_transformation import ( 614 ConfigTransformation, 615) 616from airbyte_cdk.sources.declarative.transformations.dpath_flatten_fields import ( 617 DpathFlattenFields, 618 KeyTransformation, 619) 620from airbyte_cdk.sources.declarative.transformations.flatten_fields import ( 621 FlattenFields, 622) 623from airbyte_cdk.sources.declarative.transformations.keys_replace_transformation import ( 624 KeysReplaceTransformation, 625) 626from airbyte_cdk.sources.declarative.transformations.keys_to_lower_transformation import ( 627 KeysToLowerTransformation, 628) 629from airbyte_cdk.sources.declarative.transformations.keys_to_snake_transformation import ( 630 KeysToSnakeCaseTransformation, 631) 632from airbyte_cdk.sources.declarative.validators import ( 633 DpathValidator, 634 PredicateValidator, 635 ValidateAdheresToSchema, 636) 637from airbyte_cdk.sources.http_logger import format_http_message 638from airbyte_cdk.sources.message import ( 639 InMemoryMessageRepository, 640 LogAppenderMessageRepositoryDecorator, 641 MessageRepository, 642 NoopMessageRepository, 643) 644from airbyte_cdk.sources.message.repository import StateFilteringMessageRepository 645from airbyte_cdk.sources.streams.call_rate import ( 646 APIBudget, 647 FixedWindowCallRatePolicy, 648 HttpAPIBudget, 649 HttpRequestRegexMatcher, 650 MovingWindowCallRatePolicy, 651 Rate, 652 UnlimitedCallRatePolicy, 653) 654from airbyte_cdk.sources.streams.concurrent.abstract_stream import AbstractStream 655from airbyte_cdk.sources.streams.concurrent.clamping import ( 656 ClampingEndProvider, 657 ClampingStrategy, 658 DayClampingStrategy, 659 MonthClampingStrategy, 660 NoClamping, 661 WeekClampingStrategy, 662 Weekday, 663) 664from airbyte_cdk.sources.streams.concurrent.cursor import ( 665 ConcurrentCursor, 666 Cursor, 667 CursorField, 668 FinalStateCursor, 669) 670from airbyte_cdk.sources.streams.concurrent.default_stream import DefaultStream 671from airbyte_cdk.sources.streams.concurrent.helpers import get_primary_key_from_stream 672from airbyte_cdk.sources.streams.concurrent.partitions.stream_slicer import ( 673 StreamSlicer as ConcurrentStreamSlicer, 674) 675from airbyte_cdk.sources.streams.concurrent.state_converters.datetime_stream_state_converter import ( 676 CustomFormatConcurrentStreamStateConverter, 677 DateTimeStreamStateConverter, 678) 679from airbyte_cdk.sources.streams.concurrent.state_converters.incrementing_count_stream_state_converter import ( 680 IncrementingCountStreamStateConverter, 681) 682from airbyte_cdk.sources.streams.http.error_handlers.response_models import ResponseAction 683from airbyte_cdk.sources.types import Config 684from airbyte_cdk.sources.utils.transform import TransformConfig, TypeTransformer 685 686ComponentDefinition = Mapping[str, Any] 687 688SCHEMA_TRANSFORMER_TYPE_MAPPING = { 689 SchemaNormalizationModel.None_: TransformConfig.NoTransform, 690 SchemaNormalizationModel.Default: TransformConfig.DefaultSchemaNormalization, 691} 692_NO_STREAM_SLICING = SinglePartitionRouter(parameters={}) 693 694# Ideally this should use the value defined in ConcurrentDeclarativeSource, but 695# this would be a circular import 696MAX_SLICES = 5 697 698LOGGER = logging.getLogger(f"airbyte.model_to_component_factory") 699 700 701class ModelToComponentFactory: 702 EPOCH_DATETIME_FORMAT = "%s" 703 704 def __init__( 705 self, 706 limit_pages_fetched_per_slice: Optional[int] = None, 707 limit_slices_fetched: Optional[int] = None, 708 emit_connector_builder_messages: bool = False, 709 disable_retries: bool = False, 710 disable_cache: bool = False, 711 message_repository: Optional[MessageRepository] = None, 712 connector_state_manager: Optional[ConnectorStateManager] = None, 713 max_concurrent_async_job_count: Optional[int] = None, 714 configured_catalog: Optional[ConfiguredAirbyteCatalog] = None, 715 api_budget: Optional[APIBudget] = None, 716 rate_limited_authenticators: Optional[ 717 Dict[str, RateLimitedMultipleTokenAuthenticator] 718 ] = None, 719 custom_components_trusted: bool = True, 720 ): 721 self._init_mappings() 722 self._custom_components_trusted = custom_components_trusted 723 self._limit_pages_fetched_per_slice = limit_pages_fetched_per_slice 724 self._limit_slices_fetched = limit_slices_fetched 725 self._emit_connector_builder_messages = emit_connector_builder_messages 726 self._disable_retries = disable_retries 727 self._disable_cache = disable_cache 728 self._message_repository = message_repository or InMemoryMessageRepository( 729 self._evaluate_log_level(emit_connector_builder_messages) 730 ) 731 self._stream_name_to_configured_stream = self._create_stream_name_to_configured_stream( 732 configured_catalog 733 ) 734 self._connector_state_manager = connector_state_manager or ConnectorStateManager() 735 self._api_budget: Optional[Union[APIBudget]] = api_budget 736 # Shared instances so all streams see the same token quota counters (like api_budget) 737 self._rate_limited_authenticators: Dict[str, RateLimitedMultipleTokenAuthenticator] = ( 738 rate_limited_authenticators if rate_limited_authenticators is not None else {} 739 ) 740 self._job_tracker: JobTracker = JobTracker(max_concurrent_async_job_count or 1) 741 # placeholder for deprecation warnings 742 self._collected_deprecation_logs: List[ConnectorBuilderLogMessage] = [] 743 744 def _init_mappings(self) -> None: 745 self.PYDANTIC_MODEL_TO_CONSTRUCTOR: Mapping[Type[BaseModel], Callable[..., Any]] = { 746 AddedFieldDefinitionModel: self.create_added_field_definition, 747 AddFieldsModel: self.create_add_fields, 748 ApiKeyAuthenticatorModel: self.create_api_key_authenticator, 749 BasicHttpAuthenticatorModel: self.create_basic_http_authenticator, 750 BearerAuthenticatorModel: self.create_bearer_authenticator, 751 CheckStreamModel: self.create_check_stream, 752 DynamicStreamCheckConfigModel: self.create_dynamic_stream_check_config, 753 CheckDynamicStreamModel: self.create_check_dynamic_stream, 754 CompositeErrorHandlerModel: self.create_composite_error_handler, 755 ConcurrencyLevelModel: self.create_concurrency_level, 756 ConfigMigrationModel: self.create_config_migration, 757 ConfigAddFieldsModel: self.create_config_add_fields, 758 ConfigRemapFieldModel: self.create_config_remap_field, 759 ConfigRemoveFieldsModel: self.create_config_remove_fields, 760 ConstantBackoffStrategyModel: self.create_constant_backoff_strategy, 761 CsvDecoderModel: self.create_csv_decoder, 762 CursorPaginationModel: self.create_cursor_pagination, 763 CustomAuthenticatorModel: self.create_custom_component, 764 CustomBackoffStrategyModel: self.create_custom_component, 765 CustomDecoderModel: self.create_custom_component, 766 CustomErrorHandlerModel: self.create_custom_component, 767 CustomRecordExtractorModel: self.create_custom_component, 768 CustomRecordFilterModel: self.create_custom_component, 769 CustomRequesterModel: self.create_custom_component, 770 CustomRetrieverModel: self.create_custom_component, 771 CustomSchemaLoader: self.create_custom_component, 772 CustomSchemaNormalizationModel: self.create_custom_component, 773 CustomStateMigration: self.create_custom_component, 774 CustomPaginationStrategyModel: self.create_custom_component, 775 CustomPartitionRouterModel: self.create_custom_component, 776 CustomTransformationModel: self.create_custom_component, 777 CustomValidationStrategyModel: self.create_custom_component, 778 CustomConfigTransformationModel: self.create_custom_component, 779 DeclarativeStreamModel: self.create_default_stream, 780 DefaultErrorHandlerModel: self.create_default_error_handler, 781 DefaultPaginatorModel: self.create_default_paginator, 782 DpathExtractorModel: self.create_dpath_extractor, 783 DpathValidatorModel: self.create_dpath_validator, 784 ResponseToFileExtractorModel: self.create_response_to_file_extractor, 785 ExponentialBackoffStrategyModel: self.create_exponential_backoff_strategy, 786 SessionTokenAuthenticatorModel: self.create_session_token_authenticator, 787 GroupByKeyMergeStrategyModel: self.create_group_by_key, 788 HttpRequesterModel: self.create_http_requester, 789 HttpResponseFilterModel: self.create_http_response_filter, 790 InlineSchemaLoaderModel: self.create_inline_schema_loader, 791 JsonDecoderModel: self.create_json_decoder, 792 JsonItemsDecoderModel: self.create_json_items_decoder, 793 JsonlDecoderModel: self.create_jsonl_decoder, 794 JsonSchemaPropertySelectorModel: self.create_json_schema_property_selector, 795 GzipDecoderModel: self.create_gzip_decoder, 796 KeysToLowerModel: self.create_keys_to_lower_transformation, 797 KeysToSnakeCaseModel: self.create_keys_to_snake_transformation, 798 KeysReplaceModel: self.create_keys_replace_transformation, 799 FlattenFieldsModel: self.create_flatten_fields, 800 DpathFlattenFieldsModel: self.create_dpath_flatten_fields, 801 IterableDecoderModel: self.create_iterable_decoder, 802 XmlDecoderModel: self.create_xml_decoder, 803 JsonFileSchemaLoaderModel: self.create_json_file_schema_loader, 804 DynamicSchemaLoaderModel: self.create_dynamic_schema_loader, 805 SchemaTypeIdentifierModel: self.create_schema_type_identifier, 806 TypesMapModel: self.create_types_map, 807 ComplexFieldTypeModel: self.create_complex_field_type, 808 JwtAuthenticatorModel: self.create_jwt_authenticator, 809 LegacyToPerPartitionStateMigrationModel: self.create_legacy_to_per_partition_state_migration, 810 ListPartitionRouterModel: self.create_list_partition_router, 811 MinMaxDatetimeModel: self.create_min_max_datetime, 812 NoAuthModel: self.create_no_auth, 813 NoPaginationModel: self.create_no_pagination, 814 OAuthAuthenticatorModel: self.create_oauth_authenticator, 815 OffsetIncrementModel: self.create_offset_increment, 816 PageIncrementModel: self.create_page_increment, 817 ParentStreamConfigModel: self.create_parent_stream_config_with_substream_wrapper, 818 PredicateValidatorModel: self.create_predicate_validator, 819 PropertiesFromEndpointModel: self.create_properties_from_endpoint, 820 PropertyChunkingModel: self.create_property_chunking, 821 QueryPropertiesModel: self.create_query_properties, 822 RecordExpanderModel: self.create_record_expander, 823 RecordFilterModel: self.create_record_filter, 824 RecordSelectorModel: self.create_record_selector, 825 RemoveFieldsModel: self.create_remove_fields, 826 RequestPathModel: self.create_request_path, 827 RequestOptionModel: self.create_request_option, 828 LegacySessionTokenAuthenticatorModel: self.create_legacy_session_token_authenticator, 829 SelectiveAuthenticatorModel: self.create_selective_authenticator, 830 SimpleRetrieverModel: self.create_simple_retriever, 831 StateDelegatingStreamModel: self.create_state_delegating_stream, 832 SpecModel: self.create_spec, 833 SubstreamPartitionRouterModel: self.create_substream_partition_router, 834 ValidateAdheresToSchemaModel: self.create_validate_adheres_to_schema, 835 WaitTimeFromHeaderModel: self.create_wait_time_from_header, 836 WaitUntilTimeFromHeaderModel: self.create_wait_until_time_from_header, 837 AsyncRetrieverModel: self.create_async_retriever, 838 HttpComponentsResolverModel: self.create_http_components_resolver, 839 ConfigComponentsResolverModel: self.create_config_components_resolver, 840 ParametrizedComponentsResolverModel: self.create_parametrized_components_resolver, 841 StreamConfigModel: self.create_stream_config, 842 ComponentMappingDefinitionModel: self.create_components_mapping_definition, 843 ZipfileDecoderModel: self.create_zipfile_decoder, 844 HTTPAPIBudgetModel: self.create_http_api_budget, 845 FileUploaderModel: self.create_file_uploader, 846 FixedWindowCallRatePolicyModel: self.create_fixed_window_call_rate_policy, 847 MovingWindowCallRatePolicyModel: self.create_moving_window_call_rate_policy, 848 UnlimitedCallRatePolicyModel: self.create_unlimited_call_rate_policy, 849 RateModel: self.create_rate, 850 HttpRequestRegexMatcherModel: self.create_http_request_matcher, 851 RateLimitedMultipleTokenAuthenticatorModel: self.create_rate_limited_multiple_token_authenticator, 852 GroupingPartitionRouterModel: self.create_grouping_partition_router, 853 } 854 855 # Needed for the case where we need to perform a second parse on the fields of a custom component 856 self.TYPE_NAME_TO_MODEL = {cls.__name__: cls for cls in self.PYDANTIC_MODEL_TO_CONSTRUCTOR} 857 858 @staticmethod 859 def _create_stream_name_to_configured_stream( 860 configured_catalog: Optional[ConfiguredAirbyteCatalog], 861 ) -> Mapping[str, ConfiguredAirbyteStream]: 862 return ( 863 {stream.stream.name: stream for stream in configured_catalog.streams} 864 if configured_catalog 865 else {} 866 ) 867 868 def create_component( 869 self, 870 model_type: Type[BaseModel], 871 component_definition: ComponentDefinition, 872 config: Config, 873 **kwargs: Any, 874 ) -> Any: 875 """ 876 Takes a given Pydantic model type and Mapping representing a component definition and creates a declarative component and 877 subcomponents which will be used at runtime. This is done by first parsing the mapping into a Pydantic model and then creating 878 creating declarative components from that model. 879 880 :param model_type: The type of declarative component that is being initialized 881 :param component_definition: The mapping that represents a declarative component 882 :param config: The connector config that is provided by the customer 883 :return: The declarative component to be used at runtime 884 """ 885 886 component_type = component_definition.get("type") 887 if component_definition.get("type") != model_type.__name__: 888 raise ValueError( 889 f"Expected manifest component of type {model_type.__name__}, but received {component_type} instead" 890 ) 891 892 declarative_component_model = model_type.parse_obj(component_definition) 893 894 if not isinstance(declarative_component_model, model_type): 895 raise ValueError( 896 f"Expected {model_type.__name__} component, but received {declarative_component_model.__class__.__name__}" 897 ) 898 899 return self._create_component_from_model( 900 model=declarative_component_model, config=config, **kwargs 901 ) 902 903 def _create_component_from_model(self, model: BaseModel, config: Config, **kwargs: Any) -> Any: 904 if model.__class__ not in self.PYDANTIC_MODEL_TO_CONSTRUCTOR: 905 raise ValueError( 906 f"{model.__class__} with attributes {model} is not a valid component type" 907 ) 908 component_constructor = self.PYDANTIC_MODEL_TO_CONSTRUCTOR.get(model.__class__) 909 if not component_constructor: 910 raise ValueError(f"Could not find constructor for {model.__class__}") 911 912 # collect deprecation warnings for supported models. 913 if isinstance(model, BaseModelWithDeprecations): 914 self._collect_model_deprecations(model) 915 916 return component_constructor(model=model, config=config, **kwargs) 917 918 def get_model_deprecations(self) -> List[ConnectorBuilderLogMessage]: 919 """ 920 Returns the deprecation warnings that were collected during the creation of components. 921 """ 922 return self._collected_deprecation_logs 923 924 def _collect_model_deprecations(self, model: BaseModelWithDeprecations) -> None: 925 """ 926 Collects deprecation logs from the given model and appends any new logs to the internal collection. 927 928 This method checks if the provided model has deprecation logs (identified by the presence of the DEPRECATION_LOGS_TAG attribute and a non-None `_deprecation_logs` property). It iterates through each deprecation log in the model and appends it to the `_collected_deprecation_logs` list if it has not already been collected, ensuring that duplicate logs are avoided. 929 930 Args: 931 model (BaseModelWithDeprecations): The model instance from which to collect deprecation logs. 932 """ 933 if hasattr(model, DEPRECATION_LOGS_TAG) and model._deprecation_logs is not None: 934 for log in model._deprecation_logs: 935 # avoid duplicates for deprecation logs observed. 936 if log not in self._collected_deprecation_logs: 937 self._collected_deprecation_logs.append(log) 938 939 def create_config_migration( 940 self, model: ConfigMigrationModel, config: Config 941 ) -> ConfigMigration: 942 transformations: List[ConfigTransformation] = [ 943 self._create_component_from_model(transformation, config) 944 for transformation in model.transformations 945 ] 946 947 return ConfigMigration( 948 description=model.description, 949 transformations=transformations, 950 ) 951 952 def create_config_add_fields( 953 self, model: ConfigAddFieldsModel, config: Config, **kwargs: Any 954 ) -> ConfigAddFields: 955 fields = [self._create_component_from_model(field, config) for field in model.fields] 956 return ConfigAddFields( 957 fields=fields, 958 condition=model.condition or "", 959 ) 960 961 @staticmethod 962 def create_config_remove_fields( 963 model: ConfigRemoveFieldsModel, config: Config, **kwargs: Any 964 ) -> ConfigRemoveFields: 965 return ConfigRemoveFields( 966 field_pointers=model.field_pointers, 967 condition=model.condition or "", 968 ) 969 970 @staticmethod 971 def create_config_remap_field( 972 model: ConfigRemapFieldModel, config: Config, **kwargs: Any 973 ) -> ConfigRemapField: 974 mapping = cast(Mapping[str, Any], model.map) 975 return ConfigRemapField( 976 map=mapping, 977 field_path=model.field_path, 978 config=config, 979 ) 980 981 def create_dpath_validator(self, model: DpathValidatorModel, config: Config) -> DpathValidator: 982 strategy = self._create_component_from_model(model.validation_strategy, config) 983 984 return DpathValidator( 985 field_path=model.field_path, 986 strategy=strategy, 987 ) 988 989 def create_predicate_validator( 990 self, model: PredicateValidatorModel, config: Config 991 ) -> PredicateValidator: 992 strategy = self._create_component_from_model(model.validation_strategy, config) 993 994 return PredicateValidator( 995 value=model.value, 996 strategy=strategy, 997 ) 998 999 @staticmethod 1000 def create_validate_adheres_to_schema( 1001 model: ValidateAdheresToSchemaModel, config: Config, **kwargs: Any 1002 ) -> ValidateAdheresToSchema: 1003 base_schema = cast(Mapping[str, Any], model.base_schema) 1004 return ValidateAdheresToSchema( 1005 schema=base_schema, 1006 ) 1007 1008 @staticmethod 1009 def create_added_field_definition( 1010 model: AddedFieldDefinitionModel, config: Config, **kwargs: Any 1011 ) -> AddedFieldDefinition: 1012 interpolated_value = InterpolatedString.create( 1013 model.value, parameters=model.parameters or {} 1014 ) 1015 return AddedFieldDefinition( 1016 path=model.path, 1017 value=interpolated_value, 1018 value_type=ModelToComponentFactory._json_schema_type_name_to_type(model.value_type), 1019 parameters=model.parameters or {}, 1020 ) 1021 1022 def create_add_fields(self, model: AddFieldsModel, config: Config, **kwargs: Any) -> AddFields: 1023 added_field_definitions = [ 1024 self._create_component_from_model( 1025 model=added_field_definition_model, 1026 value_type=ModelToComponentFactory._json_schema_type_name_to_type( 1027 added_field_definition_model.value_type 1028 ), 1029 config=config, 1030 ) 1031 for added_field_definition_model in model.fields 1032 ] 1033 return AddFields( 1034 fields=added_field_definitions, 1035 condition=model.condition or "", 1036 parameters=model.parameters or {}, 1037 ) 1038 1039 def create_keys_to_lower_transformation( 1040 self, model: KeysToLowerModel, config: Config, **kwargs: Any 1041 ) -> KeysToLowerTransformation: 1042 return KeysToLowerTransformation() 1043 1044 def create_keys_to_snake_transformation( 1045 self, model: KeysToSnakeCaseModel, config: Config, **kwargs: Any 1046 ) -> KeysToSnakeCaseTransformation: 1047 return KeysToSnakeCaseTransformation() 1048 1049 def create_keys_replace_transformation( 1050 self, model: KeysReplaceModel, config: Config, **kwargs: Any 1051 ) -> KeysReplaceTransformation: 1052 return KeysReplaceTransformation( 1053 old=model.old, new=model.new, parameters=model.parameters or {} 1054 ) 1055 1056 def create_flatten_fields( 1057 self, model: FlattenFieldsModel, config: Config, **kwargs: Any 1058 ) -> FlattenFields: 1059 return FlattenFields( 1060 flatten_lists=model.flatten_lists if model.flatten_lists is not None else True 1061 ) 1062 1063 def create_dpath_flatten_fields( 1064 self, model: DpathFlattenFieldsModel, config: Config, **kwargs: Any 1065 ) -> DpathFlattenFields: 1066 model_field_path: List[Union[InterpolatedString, str]] = [x for x in model.field_path] 1067 key_transformation = ( 1068 KeyTransformation( 1069 config=config, 1070 prefix=model.key_transformation.prefix, 1071 suffix=model.key_transformation.suffix, 1072 parameters=model.parameters or {}, 1073 ) 1074 if model.key_transformation is not None 1075 else None 1076 ) 1077 return DpathFlattenFields( 1078 config=config, 1079 field_path=model_field_path, 1080 delete_origin_value=model.delete_origin_value 1081 if model.delete_origin_value is not None 1082 else False, 1083 replace_record=model.replace_record if model.replace_record is not None else False, 1084 key_transformation=key_transformation, 1085 parameters=model.parameters or {}, 1086 ) 1087 1088 @staticmethod 1089 def _json_schema_type_name_to_type(value_type: Optional[ValueType]) -> Optional[Type[Any]]: 1090 if not value_type: 1091 return None 1092 names_to_types = { 1093 ValueType.string: str, 1094 ValueType.number: float, 1095 ValueType.integer: int, 1096 ValueType.boolean: bool, 1097 } 1098 return names_to_types[value_type] 1099 1100 def create_api_key_authenticator( 1101 self, 1102 model: ApiKeyAuthenticatorModel, 1103 config: Config, 1104 token_provider: Optional[TokenProvider] = None, 1105 **kwargs: Any, 1106 ) -> ApiKeyAuthenticator: 1107 if model.inject_into is None and model.header is None: 1108 raise ValueError( 1109 "Expected either inject_into or header to be set for ApiKeyAuthenticator" 1110 ) 1111 1112 if model.inject_into is not None and model.header is not None: 1113 raise ValueError( 1114 "inject_into and header cannot be set both for ApiKeyAuthenticator - remove the deprecated header option" 1115 ) 1116 1117 if token_provider is not None and model.api_token != "": 1118 raise ValueError( 1119 "If token_provider is set, api_token is ignored and has to be set to empty string." 1120 ) 1121 1122 request_option = ( 1123 self._create_component_from_model( 1124 model.inject_into, config, parameters=model.parameters or {} 1125 ) 1126 if model.inject_into 1127 else RequestOption( 1128 inject_into=RequestOptionType.header, 1129 field_name=model.header or "", 1130 parameters=model.parameters or {}, 1131 ) 1132 ) 1133 1134 return ApiKeyAuthenticator( 1135 token_provider=( 1136 token_provider 1137 if token_provider is not None 1138 else InterpolatedStringTokenProvider( 1139 api_token=model.api_token or "", 1140 config=config, 1141 parameters=model.parameters or {}, 1142 ) 1143 ), 1144 request_option=request_option, 1145 config=config, 1146 parameters=model.parameters or {}, 1147 ) 1148 1149 def create_legacy_to_per_partition_state_migration( 1150 self, 1151 model: LegacyToPerPartitionStateMigrationModel, 1152 config: Mapping[str, Any], 1153 declarative_stream: DeclarativeStreamModel, 1154 ) -> LegacyToPerPartitionStateMigration: 1155 retriever = declarative_stream.retriever 1156 if not isinstance(retriever, (SimpleRetrieverModel, AsyncRetrieverModel)): 1157 raise ValueError( 1158 f"LegacyToPerPartitionStateMigrations can only be applied on a DeclarativeStream with a SimpleRetriever or AsyncRetriever. Got {type(retriever)}" 1159 ) 1160 partition_router = retriever.partition_router 1161 if not isinstance( 1162 partition_router, (SubstreamPartitionRouterModel, CustomPartitionRouterModel) 1163 ): 1164 raise ValueError( 1165 f"LegacyToPerPartitionStateMigrations can only be applied on a SimpleRetriever with a Substream partition router. Got {type(partition_router)}" 1166 ) 1167 if not hasattr(partition_router, "parent_stream_configs"): 1168 raise ValueError( 1169 "LegacyToPerPartitionStateMigrations can only be applied with a parent stream configuration." 1170 ) 1171 1172 if not hasattr(declarative_stream, "incremental_sync"): 1173 raise ValueError( 1174 "LegacyToPerPartitionStateMigrations can only be applied with an incremental_sync configuration." 1175 ) 1176 1177 return LegacyToPerPartitionStateMigration( 1178 partition_router, # type: ignore # was already checked above 1179 declarative_stream.incremental_sync, # type: ignore # was already checked. Migration can be applied only to incremental streams. 1180 config, 1181 declarative_stream.parameters, # type: ignore # different type is expected here Mapping[str, Any], got Dict[str, Any] 1182 ) 1183 1184 def create_session_token_authenticator( 1185 self, model: SessionTokenAuthenticatorModel, config: Config, name: str, **kwargs: Any 1186 ) -> Union[ApiKeyAuthenticator, BearerAuthenticator]: 1187 decoder = ( 1188 self._create_component_from_model(model=model.decoder, config=config) 1189 if model.decoder 1190 else JsonDecoder(parameters={}) 1191 ) 1192 login_requester = self._create_component_from_model( 1193 model=model.login_requester, 1194 config=config, 1195 name=f"{name}_login_requester", 1196 decoder=decoder, 1197 ) 1198 token_provider = SessionTokenProvider( 1199 login_requester=login_requester, 1200 session_token_path=model.session_token_path, 1201 expiration_duration=parse_duration(model.expiration_duration) 1202 if model.expiration_duration 1203 else None, 1204 parameters=model.parameters or {}, 1205 message_repository=self._message_repository, 1206 decoder=decoder, 1207 ) 1208 if model.request_authentication.type == "Bearer": 1209 return ModelToComponentFactory.create_bearer_authenticator( 1210 BearerAuthenticatorModel(type="BearerAuthenticator", api_token=""), # type: ignore # $parameters has a default value 1211 config, 1212 token_provider=token_provider, 1213 ) 1214 else: 1215 # Get the api_token template if specified, default to just the session token 1216 api_token_template = ( 1217 getattr(model.request_authentication, "api_token", None) or "{{ session_token }}" 1218 ) 1219 final_token_provider: TokenProvider = InterpolatedSessionTokenProvider( 1220 config=config, 1221 api_token=api_token_template, 1222 session_token_provider=token_provider, 1223 parameters=model.parameters or {}, 1224 ) 1225 return self.create_api_key_authenticator( 1226 ApiKeyAuthenticatorModel( 1227 type="ApiKeyAuthenticator", 1228 api_token="", 1229 inject_into=model.request_authentication.inject_into, 1230 ), # type: ignore # $parameters and headers default to None 1231 config=config, 1232 token_provider=final_token_provider, 1233 ) 1234 1235 @staticmethod 1236 def create_basic_http_authenticator( 1237 model: BasicHttpAuthenticatorModel, config: Config, **kwargs: Any 1238 ) -> BasicHttpAuthenticator: 1239 return BasicHttpAuthenticator( 1240 password=model.password or "", 1241 username=model.username, 1242 config=config, 1243 parameters=model.parameters or {}, 1244 ) 1245 1246 @staticmethod 1247 def create_bearer_authenticator( 1248 model: BearerAuthenticatorModel, 1249 config: Config, 1250 token_provider: Optional[TokenProvider] = None, 1251 **kwargs: Any, 1252 ) -> BearerAuthenticator: 1253 if token_provider is not None and model.api_token != "": 1254 raise ValueError( 1255 "If token_provider is set, api_token is ignored and has to be set to empty string." 1256 ) 1257 return BearerAuthenticator( 1258 token_provider=( 1259 token_provider 1260 if token_provider is not None 1261 else InterpolatedStringTokenProvider( 1262 api_token=model.api_token or "", 1263 config=config, 1264 parameters=model.parameters or {}, 1265 ) 1266 ), 1267 config=config, 1268 parameters=model.parameters or {}, 1269 ) 1270 1271 @staticmethod 1272 def create_dynamic_stream_check_config( 1273 model: DynamicStreamCheckConfigModel, config: Config, **kwargs: Any 1274 ) -> DynamicStreamCheckConfig: 1275 return DynamicStreamCheckConfig( 1276 dynamic_stream_name=model.dynamic_stream_name, 1277 stream_count=model.stream_count, 1278 ) 1279 1280 def create_check_stream( 1281 self, model: CheckStreamModel, config: Config, **kwargs: Any 1282 ) -> CheckStream: 1283 if model.dynamic_streams_check_configs is None and model.stream_names is None: 1284 raise ValueError( 1285 "Expected either stream_names or dynamic_streams_check_configs to be set for CheckStream" 1286 ) 1287 1288 dynamic_streams_check_configs = ( 1289 [ 1290 self._create_component_from_model(model=dynamic_stream_check_config, config=config) 1291 for dynamic_stream_check_config in model.dynamic_streams_check_configs 1292 ] 1293 if model.dynamic_streams_check_configs 1294 else [] 1295 ) 1296 1297 return CheckStream( 1298 stream_names=model.stream_names or [], 1299 dynamic_streams_check_configs=dynamic_streams_check_configs, 1300 parameters={}, 1301 ) 1302 1303 @staticmethod 1304 def create_check_dynamic_stream( 1305 model: CheckDynamicStreamModel, config: Config, **kwargs: Any 1306 ) -> CheckDynamicStream: 1307 assert model.use_check_availability is not None # for mypy 1308 1309 use_check_availability = model.use_check_availability 1310 1311 return CheckDynamicStream( 1312 stream_count=model.stream_count, 1313 use_check_availability=use_check_availability, 1314 parameters={}, 1315 ) 1316 1317 def create_composite_error_handler( 1318 self, model: CompositeErrorHandlerModel, config: Config, **kwargs: Any 1319 ) -> CompositeErrorHandler: 1320 error_handlers = [ 1321 self._create_component_from_model(model=error_handler_model, config=config) 1322 for error_handler_model in model.error_handlers 1323 ] 1324 return CompositeErrorHandler( 1325 error_handlers=error_handlers, parameters=model.parameters or {} 1326 ) 1327 1328 @staticmethod 1329 def create_concurrency_level( 1330 model: ConcurrencyLevelModel, config: Config, **kwargs: Any 1331 ) -> ConcurrencyLevel: 1332 return ConcurrencyLevel( 1333 default_concurrency=model.default_concurrency, 1334 max_concurrency=model.max_concurrency, 1335 config=config, 1336 parameters={}, 1337 ) 1338 1339 @staticmethod 1340 def apply_stream_state_migrations( 1341 stream_state_migrations: List[Any] | None, stream_state: MutableMapping[str, Any] 1342 ) -> MutableMapping[str, Any]: 1343 if stream_state_migrations: 1344 for state_migration in stream_state_migrations: 1345 if state_migration.should_migrate(stream_state): 1346 # The state variable is expected to be mutable but the migrate method returns an immutable mapping. 1347 stream_state = dict(state_migration.migrate(stream_state)) 1348 return stream_state 1349 1350 def create_concurrent_cursor_from_datetime_based_cursor( 1351 self, 1352 model_type: Type[BaseModel], 1353 component_definition: ComponentDefinition, 1354 stream_name: str, 1355 stream_namespace: Optional[str], 1356 stream_state: MutableMapping[str, Any], 1357 config: Config, 1358 message_repository: Optional[MessageRepository] = None, 1359 runtime_lookback_window: Optional[datetime.timedelta] = None, 1360 **kwargs: Any, 1361 ) -> ConcurrentCursor: 1362 component_type = component_definition.get("type") 1363 if component_definition.get("type") != model_type.__name__: 1364 raise ValueError( 1365 f"Expected manifest component of type {model_type.__name__}, but received {component_type} instead" 1366 ) 1367 1368 # FIXME the interfaces of the concurrent cursor are kind of annoying as they take a `ComponentDefinition` instead of the actual model. This was done because the ConcurrentDeclarativeSource didn't have access to the models [here for example](https://github.com/airbytehq/airbyte-python-cdk/blob/f525803b3fec9329e4cc8478996a92bf884bfde9/airbyte_cdk/sources/declarative/concurrent_declarative_source.py#L354C54-L354C91). So now we have two cases: 1369 # * The ComponentDefinition comes from model.__dict__ in which case we have `parameters` 1370 # * The ComponentDefinition comes from the manifest as a dict in which case we have `$parameters` 1371 # We should change those interfaces to use the model once we clean up the code in CDS at which point the parameter propagation should happen as part of the ModelToComponentFactory. 1372 if "$parameters" not in component_definition and "parameters" in component_definition: 1373 component_definition["$parameters"] = component_definition.get("parameters") # type: ignore # This is a dict 1374 datetime_based_cursor_model = model_type.parse_obj(component_definition) 1375 1376 if not isinstance(datetime_based_cursor_model, DatetimeBasedCursorModel): 1377 raise ValueError( 1378 f"Expected {model_type.__name__} component, but received {datetime_based_cursor_model.__class__.__name__}" 1379 ) 1380 1381 model_parameters = datetime_based_cursor_model.parameters or {} 1382 1383 cursor_field = self._get_catalog_defined_cursor_field( 1384 stream_name=stream_name, 1385 allow_catalog_defined_cursor_field=datetime_based_cursor_model.allow_catalog_defined_cursor_field 1386 or False, 1387 ) 1388 1389 if not cursor_field: 1390 interpolated_cursor_field = InterpolatedString.create( 1391 datetime_based_cursor_model.cursor_field, 1392 parameters=model_parameters, 1393 ) 1394 cursor_field = CursorField( 1395 cursor_field_key=interpolated_cursor_field.eval(config=config), 1396 supports_catalog_defined_cursor_field=datetime_based_cursor_model.allow_catalog_defined_cursor_field 1397 or False, 1398 ) 1399 1400 interpolated_partition_field_start = InterpolatedString.create( 1401 datetime_based_cursor_model.partition_field_start or "start_time", 1402 parameters=model_parameters, 1403 ) 1404 interpolated_partition_field_end = InterpolatedString.create( 1405 datetime_based_cursor_model.partition_field_end or "end_time", 1406 parameters=model_parameters, 1407 ) 1408 1409 slice_boundary_fields = ( 1410 interpolated_partition_field_start.eval(config=config), 1411 interpolated_partition_field_end.eval(config=config), 1412 ) 1413 1414 datetime_format = datetime_based_cursor_model.datetime_format 1415 1416 cursor_granularity = ( 1417 parse_duration(datetime_based_cursor_model.cursor_granularity) 1418 if datetime_based_cursor_model.cursor_granularity 1419 else None 1420 ) 1421 1422 lookback_window = None 1423 interpolated_lookback_window = ( 1424 InterpolatedString.create( 1425 datetime_based_cursor_model.lookback_window, 1426 parameters=model_parameters, 1427 ) 1428 if datetime_based_cursor_model.lookback_window 1429 else None 1430 ) 1431 if interpolated_lookback_window: 1432 evaluated_lookback_window = interpolated_lookback_window.eval(config=config) 1433 if evaluated_lookback_window: 1434 lookback_window = parse_duration(evaluated_lookback_window) 1435 1436 connector_state_converter: DateTimeStreamStateConverter 1437 connector_state_converter = CustomFormatConcurrentStreamStateConverter( 1438 datetime_format=datetime_format, 1439 input_datetime_formats=datetime_based_cursor_model.cursor_datetime_formats, 1440 is_sequential_state=True, # ConcurrentPerPartitionCursor only works with sequential state 1441 cursor_granularity=cursor_granularity, 1442 ) 1443 1444 # Adjusts the stream state by applying the runtime lookback window. 1445 # This is used to ensure correct state handling in case of failed partitions. 1446 stream_state_value = stream_state.get(cursor_field.cursor_field_key) 1447 if runtime_lookback_window and stream_state_value: 1448 new_stream_state = ( 1449 connector_state_converter.parse_timestamp(stream_state_value) 1450 - runtime_lookback_window 1451 ) 1452 stream_state[cursor_field.cursor_field_key] = connector_state_converter.output_format( 1453 new_stream_state 1454 ) 1455 1456 start_date_runtime_value: Union[InterpolatedString, str, MinMaxDatetime] 1457 if isinstance(datetime_based_cursor_model.start_datetime, MinMaxDatetimeModel): 1458 start_date_runtime_value = self.create_min_max_datetime( 1459 model=datetime_based_cursor_model.start_datetime, config=config 1460 ) 1461 else: 1462 start_date_runtime_value = datetime_based_cursor_model.start_datetime 1463 1464 end_date_runtime_value: Optional[Union[InterpolatedString, str, MinMaxDatetime]] 1465 if isinstance(datetime_based_cursor_model.end_datetime, MinMaxDatetimeModel): 1466 end_date_runtime_value = self.create_min_max_datetime( 1467 model=datetime_based_cursor_model.end_datetime, config=config 1468 ) 1469 else: 1470 end_date_runtime_value = datetime_based_cursor_model.end_datetime 1471 1472 interpolated_start_date = MinMaxDatetime.create( 1473 interpolated_string_or_min_max_datetime=start_date_runtime_value, 1474 parameters=datetime_based_cursor_model.parameters, 1475 ) 1476 interpolated_end_date = ( 1477 None 1478 if not end_date_runtime_value 1479 else MinMaxDatetime.create( 1480 end_date_runtime_value, datetime_based_cursor_model.parameters 1481 ) 1482 ) 1483 1484 # If datetime format is not specified then start/end datetime should inherit it from the stream slicer 1485 if not interpolated_start_date.datetime_format: 1486 interpolated_start_date.datetime_format = datetime_format 1487 if interpolated_end_date and not interpolated_end_date.datetime_format: 1488 interpolated_end_date.datetime_format = datetime_format 1489 1490 start_date = interpolated_start_date.get_datetime(config=config) 1491 end_date_provider = ( 1492 partial(interpolated_end_date.get_datetime, config) 1493 if interpolated_end_date 1494 else connector_state_converter.get_end_provider() 1495 ) 1496 1497 if ( 1498 datetime_based_cursor_model.step and not datetime_based_cursor_model.cursor_granularity 1499 ) or ( 1500 not datetime_based_cursor_model.step and datetime_based_cursor_model.cursor_granularity 1501 ): 1502 raise ValueError( 1503 f"If step is defined, cursor_granularity should be as well and vice-versa. " 1504 f"Right now, step is `{datetime_based_cursor_model.step}` and cursor_granularity is `{datetime_based_cursor_model.cursor_granularity}`" 1505 ) 1506 1507 # When step is not defined, default to a step size from the starting date to the present moment 1508 step_length = datetime.timedelta.max 1509 interpolated_step = ( 1510 InterpolatedString.create( 1511 datetime_based_cursor_model.step, 1512 parameters=model_parameters, 1513 ) 1514 if datetime_based_cursor_model.step 1515 else None 1516 ) 1517 if interpolated_step: 1518 evaluated_step = interpolated_step.eval(config) 1519 if evaluated_step: 1520 step_length = parse_duration(evaluated_step) 1521 1522 clamping_strategy: ClampingStrategy = NoClamping() 1523 if datetime_based_cursor_model.clamping: 1524 # While it is undesirable to interpolate within the model factory (as opposed to at runtime), 1525 # it is still better than shifting interpolation low-code concept into the ConcurrentCursor runtime 1526 # object which we want to keep agnostic of being low-code 1527 target = InterpolatedString( 1528 string=datetime_based_cursor_model.clamping.target, 1529 parameters=model_parameters, 1530 ) 1531 evaluated_target = target.eval(config=config) 1532 match evaluated_target: 1533 case "DAY": 1534 clamping_strategy = DayClampingStrategy() 1535 end_date_provider = ClampingEndProvider( 1536 DayClampingStrategy(is_ceiling=False), 1537 end_date_provider, # type: ignore # Having issues w/ inspection for GapType and CursorValueType as shown in existing tests. Confirmed functionality is working in practice 1538 granularity=cursor_granularity or datetime.timedelta(seconds=1), 1539 ) 1540 case "WEEK": 1541 if ( 1542 not datetime_based_cursor_model.clamping.target_details 1543 or "weekday" not in datetime_based_cursor_model.clamping.target_details 1544 ): 1545 raise ValueError( 1546 "Given WEEK clamping, weekday needs to be provided as target_details" 1547 ) 1548 weekday = self._assemble_weekday( 1549 datetime_based_cursor_model.clamping.target_details["weekday"] 1550 ) 1551 clamping_strategy = WeekClampingStrategy(weekday) 1552 end_date_provider = ClampingEndProvider( 1553 WeekClampingStrategy(weekday, is_ceiling=False), 1554 end_date_provider, # type: ignore # Having issues w/ inspection for GapType and CursorValueType as shown in existing tests. Confirmed functionality is working in practice 1555 granularity=cursor_granularity or datetime.timedelta(days=1), 1556 ) 1557 case "MONTH": 1558 clamping_strategy = MonthClampingStrategy() 1559 end_date_provider = ClampingEndProvider( 1560 MonthClampingStrategy(is_ceiling=False), 1561 end_date_provider, # type: ignore # Having issues w/ inspection for GapType and CursorValueType as shown in existing tests. Confirmed functionality is working in practice 1562 granularity=cursor_granularity or datetime.timedelta(days=1), 1563 ) 1564 case _: 1565 raise ValueError( 1566 f"Invalid clamping target {evaluated_target}, expected DAY, WEEK, MONTH" 1567 ) 1568 1569 return ConcurrentCursor( 1570 stream_name=stream_name, 1571 stream_namespace=stream_namespace, 1572 stream_state=stream_state, 1573 message_repository=message_repository or self._message_repository, 1574 connector_state_manager=self._connector_state_manager, 1575 connector_state_converter=connector_state_converter, 1576 cursor_field=cursor_field, 1577 slice_boundary_fields=slice_boundary_fields, 1578 start=start_date, # type: ignore # Having issues w/ inspection for GapType and CursorValueType as shown in existing tests. Confirmed functionality is working in practice 1579 end_provider=end_date_provider, # type: ignore # Having issues w/ inspection for GapType and CursorValueType as shown in existing tests. Confirmed functionality is working in practice 1580 lookback_window=lookback_window, 1581 slice_range=step_length, 1582 cursor_granularity=cursor_granularity, 1583 clamping_strategy=clamping_strategy, 1584 ) 1585 1586 def create_concurrent_cursor_from_incrementing_count_cursor( 1587 self, 1588 model_type: Type[BaseModel], 1589 component_definition: ComponentDefinition, 1590 stream_name: str, 1591 stream_namespace: Optional[str], 1592 stream_state: MutableMapping[str, Any], 1593 config: Config, 1594 message_repository: Optional[MessageRepository] = None, 1595 **kwargs: Any, 1596 ) -> ConcurrentCursor: 1597 component_type = component_definition.get("type") 1598 if component_definition.get("type") != model_type.__name__: 1599 raise ValueError( 1600 f"Expected manifest component of type {model_type.__name__}, but received {component_type} instead" 1601 ) 1602 1603 incrementing_count_cursor_model = model_type.parse_obj(component_definition) 1604 1605 if not isinstance(incrementing_count_cursor_model, IncrementingCountCursorModel): 1606 raise ValueError( 1607 f"Expected {model_type.__name__} component, but received {incrementing_count_cursor_model.__class__.__name__}" 1608 ) 1609 1610 start_value: Union[int, str, None] = incrementing_count_cursor_model.start_value 1611 # Pydantic Union type coercion can convert int 0 to string '0' depending on Union order. 1612 # We need to handle both int and str representations of numeric values. 1613 # Evaluate the InterpolatedString and convert to int for the ConcurrentCursor. 1614 if start_value is not None: 1615 interpolated_start_value = InterpolatedString.create( 1616 str(start_value), # Ensure we pass a string to InterpolatedString.create 1617 parameters=incrementing_count_cursor_model.parameters or {}, 1618 ) 1619 evaluated_start_value: int = int(interpolated_start_value.eval(config=config)) 1620 else: 1621 evaluated_start_value = 0 1622 1623 cursor_field = self._get_catalog_defined_cursor_field( 1624 stream_name=stream_name, 1625 allow_catalog_defined_cursor_field=incrementing_count_cursor_model.allow_catalog_defined_cursor_field 1626 or False, 1627 ) 1628 1629 if not cursor_field: 1630 interpolated_cursor_field = InterpolatedString.create( 1631 incrementing_count_cursor_model.cursor_field, 1632 parameters=incrementing_count_cursor_model.parameters or {}, 1633 ) 1634 cursor_field = CursorField( 1635 cursor_field_key=interpolated_cursor_field.eval(config=config), 1636 supports_catalog_defined_cursor_field=incrementing_count_cursor_model.allow_catalog_defined_cursor_field 1637 or False, 1638 ) 1639 1640 connector_state_converter = IncrementingCountStreamStateConverter( 1641 is_sequential_state=True, # ConcurrentPerPartitionCursor only works with sequential state 1642 ) 1643 1644 return ConcurrentCursor( 1645 stream_name=stream_name, 1646 stream_namespace=stream_namespace, 1647 stream_state=stream_state, 1648 message_repository=message_repository or self._message_repository, 1649 connector_state_manager=self._connector_state_manager, 1650 connector_state_converter=connector_state_converter, 1651 cursor_field=cursor_field, 1652 slice_boundary_fields=None, 1653 start=evaluated_start_value, # type: ignore # Having issues w/ inspection for GapType and CursorValueType as shown in existing tests. Confirmed functionality is working in practice 1654 end_provider=connector_state_converter.get_end_provider(), # type: ignore # Having issues w/ inspection for GapType and CursorValueType as shown in existing tests. Confirmed functionality is working in practice 1655 ) 1656 1657 def _assemble_weekday(self, weekday: str) -> Weekday: 1658 match weekday: 1659 case "MONDAY": 1660 return Weekday.MONDAY 1661 case "TUESDAY": 1662 return Weekday.TUESDAY 1663 case "WEDNESDAY": 1664 return Weekday.WEDNESDAY 1665 case "THURSDAY": 1666 return Weekday.THURSDAY 1667 case "FRIDAY": 1668 return Weekday.FRIDAY 1669 case "SATURDAY": 1670 return Weekday.SATURDAY 1671 case "SUNDAY": 1672 return Weekday.SUNDAY 1673 case _: 1674 raise ValueError(f"Unknown weekday {weekday}") 1675 1676 def create_concurrent_cursor_from_perpartition_cursor( 1677 self, 1678 state_manager: ConnectorStateManager, 1679 model_type: Type[BaseModel], 1680 component_definition: ComponentDefinition, 1681 stream_name: str, 1682 stream_namespace: Optional[str], 1683 config: Config, 1684 stream_state: MutableMapping[str, Any], 1685 partition_router: PartitionRouter, 1686 attempt_to_create_cursor_if_not_provided: bool = False, 1687 **kwargs: Any, 1688 ) -> ConcurrentPerPartitionCursor: 1689 component_type = component_definition.get("type") 1690 if component_definition.get("type") != model_type.__name__: 1691 raise ValueError( 1692 f"Expected manifest component of type {model_type.__name__}, but received {component_type} instead" 1693 ) 1694 1695 # FIXME the interfaces of the concurrent cursor are kind of annoying as they take a `ComponentDefinition` instead of the actual model. This was done because the ConcurrentDeclarativeSource didn't have access to the models [here for example](https://github.com/airbytehq/airbyte-python-cdk/blob/f525803b3fec9329e4cc8478996a92bf884bfde9/airbyte_cdk/sources/declarative/concurrent_declarative_source.py#L354C54-L354C91). So now we have two cases: 1696 # * The ComponentDefinition comes from model.__dict__ in which case we have `parameters` 1697 # * The ComponentDefinition comes from the manifest as a dict in which case we have `$parameters` 1698 # We should change those interfaces to use the model once we clean up the code in CDS at which point the parameter propagation should happen as part of the ModelToComponentFactory. 1699 if "$parameters" not in component_definition and "parameters" in component_definition: 1700 component_definition["$parameters"] = component_definition.get("parameters") # type: ignore # This is a dict 1701 datetime_based_cursor_model = model_type.parse_obj(component_definition) 1702 1703 if not isinstance(datetime_based_cursor_model, DatetimeBasedCursorModel): 1704 raise ValueError( 1705 f"Expected {model_type.__name__} component, but received {datetime_based_cursor_model.__class__.__name__}" 1706 ) 1707 1708 cursor_field = self._get_catalog_defined_cursor_field( 1709 stream_name=stream_name, 1710 allow_catalog_defined_cursor_field=datetime_based_cursor_model.allow_catalog_defined_cursor_field 1711 or False, 1712 ) 1713 1714 if not cursor_field: 1715 interpolated_cursor_field = InterpolatedString.create( 1716 datetime_based_cursor_model.cursor_field, 1717 # FIXME the interfaces of the concurrent cursor are kind of annoying as they take a `ComponentDefinition` instead of the actual model. This was done because the ConcurrentDeclarativeSource didn't have access to the models [here for example](https://github.com/airbytehq/airbyte-python-cdk/blob/f525803b3fec9329e4cc8478996a92bf884bfde9/airbyte_cdk/sources/declarative/concurrent_declarative_source.py#L354C54-L354C91). So now we have two cases: 1718 # * The ComponentDefinition comes from model.__dict__ in which case we have `parameters` 1719 # * The ComponentDefinition comes from the manifest as a dict in which case we have `$parameters` 1720 # We should change those interfaces to use the model once we clean up the code in CDS at which point the parameter propagation should happen as part of the ModelToComponentFactory. 1721 parameters=datetime_based_cursor_model.parameters or {}, 1722 ) 1723 cursor_field = CursorField( 1724 cursor_field_key=interpolated_cursor_field.eval(config=config), 1725 supports_catalog_defined_cursor_field=datetime_based_cursor_model.allow_catalog_defined_cursor_field 1726 or False, 1727 ) 1728 1729 datetime_format = datetime_based_cursor_model.datetime_format 1730 1731 cursor_granularity = ( 1732 parse_duration(datetime_based_cursor_model.cursor_granularity) 1733 if datetime_based_cursor_model.cursor_granularity 1734 else None 1735 ) 1736 1737 connector_state_converter: DateTimeStreamStateConverter 1738 connector_state_converter = CustomFormatConcurrentStreamStateConverter( 1739 datetime_format=datetime_format, 1740 input_datetime_formats=datetime_based_cursor_model.cursor_datetime_formats, 1741 is_sequential_state=True, # ConcurrentPerPartitionCursor only works with sequential state 1742 cursor_granularity=cursor_granularity, 1743 ) 1744 1745 # Create the cursor factory 1746 cursor_factory = ConcurrentCursorFactory( 1747 partial( 1748 self.create_concurrent_cursor_from_datetime_based_cursor, 1749 state_manager=state_manager, 1750 model_type=model_type, 1751 component_definition=component_definition, 1752 stream_name=stream_name, 1753 stream_namespace=stream_namespace, 1754 config=config, 1755 message_repository=NoopMessageRepository(), 1756 ) 1757 ) 1758 1759 # Per-partition state doesn't make sense for GroupingPartitionRouter, so force the global state 1760 use_global_cursor = isinstance( 1761 partition_router, GroupingPartitionRouter 1762 ) or component_definition.get("global_substream_cursor", False) 1763 1764 # Return the concurrent cursor and state converter 1765 return ConcurrentPerPartitionCursor( 1766 cursor_factory=cursor_factory, 1767 partition_router=partition_router, 1768 stream_name=stream_name, 1769 stream_namespace=stream_namespace, 1770 stream_state=stream_state, 1771 message_repository=self._message_repository, # type: ignore 1772 connector_state_manager=state_manager, 1773 connector_state_converter=connector_state_converter, 1774 cursor_field=cursor_field, 1775 use_global_cursor=use_global_cursor, 1776 attempt_to_create_cursor_if_not_provided=attempt_to_create_cursor_if_not_provided, 1777 ) 1778 1779 @staticmethod 1780 def create_constant_backoff_strategy( 1781 model: ConstantBackoffStrategyModel, config: Config, **kwargs: Any 1782 ) -> ConstantBackoffStrategy: 1783 ModelToComponentFactory._validate_jitter_range(model.jitter_range_in_seconds) 1784 return ConstantBackoffStrategy( 1785 backoff_time_in_seconds=model.backoff_time_in_seconds, 1786 jitter_range_in_seconds=model.jitter_range_in_seconds, 1787 config=config, 1788 parameters=model.parameters or {}, 1789 ) 1790 1791 @staticmethod 1792 def _validate_jitter_range(jitter_range_in_seconds: Optional[float]) -> None: 1793 if jitter_range_in_seconds is not None and jitter_range_in_seconds < 0: 1794 raise ValueError("jitter_range_in_seconds must be greater than or equal to 0") 1795 1796 def create_cursor_pagination( 1797 self, model: CursorPaginationModel, config: Config, decoder: Decoder, **kwargs: Any 1798 ) -> CursorPaginationStrategy: 1799 if isinstance(decoder, PaginationDecoderDecorator): 1800 inner_decoder = decoder.decoder 1801 else: 1802 inner_decoder = decoder 1803 decoder = PaginationDecoderDecorator(decoder=decoder) 1804 1805 if self._is_supported_decoder_for_pagination(inner_decoder): 1806 decoder_to_use = decoder 1807 else: 1808 raise ValueError( 1809 self._UNSUPPORTED_DECODER_ERROR.format(decoder_type=type(inner_decoder)) 1810 ) 1811 1812 # Pydantic v1 Union type coercion can convert int to string depending on Union order. 1813 # If page_size is a string that represents an integer (not an interpolation), convert it back. 1814 page_size = model.page_size 1815 if isinstance(page_size, str) and page_size.isdigit(): 1816 page_size = int(page_size) 1817 1818 return CursorPaginationStrategy( 1819 cursor_value=model.cursor_value, 1820 decoder=decoder_to_use, 1821 page_size=page_size, 1822 stop_condition=model.stop_condition, 1823 config=config, 1824 parameters=model.parameters or {}, 1825 ) 1826 1827 def create_custom_component(self, model: Any, config: Config, **kwargs: Any) -> Any: 1828 """ 1829 Generically creates a custom component based on the model type and a class_name reference to the custom Python class being 1830 instantiated. Only the model's additional properties that match the custom class definition are passed to the constructor 1831 :param model: The Pydantic model of the custom component being created 1832 :param config: The custom defined connector config 1833 :return: The declarative component built from the Pydantic model to be used at runtime 1834 """ 1835 # Instantiating a custom component means importing and executing arbitrary code referenced 1836 # by `class_name`. Manifests supplied by a caller, whether through the config or directly to 1837 # the manifest server, are untrusted input and could point `class_name` at any importable 1838 # callable, so they honor the same `AIRBYTE_ENABLE_UNSAFE_CODE` gate as injected 1839 # `components.py` code. Manifests bundled in a published connector image are trusted and may 1840 # always use their bundled custom components. 1841 manifest_is_untrusted = not self._custom_components_trusted or bool( 1842 config.get(INJECTED_MANIFEST) 1843 ) 1844 if manifest_is_untrusted and not custom_code_execution_permitted(): 1845 raise AirbyteCustomCodeNotPermittedError 1846 1847 custom_component_class = self._get_class_from_fully_qualified_class_name(model.class_name) 1848 component_fields = get_type_hints(custom_component_class) 1849 model_args = model.dict() 1850 model_args["config"] = config 1851 1852 # There are cases where a parent component will pass arguments to a child component via kwargs. When there are field collisions 1853 # we defer to these arguments over the component's definition 1854 for key, arg in kwargs.items(): 1855 model_args[key] = arg 1856 1857 # Pydantic is unable to parse a custom component's fields that are subcomponents into models because their fields and types are not 1858 # defined in the schema. The fields and types are defined within the Python class implementation. Pydantic can only parse down to 1859 # the custom component and this code performs a second parse to convert the sub-fields first into models, then declarative components 1860 for model_field, model_value in model_args.items(): 1861 # If a custom component field doesn't have a type set, we try to use the type hints to infer the type 1862 if ( 1863 isinstance(model_value, dict) 1864 and "type" not in model_value 1865 and model_field in component_fields 1866 ): 1867 derived_type = self._derive_component_type_from_type_hints( 1868 component_fields.get(model_field) 1869 ) 1870 if derived_type: 1871 model_value["type"] = derived_type 1872 1873 if self._is_component(model_value): 1874 model_args[model_field] = self._create_nested_component( 1875 model, 1876 model_field, 1877 model_value, 1878 config, 1879 **kwargs, 1880 ) 1881 elif isinstance(model_value, list): 1882 vals = [] 1883 for v in model_value: 1884 if isinstance(v, dict) and "type" not in v and model_field in component_fields: 1885 derived_type = self._derive_component_type_from_type_hints( 1886 component_fields.get(model_field) 1887 ) 1888 if derived_type: 1889 v["type"] = derived_type 1890 if self._is_component(v): 1891 vals.append( 1892 self._create_nested_component( 1893 model, 1894 model_field, 1895 v, 1896 config, 1897 **kwargs, 1898 ) 1899 ) 1900 else: 1901 vals.append(v) 1902 model_args[model_field] = vals 1903 1904 kwargs = { 1905 class_field: model_args[class_field] 1906 for class_field in component_fields.keys() 1907 if class_field in model_args 1908 } 1909 1910 if "api_budget" in component_fields and kwargs.get("api_budget") is None: 1911 kwargs["api_budget"] = self._api_budget 1912 1913 return custom_component_class(**kwargs) 1914 1915 @staticmethod 1916 def _get_class_from_fully_qualified_class_name( 1917 full_qualified_class_name: str, 1918 ) -> Any: 1919 """Get a class from its fully qualified name. 1920 1921 If a custom components module is needed, we assume it is already registered - probably 1922 as `source_declarative_manifest.components` or `components`. 1923 1924 Args: 1925 full_qualified_class_name (str): The fully qualified name of the class (e.g., "module.ClassName"). 1926 1927 Returns: 1928 Any: The class object. 1929 1930 Raises: 1931 ValueError: If the class cannot be loaded. 1932 """ 1933 split = full_qualified_class_name.split(".") 1934 module_name_full = ".".join(split[:-1]) 1935 class_name = split[-1] 1936 1937 try: 1938 module_ref = importlib.import_module(module_name_full) 1939 except ModuleNotFoundError as e: 1940 if split[0] == "source_declarative_manifest": 1941 # During testing, the modules containing the custom components are not moved to source_declarative_manifest. In order to run the test, add the source folder to your PYTHONPATH or add it runtime using sys.path.append 1942 try: 1943 import os 1944 1945 module_name_with_source_declarative_manifest = ".".join(split[1:-1]) 1946 module_ref = importlib.import_module( 1947 module_name_with_source_declarative_manifest 1948 ) 1949 except ModuleNotFoundError: 1950 raise ValueError(f"Could not load module `{module_name_full}`.") from e 1951 else: 1952 raise ValueError(f"Could not load module `{module_name_full}`.") from e 1953 1954 try: 1955 return getattr(module_ref, class_name) 1956 except AttributeError as e: 1957 raise ValueError( 1958 f"Could not load class `{class_name}` from module `{module_name_full}`.", 1959 ) from e 1960 1961 @staticmethod 1962 def _derive_component_type_from_type_hints(field_type: Any) -> Optional[str]: 1963 interface = field_type 1964 while True: 1965 origin = get_origin(interface) 1966 if origin: 1967 # Unnest types until we reach the raw type 1968 # List[T] -> T 1969 # Optional[List[T]] -> T 1970 args = get_args(interface) 1971 interface = args[0] 1972 else: 1973 break 1974 if isinstance(interface, type) and not ModelToComponentFactory.is_builtin_type(interface): 1975 return interface.__name__ 1976 return None 1977 1978 @staticmethod 1979 def is_builtin_type(cls: Optional[Type[Any]]) -> bool: 1980 if not cls: 1981 return False 1982 return cls.__module__ == "builtins" 1983 1984 @staticmethod 1985 def _extract_missing_parameters(error: TypeError) -> List[str]: 1986 parameter_search = re.search(r"keyword-only.*:\s(.*)", str(error)) 1987 if parameter_search: 1988 return re.findall(r"\'(.+?)\'", parameter_search.group(1)) 1989 else: 1990 return [] 1991 1992 def _create_nested_component( 1993 self, model: Any, model_field: str, model_value: Any, config: Config, **kwargs: Any 1994 ) -> Any: 1995 type_name = model_value.get("type", None) 1996 if not type_name: 1997 # If no type is specified, we can assume this is a dictionary object which can be returned instead of a subcomponent 1998 return model_value 1999 2000 model_type = self.TYPE_NAME_TO_MODEL.get(type_name, None) 2001 if model_type: 2002 parsed_model = model_type.parse_obj(model_value) 2003 try: 2004 # To improve usability of the language, certain fields are shared between components. This can come in the form of 2005 # a parent component passing some of its fields to a child component or the parent extracting fields from other child 2006 # components and passing it to others. One example is the DefaultPaginator referencing the HttpRequester url_base 2007 # while constructing a SimpleRetriever. However, custom components don't support this behavior because they are created 2008 # generically in create_custom_component(). This block allows developers to specify extra arguments in $parameters that 2009 # are needed by a component and could not be shared. 2010 model_constructor = self.PYDANTIC_MODEL_TO_CONSTRUCTOR.get(parsed_model.__class__) 2011 constructor_kwargs = inspect.getfullargspec(model_constructor).kwonlyargs 2012 model_parameters = model_value.get("$parameters", {}) 2013 matching_parameters = { 2014 kwarg: model_parameters[kwarg] 2015 for kwarg in constructor_kwargs 2016 if kwarg in model_parameters 2017 } 2018 matching_kwargs = { 2019 kwarg: kwargs[kwarg] for kwarg in constructor_kwargs if kwarg in kwargs 2020 } 2021 return self._create_component_from_model( 2022 model=parsed_model, config=config, **(matching_parameters | matching_kwargs) 2023 ) 2024 except TypeError as error: 2025 missing_parameters = self._extract_missing_parameters(error) 2026 if missing_parameters: 2027 raise ValueError( 2028 f"Error creating component '{type_name}' with parent custom component {model.class_name}: Please provide " 2029 + ", ".join( 2030 ( 2031 f"{type_name}.$parameters.{parameter}" 2032 for parameter in missing_parameters 2033 ) 2034 ) 2035 ) 2036 raise TypeError( 2037 f"Error creating component '{type_name}' with parent custom component {model.class_name}: {error}" 2038 ) 2039 else: 2040 raise ValueError( 2041 f"Error creating custom component {model.class_name}. Subcomponent creation has not been implemented for '{type_name}'" 2042 ) 2043 2044 @staticmethod 2045 def _is_component(model_value: Any) -> bool: 2046 return isinstance(model_value, dict) and model_value.get("type") is not None 2047 2048 def create_default_stream( 2049 self, model: DeclarativeStreamModel, config: Config, is_parent: bool = False, **kwargs: Any 2050 ) -> AbstractStream: 2051 primary_key = model.primary_key.__root__ if model.primary_key else None 2052 self._migrate_state(model, config) 2053 2054 partition_router = self._build_stream_slicer_from_partition_router( 2055 model.retriever, 2056 config, 2057 stream_name=model.name, 2058 **kwargs, 2059 ) 2060 concurrent_cursor = self._build_concurrent_cursor(model, partition_router, config) 2061 if model.incremental_sync and isinstance(model.incremental_sync, DatetimeBasedCursorModel): 2062 cursor_model: DatetimeBasedCursorModel = model.incremental_sync 2063 2064 end_time_option = ( 2065 self._create_component_from_model( 2066 cursor_model.end_time_option, config, parameters=cursor_model.parameters or {} 2067 ) 2068 if cursor_model.end_time_option 2069 else None 2070 ) 2071 start_time_option = ( 2072 self._create_component_from_model( 2073 cursor_model.start_time_option, config, parameters=cursor_model.parameters or {} 2074 ) 2075 if cursor_model.start_time_option 2076 else None 2077 ) 2078 2079 datetime_request_options_provider = DatetimeBasedRequestOptionsProvider( 2080 start_time_option=start_time_option, 2081 end_time_option=end_time_option, 2082 partition_field_start=cursor_model.partition_field_start, 2083 partition_field_end=cursor_model.partition_field_end, 2084 config=config, 2085 parameters=model.parameters or {}, 2086 ) 2087 request_options_provider = ( 2088 datetime_request_options_provider 2089 if not isinstance(concurrent_cursor, ConcurrentPerPartitionCursor) 2090 else PerPartitionRequestOptionsProvider( 2091 partition_router, datetime_request_options_provider 2092 ) 2093 ) 2094 elif model.incremental_sync and isinstance( 2095 model.incremental_sync, IncrementingCountCursorModel 2096 ): 2097 if isinstance(concurrent_cursor, ConcurrentPerPartitionCursor): 2098 raise ValueError( 2099 "PerPartition does not support per partition states because switching to global state is time based" 2100 ) 2101 2102 cursor_model: IncrementingCountCursorModel = model.incremental_sync # type: ignore 2103 2104 start_time_option = ( 2105 self._create_component_from_model( 2106 cursor_model.start_value_option, # type: ignore # mypy still thinks cursor_model of type DatetimeBasedCursor 2107 config, 2108 parameters=cursor_model.parameters or {}, 2109 ) 2110 if cursor_model.start_value_option # type: ignore # mypy still thinks cursor_model of type DatetimeBasedCursor 2111 else None 2112 ) 2113 2114 # The concurrent engine defaults the start/end fields on the slice to "start" and "end", but 2115 # the default DatetimeBasedRequestOptionsProvider() sets them to start_time/end_time 2116 partition_field_start = "start" 2117 2118 request_options_provider = DatetimeBasedRequestOptionsProvider( 2119 start_time_option=start_time_option, 2120 partition_field_start=partition_field_start, 2121 config=config, 2122 parameters=model.parameters or {}, 2123 ) 2124 else: 2125 request_options_provider = None 2126 2127 transformations = [] 2128 if model.transformations: 2129 for transformation_model in model.transformations: 2130 transformations.append( 2131 self._create_component_from_model(model=transformation_model, config=config) 2132 ) 2133 file_uploader = None 2134 if model.file_uploader: 2135 file_uploader = self._create_component_from_model( 2136 model=model.file_uploader, config=config 2137 ) 2138 2139 stream_slicer: ConcurrentStreamSlicer = ( 2140 partition_router 2141 if isinstance(concurrent_cursor, FinalStateCursor) 2142 else concurrent_cursor 2143 ) 2144 2145 retriever = self._create_component_from_model( 2146 model=model.retriever, 2147 config=config, 2148 name=model.name, 2149 primary_key=primary_key, 2150 request_options_provider=request_options_provider, 2151 stream_slicer=stream_slicer, 2152 partition_router=partition_router, 2153 has_stop_condition_cursor=self._is_stop_condition_on_cursor(model), 2154 is_client_side_incremental_sync=self._is_client_side_filtering_enabled(model), 2155 cursor=concurrent_cursor, 2156 transformations=transformations, 2157 file_uploader=file_uploader, 2158 incremental_sync=model.incremental_sync, 2159 ) 2160 if isinstance(retriever, AsyncRetriever): 2161 stream_slicer = retriever.stream_slicer 2162 2163 schema_loader: SchemaLoader 2164 if model.schema_loader and isinstance(model.schema_loader, list): 2165 nested_schema_loaders = [ 2166 self._create_component_from_model(model=nested_schema_loader, config=config) 2167 for nested_schema_loader in model.schema_loader 2168 ] 2169 schema_loader = CompositeSchemaLoader( 2170 schema_loaders=nested_schema_loaders, parameters={} 2171 ) 2172 elif model.schema_loader: 2173 schema_loader = self._create_component_from_model( 2174 model=model.schema_loader, # type: ignore # If defined, schema_loader is guaranteed not to be a list and will be one of the existing base models 2175 config=config, 2176 ) 2177 else: 2178 options = model.parameters or {} 2179 if "name" not in options: 2180 options["name"] = model.name 2181 schema_loader = DefaultSchemaLoader(config=config, parameters=options) 2182 schema_loader = CachingSchemaLoaderDecorator(schema_loader) 2183 2184 stream_name = model.name or "" 2185 return DefaultStream( 2186 partition_generator=StreamSlicerPartitionGenerator( 2187 DeclarativePartitionFactory( 2188 stream_name, 2189 schema_loader, 2190 retriever, 2191 self._message_repository, 2192 ), 2193 stream_slicer, 2194 slice_limit=self._limit_slices_fetched, 2195 ), 2196 name=stream_name, 2197 json_schema=schema_loader.get_json_schema, 2198 primary_key=get_primary_key_from_stream(primary_key), 2199 cursor_field=( 2200 concurrent_cursor.cursor_field 2201 if hasattr(concurrent_cursor, "cursor_field") 2202 else None 2203 ), 2204 logger=logging.getLogger(f"airbyte.{stream_name}"), 2205 cursor=concurrent_cursor, 2206 supports_file_transfer=hasattr(model, "file_uploader") and bool(model.file_uploader), 2207 ) 2208 2209 def _migrate_state(self, model: DeclarativeStreamModel, config: Config) -> None: 2210 stream_name = model.name or "" 2211 stream_state = self._connector_state_manager.get_stream_state( 2212 stream_name=stream_name, namespace=None 2213 ) 2214 if model.state_migrations: 2215 state_transformations = [ 2216 self._create_component_from_model(state_migration, config, declarative_stream=model) 2217 for state_migration in model.state_migrations 2218 ] 2219 else: 2220 state_transformations = [] 2221 stream_state = self.apply_stream_state_migrations(state_transformations, stream_state) 2222 self._connector_state_manager.update_state_for_stream( 2223 stream_name=stream_name, namespace=None, value=stream_state 2224 ) 2225 2226 def _is_stop_condition_on_cursor(self, model: DeclarativeStreamModel) -> bool: 2227 return bool( 2228 model.incremental_sync 2229 and hasattr(model.incremental_sync, "is_data_feed") 2230 and model.incremental_sync.is_data_feed 2231 ) 2232 2233 def _is_client_side_filtering_enabled(self, model: DeclarativeStreamModel) -> bool: 2234 return bool( 2235 model.incremental_sync 2236 and hasattr(model.incremental_sync, "is_client_side_incremental") 2237 and model.incremental_sync.is_client_side_incremental 2238 ) 2239 2240 def _build_stream_slicer_from_partition_router( 2241 self, 2242 model: Union[ 2243 AsyncRetrieverModel, 2244 CustomRetrieverModel, 2245 SimpleRetrieverModel, 2246 ], 2247 config: Config, 2248 stream_name: Optional[str] = None, 2249 **kwargs: Any, 2250 ) -> PartitionRouter: 2251 if ( 2252 hasattr(model, "partition_router") 2253 and isinstance(model, (SimpleRetrieverModel, AsyncRetrieverModel, CustomRetrieverModel)) 2254 and model.partition_router 2255 ): 2256 stream_slicer_model = model.partition_router 2257 if isinstance(stream_slicer_model, list): 2258 return CartesianProductStreamSlicer( 2259 [ 2260 self._create_component_from_model( 2261 model=slicer, config=config, stream_name=stream_name or "" 2262 ) 2263 for slicer in stream_slicer_model 2264 ], 2265 parameters={}, 2266 ) 2267 elif isinstance(stream_slicer_model, dict): 2268 # partition router comes from CustomRetrieverModel therefore has not been parsed as a model 2269 params = stream_slicer_model.get("$parameters") 2270 if not isinstance(params, dict): 2271 params = {} 2272 stream_slicer_model["$parameters"] = params 2273 2274 if stream_name is not None: 2275 params["stream_name"] = stream_name 2276 2277 return self._create_nested_component( # type: ignore[no-any-return] # There is no guarantee that this will return a stream slicer. If not, we expect an AttributeError during the call to `stream_slices` 2278 model, 2279 "partition_router", 2280 stream_slicer_model, 2281 config, 2282 **kwargs, 2283 ) 2284 else: 2285 return self._create_component_from_model( # type: ignore[no-any-return] # Will be created PartitionRouter as stream_slicer_model is model.partition_router 2286 model=stream_slicer_model, config=config, stream_name=stream_name or "" 2287 ) 2288 return SinglePartitionRouter(parameters={}) 2289 2290 def _build_concurrent_cursor( 2291 self, 2292 model: DeclarativeStreamModel, 2293 stream_slicer: Optional[PartitionRouter], 2294 config: Config, 2295 ) -> Cursor: 2296 stream_name = model.name or "" 2297 stream_state = self._connector_state_manager.get_stream_state(stream_name, None) 2298 2299 if ( 2300 model.incremental_sync 2301 and stream_slicer 2302 and not isinstance(stream_slicer, SinglePartitionRouter) 2303 ): 2304 if isinstance(model.incremental_sync, IncrementingCountCursorModel): 2305 # We don't currently support usage of partition routing and IncrementingCountCursor at the 2306 # same time because we didn't solve for design questions like what the lookback window would 2307 # be as well as global cursor fall backs. We have not seen customers that have needed both 2308 # at the same time yet and are currently punting on this until we need to solve it. 2309 raise ValueError( 2310 f"The low-code framework does not currently support usage of a PartitionRouter and an IncrementingCountCursor at the same time. Please specify only one of these options for stream {stream_name}." 2311 ) 2312 return self.create_concurrent_cursor_from_perpartition_cursor( # type: ignore # This is a known issue that we are creating and returning a ConcurrentCursor which does not technically implement the (low-code) StreamSlicer. However, (low-code) StreamSlicer and ConcurrentCursor both implement StreamSlicer.stream_slices() which is the primary method needed for checkpointing 2313 state_manager=self._connector_state_manager, 2314 model_type=DatetimeBasedCursorModel, 2315 component_definition=model.incremental_sync.__dict__, 2316 stream_name=stream_name, 2317 stream_state=stream_state, 2318 stream_namespace=None, 2319 config=config or {}, 2320 partition_router=stream_slicer, 2321 attempt_to_create_cursor_if_not_provided=True, # FIXME can we remove that now? 2322 ) 2323 elif model.incremental_sync: 2324 if type(model.incremental_sync) == IncrementingCountCursorModel: 2325 return self.create_concurrent_cursor_from_incrementing_count_cursor( # type: ignore # This is a known issue that we are creating and returning a ConcurrentCursor which does not technically implement the (low-code) StreamSlicer. However, (low-code) StreamSlicer and ConcurrentCursor both implement StreamSlicer.stream_slices() which is the primary method needed for checkpointing 2326 model_type=IncrementingCountCursorModel, 2327 component_definition=model.incremental_sync.__dict__, 2328 stream_name=stream_name, 2329 stream_namespace=None, 2330 stream_state=stream_state, 2331 config=config or {}, 2332 ) 2333 elif type(model.incremental_sync) == DatetimeBasedCursorModel: 2334 return self.create_concurrent_cursor_from_datetime_based_cursor( # type: ignore # This is a known issue that we are creating and returning a ConcurrentCursor which does not technically implement the (low-code) StreamSlicer. However, (low-code) StreamSlicer and ConcurrentCursor both implement StreamSlicer.stream_slices() which is the primary method needed for checkpointing 2335 model_type=type(model.incremental_sync), 2336 component_definition=model.incremental_sync.__dict__, 2337 stream_name=stream_name, 2338 stream_namespace=None, 2339 stream_state=stream_state, 2340 config=config or {}, 2341 attempt_to_create_cursor_if_not_provided=True, 2342 ) 2343 else: 2344 raise ValueError( 2345 f"Incremental sync of type {type(model.incremental_sync)} is not supported" 2346 ) 2347 return FinalStateCursor(stream_name, None, self._message_repository) 2348 2349 def create_default_error_handler( 2350 self, model: DefaultErrorHandlerModel, config: Config, **kwargs: Any 2351 ) -> DefaultErrorHandler: 2352 backoff_strategies = [] 2353 if model.backoff_strategies: 2354 for backoff_strategy_model in model.backoff_strategies: 2355 backoff_strategies.append( 2356 self._create_component_from_model(model=backoff_strategy_model, config=config) 2357 ) 2358 2359 response_filters = [] 2360 if model.response_filters: 2361 for response_filter_model in model.response_filters: 2362 response_filters.append( 2363 self._create_component_from_model(model=response_filter_model, config=config) 2364 ) 2365 response_filters.append( 2366 HttpResponseFilter(config=config, parameters=model.parameters or {}) 2367 ) 2368 2369 return DefaultErrorHandler( 2370 backoff_strategies=backoff_strategies, 2371 max_retries=model.max_retries, 2372 response_filters=response_filters, 2373 config=config, 2374 parameters=model.parameters or {}, 2375 ) 2376 2377 def create_default_paginator( 2378 self, 2379 model: DefaultPaginatorModel, 2380 config: Config, 2381 *, 2382 url_base: str, 2383 extractor_model: Optional[Union[CustomRecordExtractorModel, DpathExtractorModel]] = None, 2384 decoder: Optional[Decoder] = None, 2385 cursor_used_for_stop_condition: Optional[Cursor] = None, 2386 ) -> Union[DefaultPaginator, PaginatorTestReadDecorator]: 2387 if decoder: 2388 if self._is_supported_decoder_for_pagination(decoder): 2389 decoder_to_use = PaginationDecoderDecorator(decoder=decoder) 2390 else: 2391 raise ValueError(self._UNSUPPORTED_DECODER_ERROR.format(decoder_type=type(decoder))) 2392 else: 2393 decoder_to_use = PaginationDecoderDecorator(decoder=JsonDecoder(parameters={})) 2394 page_size_option = ( 2395 self._create_component_from_model(model=model.page_size_option, config=config) 2396 if model.page_size_option 2397 else None 2398 ) 2399 page_token_option = ( 2400 self._create_component_from_model(model=model.page_token_option, config=config) 2401 if model.page_token_option 2402 else None 2403 ) 2404 pagination_strategy = self._create_component_from_model( 2405 model=model.pagination_strategy, 2406 config=config, 2407 decoder=decoder_to_use, 2408 extractor_model=extractor_model, 2409 ) 2410 if cursor_used_for_stop_condition: 2411 pagination_strategy = StopConditionPaginationStrategyDecorator( 2412 pagination_strategy, CursorStopCondition(cursor_used_for_stop_condition) 2413 ) 2414 paginator = DefaultPaginator( 2415 decoder=decoder_to_use, 2416 page_size_option=page_size_option, 2417 page_token_option=page_token_option, 2418 pagination_strategy=pagination_strategy, 2419 url_base=url_base, 2420 config=config, 2421 parameters=model.parameters or {}, 2422 ) 2423 if self._limit_pages_fetched_per_slice: 2424 return PaginatorTestReadDecorator(paginator, self._limit_pages_fetched_per_slice) 2425 return paginator 2426 2427 def create_dpath_extractor( 2428 self, 2429 model: DpathExtractorModel, 2430 config: Config, 2431 decoder: Optional[Decoder] = None, 2432 **kwargs: Any, 2433 ) -> DpathExtractor: 2434 if decoder: 2435 decoder_to_use = decoder 2436 else: 2437 decoder_to_use = JsonDecoder(parameters={}) 2438 model_field_path: List[Union[InterpolatedString, str]] = [x for x in model.field_path] 2439 2440 record_expander = None 2441 if model.record_expander: 2442 record_expander = self._create_component_from_model( 2443 model=model.record_expander, 2444 config=config, 2445 ) 2446 2447 return DpathExtractor( 2448 decoder=decoder_to_use, 2449 field_path=model_field_path, 2450 config=config, 2451 parameters=model.parameters or {}, 2452 record_expander=record_expander, 2453 ) 2454 2455 def create_record_expander( 2456 self, 2457 model: RecordExpanderModel, 2458 config: Config, 2459 **kwargs: Any, 2460 ) -> RecordExpander: 2461 return RecordExpander( 2462 expand_records_from_field=model.expand_records_from_field, 2463 config=config, 2464 parameters=model.parameters or {}, 2465 remain_original_record=model.remain_original_record or False, 2466 on_no_records=OnNoRecords(model.on_no_records.value) 2467 if model.on_no_records 2468 else OnNoRecords.skip, 2469 ) 2470 2471 @staticmethod 2472 def create_response_to_file_extractor( 2473 model: ResponseToFileExtractorModel, 2474 **kwargs: Any, 2475 ) -> ResponseToFileExtractor: 2476 return ResponseToFileExtractor( 2477 parameters=model.parameters or {}, 2478 preserve_na_values=model.preserve_na_values or False, 2479 ) 2480 2481 @staticmethod 2482 def create_exponential_backoff_strategy( 2483 model: ExponentialBackoffStrategyModel, config: Config 2484 ) -> ExponentialBackoffStrategy: 2485 ModelToComponentFactory._validate_jitter_range(model.jitter_range_in_seconds) 2486 return ExponentialBackoffStrategy( 2487 factor=model.factor or 5, 2488 jitter_range_in_seconds=model.jitter_range_in_seconds, 2489 parameters=model.parameters or {}, 2490 config=config, 2491 ) 2492 2493 @staticmethod 2494 def create_group_by_key(model: GroupByKeyMergeStrategyModel, config: Config) -> GroupByKey: 2495 return GroupByKey(model.key, config=config, parameters=model.parameters or {}) 2496 2497 def create_http_requester( 2498 self, 2499 model: HttpRequesterModel, 2500 config: Config, 2501 decoder: Decoder = JsonDecoder(parameters={}), 2502 query_properties_key: Optional[str] = None, 2503 use_cache: Optional[bool] = None, 2504 *, 2505 name: str, 2506 ) -> HttpRequester: 2507 authenticator = ( 2508 self._create_component_from_model( 2509 model=model.authenticator, 2510 config=config, 2511 url_base=model.url or model.url_base, 2512 name=name, 2513 decoder=decoder, 2514 ) 2515 if model.authenticator 2516 else None 2517 ) 2518 error_handler = ( 2519 self._create_component_from_model(model=model.error_handler, config=config) 2520 if model.error_handler 2521 else DefaultErrorHandler( 2522 backoff_strategies=[], 2523 response_filters=[], 2524 config=config, 2525 parameters=model.parameters or {}, 2526 ) 2527 ) 2528 2529 api_budget = self._api_budget 2530 2531 request_options_provider = InterpolatedRequestOptionsProvider( 2532 request_body=model.request_body, 2533 request_body_data=model.request_body_data, 2534 request_body_json=model.request_body_json, 2535 request_headers=model.request_headers, 2536 request_parameters=model.request_parameters, # type: ignore # QueryProperties have been removed in `create_simple_retriever` 2537 query_properties_key=query_properties_key, 2538 config=config, 2539 parameters=model.parameters or {}, 2540 ) 2541 2542 assert model.use_cache is not None # for mypy 2543 assert model.http_method is not None # for mypy 2544 2545 should_use_cache = (model.use_cache or bool(use_cache)) and not self._disable_cache 2546 2547 return HttpRequester( 2548 name=name, 2549 url=model.url, 2550 url_base=model.url_base, 2551 path=model.path, 2552 authenticator=authenticator, 2553 error_handler=error_handler, 2554 api_budget=api_budget, 2555 http_method=HttpMethod[model.http_method.value], 2556 request_options_provider=request_options_provider, 2557 config=config, 2558 disable_retries=self._disable_retries, 2559 parameters=model.parameters or {}, 2560 message_repository=self._message_repository, 2561 use_cache=should_use_cache, 2562 decoder=decoder, 2563 stream_response=decoder.is_stream_response() if decoder else False, 2564 ) 2565 2566 @staticmethod 2567 def create_http_response_filter( 2568 model: HttpResponseFilterModel, config: Config, **kwargs: Any 2569 ) -> HttpResponseFilter: 2570 if model.action: 2571 action = ResponseAction(model.action.value) 2572 else: 2573 action = None 2574 2575 failure_type = FailureType(model.failure_type.value) if model.failure_type else None 2576 2577 http_codes = ( 2578 set(model.http_codes) if model.http_codes else set() 2579 ) # JSON schema notation has no set data type. The schema enforces an array of unique elements 2580 2581 return HttpResponseFilter( 2582 action=action, 2583 failure_type=failure_type, 2584 error_message=model.error_message or "", 2585 error_message_contains=model.error_message_contains or "", 2586 http_codes=http_codes, 2587 predicate=model.predicate or "", 2588 config=config, 2589 parameters=model.parameters or {}, 2590 ) 2591 2592 @staticmethod 2593 def create_inline_schema_loader( 2594 model: InlineSchemaLoaderModel, config: Config, **kwargs: Any 2595 ) -> InlineSchemaLoader: 2596 return InlineSchemaLoader(schema=model.schema_ or {}, parameters={}) 2597 2598 def create_complex_field_type( 2599 self, model: ComplexFieldTypeModel, config: Config, **kwargs: Any 2600 ) -> ComplexFieldType: 2601 items = ( 2602 self._create_component_from_model(model=model.items, config=config) 2603 if isinstance(model.items, ComplexFieldTypeModel) 2604 else model.items 2605 ) 2606 2607 return ComplexFieldType(field_type=model.field_type, items=items) 2608 2609 def create_types_map(self, model: TypesMapModel, config: Config, **kwargs: Any) -> TypesMap: 2610 target_type = ( 2611 self._create_component_from_model(model=model.target_type, config=config) 2612 if isinstance(model.target_type, ComplexFieldTypeModel) 2613 else model.target_type 2614 ) 2615 2616 return TypesMap( 2617 target_type=target_type, 2618 current_type=model.current_type, 2619 condition=model.condition if model.condition is not None else "True", 2620 ) 2621 2622 def create_schema_type_identifier( 2623 self, model: SchemaTypeIdentifierModel, config: Config, **kwargs: Any 2624 ) -> SchemaTypeIdentifier: 2625 types_mapping = [] 2626 if model.types_mapping: 2627 types_mapping.extend( 2628 [ 2629 self._create_component_from_model(types_map, config=config) 2630 for types_map in model.types_mapping 2631 ] 2632 ) 2633 model_schema_pointer: List[Union[InterpolatedString, str]] = ( 2634 [x for x in model.schema_pointer] if model.schema_pointer else [] 2635 ) 2636 model_key_pointer: List[Union[InterpolatedString, str]] = [x for x in model.key_pointer] 2637 model_type_pointer: Optional[List[Union[InterpolatedString, str]]] = ( 2638 [x for x in model.type_pointer] if model.type_pointer else None 2639 ) 2640 2641 return SchemaTypeIdentifier( 2642 schema_pointer=model_schema_pointer, 2643 key_pointer=model_key_pointer, 2644 type_pointer=model_type_pointer, 2645 types_mapping=types_mapping, 2646 parameters=model.parameters or {}, 2647 ) 2648 2649 def create_dynamic_schema_loader( 2650 self, model: DynamicSchemaLoaderModel, config: Config, **kwargs: Any 2651 ) -> DynamicSchemaLoader: 2652 schema_transformations = [] 2653 if model.schema_transformations: 2654 for transformation_model in model.schema_transformations: 2655 schema_transformations.append( 2656 self._create_component_from_model(model=transformation_model, config=config) 2657 ) 2658 name = "dynamic_properties" 2659 retriever = self._create_component_from_model( 2660 model=model.retriever, 2661 config=config, 2662 name=name, 2663 primary_key=None, 2664 partition_router=self._build_stream_slicer_from_partition_router( 2665 model.retriever, config 2666 ), 2667 transformations=[], 2668 use_cache=True, 2669 log_formatter=( 2670 lambda response: format_http_message( 2671 response, 2672 f"Schema loader '{name}' request", 2673 f"Request performed in order to extract schema.", 2674 name, 2675 is_auxiliary=True, 2676 ) 2677 ), 2678 ) 2679 schema_type_identifier = self._create_component_from_model( 2680 model.schema_type_identifier, config=config, parameters=model.parameters or {} 2681 ) 2682 schema_filter = ( 2683 self._create_component_from_model( 2684 model.schema_filter, config=config, parameters=model.parameters or {} 2685 ) 2686 if model.schema_filter is not None 2687 else None 2688 ) 2689 2690 return DynamicSchemaLoader( 2691 retriever=retriever, 2692 config=config, 2693 schema_transformations=schema_transformations, 2694 schema_filter=schema_filter, 2695 schema_type_identifier=schema_type_identifier, 2696 parameters=model.parameters or {}, 2697 ) 2698 2699 @staticmethod 2700 def create_json_decoder(model: JsonDecoderModel, config: Config, **kwargs: Any) -> Decoder: 2701 return JsonDecoder(parameters={}) 2702 2703 def create_csv_decoder(self, model: CsvDecoderModel, config: Config, **kwargs: Any) -> Decoder: 2704 return CompositeRawDecoder( 2705 parser=ModelToComponentFactory._get_parser(model, config), 2706 stream_response=False if self._emit_connector_builder_messages else True, 2707 ) 2708 2709 def create_jsonl_decoder( 2710 self, model: JsonlDecoderModel, config: Config, **kwargs: Any 2711 ) -> Decoder: 2712 return CompositeRawDecoder( 2713 parser=ModelToComponentFactory._get_parser(model, config), 2714 stream_response=False if self._emit_connector_builder_messages else True, 2715 ) 2716 2717 def create_json_items_decoder( 2718 self, model: JsonItemsDecoderModel, config: Config, **kwargs: Any 2719 ) -> Decoder: 2720 return CompositeRawDecoder( 2721 parser=ModelToComponentFactory._get_parser(model, config), 2722 stream_response=False if self._emit_connector_builder_messages else True, 2723 ) 2724 2725 def create_gzip_decoder( 2726 self, model: GzipDecoderModel, config: Config, **kwargs: Any 2727 ) -> Decoder: 2728 _compressed_response_types = { 2729 "gzip", 2730 "x-gzip", 2731 "gzip, deflate", 2732 "x-gzip, deflate", 2733 "application/zip", 2734 "application/gzip", 2735 "application/x-gzip", 2736 "application/x-zip-compressed", 2737 } 2738 2739 gzip_parser: GzipParser = ModelToComponentFactory._get_parser(model, config) # type: ignore # based on the model, we know this will be a GzipParser 2740 2741 if self._emit_connector_builder_messages: 2742 # This is very surprising but if the response is not streamed, 2743 # CompositeRawDecoder calls response.content and the requests library actually uncompress the data as opposed to response.raw, 2744 # which uses urllib3 directly and does not uncompress the data. 2745 return CompositeRawDecoder(gzip_parser.inner_parser, False) 2746 2747 return CompositeRawDecoder.by_headers( 2748 [({"Content-Encoding", "Content-Type"}, _compressed_response_types, gzip_parser)], 2749 stream_response=True, 2750 fallback_parser=gzip_parser.inner_parser, 2751 ) 2752 2753 @staticmethod 2754 def create_iterable_decoder( 2755 model: IterableDecoderModel, config: Config, **kwargs: Any 2756 ) -> IterableDecoder: 2757 return IterableDecoder(parameters={}) 2758 2759 @staticmethod 2760 def create_xml_decoder(model: XmlDecoderModel, config: Config, **kwargs: Any) -> XmlDecoder: 2761 return XmlDecoder(parameters={}) 2762 2763 def create_zipfile_decoder( 2764 self, model: ZipfileDecoderModel, config: Config, **kwargs: Any 2765 ) -> ZipfileDecoder: 2766 return ZipfileDecoder(parser=ModelToComponentFactory._get_parser(model.decoder, config)) 2767 2768 @staticmethod 2769 def _get_parser(model: BaseModel, config: Config) -> Parser: 2770 if isinstance(model, JsonDecoderModel): 2771 # Note that the logic is a bit different from the JsonDecoder as there is some legacy that is maintained to return {} on error cases 2772 return JsonParser() 2773 elif isinstance(model, JsonItemsDecoderModel): 2774 return JsonItemsParser( 2775 items_path=model.items_path, 2776 encoding=model.encoding, 2777 ) 2778 elif isinstance(model, JsonlDecoderModel): 2779 return JsonLineParser() 2780 elif isinstance(model, CsvDecoderModel): 2781 return CsvParser( 2782 encoding=model.encoding, 2783 delimiter=model.delimiter, 2784 set_values_to_none=model.set_values_to_none, 2785 ) 2786 elif isinstance(model, GzipDecoderModel): 2787 return GzipParser( 2788 inner_parser=ModelToComponentFactory._get_parser(model.decoder, config) 2789 ) 2790 elif isinstance( 2791 model, (CustomDecoderModel, IterableDecoderModel, XmlDecoderModel, ZipfileDecoderModel) 2792 ): 2793 raise ValueError(f"Decoder type {model} does not have parser associated to it") 2794 2795 raise ValueError(f"Unknown decoder type {model}") 2796 2797 @staticmethod 2798 def create_json_file_schema_loader( 2799 model: JsonFileSchemaLoaderModel, config: Config, **kwargs: Any 2800 ) -> JsonFileSchemaLoader: 2801 return JsonFileSchemaLoader( 2802 file_path=model.file_path or "", config=config, parameters=model.parameters or {} 2803 ) 2804 2805 def create_jwt_authenticator( 2806 self, model: JwtAuthenticatorModel, config: Config, **kwargs: Any 2807 ) -> JwtAuthenticator: 2808 jwt_headers = model.jwt_headers or JwtHeadersModel(kid=None, typ="JWT", cty=None) 2809 jwt_payload = model.jwt_payload or JwtPayloadModel(iss=None, sub=None, aud=None) 2810 request_option = ( 2811 self._create_component_from_model(model.request_option, config) 2812 if model.request_option 2813 else None 2814 ) 2815 return JwtAuthenticator( 2816 config=config, 2817 parameters=model.parameters or {}, 2818 algorithm=JwtAlgorithm(model.algorithm.value), 2819 secret_key=model.secret_key, 2820 base64_encode_secret_key=model.base64_encode_secret_key, 2821 token_duration=model.token_duration, 2822 header_prefix=model.header_prefix, 2823 kid=jwt_headers.kid, 2824 typ=jwt_headers.typ, 2825 cty=jwt_headers.cty, 2826 iss=jwt_payload.iss, 2827 sub=jwt_payload.sub, 2828 aud=jwt_payload.aud, 2829 additional_jwt_headers=model.additional_jwt_headers, 2830 additional_jwt_payload=model.additional_jwt_payload, 2831 passphrase=model.passphrase, 2832 request_option=request_option, 2833 ) 2834 2835 def create_list_partition_router( 2836 self, model: ListPartitionRouterModel, config: Config, **kwargs: Any 2837 ) -> ListPartitionRouter: 2838 request_option = ( 2839 self._create_component_from_model(model.request_option, config) 2840 if model.request_option 2841 else None 2842 ) 2843 return ListPartitionRouter( 2844 cursor_field=model.cursor_field, 2845 request_option=request_option, 2846 values=model.values, 2847 config=config, 2848 parameters=model.parameters or {}, 2849 ) 2850 2851 @staticmethod 2852 def create_min_max_datetime( 2853 model: MinMaxDatetimeModel, config: Config, **kwargs: Any 2854 ) -> MinMaxDatetime: 2855 return MinMaxDatetime( 2856 datetime=model.datetime, 2857 datetime_format=model.datetime_format or "", 2858 max_datetime=model.max_datetime or "", 2859 min_datetime=model.min_datetime or "", 2860 parameters=model.parameters or {}, 2861 ) 2862 2863 @staticmethod 2864 def create_no_auth(model: NoAuthModel, config: Config, **kwargs: Any) -> NoAuth: 2865 return NoAuth(parameters=model.parameters or {}) 2866 2867 @staticmethod 2868 def create_no_pagination( 2869 model: NoPaginationModel, config: Config, **kwargs: Any 2870 ) -> NoPagination: 2871 return NoPagination(parameters={}) 2872 2873 def create_oauth_authenticator( 2874 self, model: OAuthAuthenticatorModel, config: Config, **kwargs: Any 2875 ) -> DeclarativeOauth2Authenticator: 2876 profile_assertion = ( 2877 self._create_component_from_model(model.profile_assertion, config=config) 2878 if model.profile_assertion 2879 else None 2880 ) 2881 2882 refresh_token_error_status_codes, refresh_token_error_key, refresh_token_error_values = ( 2883 self._get_refresh_token_error_information(model) 2884 ) 2885 if model.refresh_token_updater: 2886 # ignore type error because fixing it would have a lot of dependencies, revisit later 2887 return DeclarativeSingleUseRefreshTokenOauth2Authenticator( # type: ignore 2888 config, 2889 InterpolatedString.create( 2890 model.token_refresh_endpoint, # type: ignore 2891 parameters=model.parameters or {}, 2892 ).eval(config), 2893 access_token_name=InterpolatedString.create( 2894 model.access_token_name or "access_token", parameters=model.parameters or {} 2895 ).eval(config), 2896 refresh_token_name=model.refresh_token_updater.refresh_token_name, 2897 expires_in_name=InterpolatedString.create( 2898 model.expires_in_name or "expires_in", parameters=model.parameters or {} 2899 ).eval(config), 2900 client_id_name=InterpolatedString.create( 2901 model.client_id_name or "client_id", parameters=model.parameters or {} 2902 ).eval(config), 2903 client_id=InterpolatedString.create( 2904 model.client_id, parameters=model.parameters or {} 2905 ).eval(config) 2906 if model.client_id 2907 else model.client_id, 2908 client_secret_name=InterpolatedString.create( 2909 model.client_secret_name or "client_secret", parameters=model.parameters or {} 2910 ).eval(config), 2911 client_secret=InterpolatedString.create( 2912 model.client_secret, parameters=model.parameters or {} 2913 ).eval(config) 2914 if model.client_secret 2915 else model.client_secret, 2916 access_token_config_path=model.refresh_token_updater.access_token_config_path, 2917 refresh_token_config_path=model.refresh_token_updater.refresh_token_config_path, 2918 token_expiry_date_config_path=model.refresh_token_updater.token_expiry_date_config_path, 2919 grant_type_name=InterpolatedString.create( 2920 model.grant_type_name or "grant_type", parameters=model.parameters or {} 2921 ).eval(config), 2922 grant_type=InterpolatedString.create( 2923 model.grant_type or "refresh_token", parameters=model.parameters or {} 2924 ).eval(config), 2925 refresh_request_body=InterpolatedMapping( 2926 model.refresh_request_body or {}, parameters=model.parameters or {} 2927 ).eval(config), 2928 refresh_request_headers=InterpolatedMapping( 2929 model.refresh_request_headers or {}, parameters=model.parameters or {} 2930 ).eval(config), 2931 send_refresh_request_as_query_params=bool( 2932 model.send_refresh_request_as_query_params 2933 ), 2934 scopes=model.scopes, 2935 token_expiry_date_format=model.token_expiry_date_format, 2936 token_expiry_is_time_of_expiration=bool(model.token_expiry_date_format), 2937 message_repository=self._message_repository, 2938 refresh_token_error_status_codes=refresh_token_error_status_codes, 2939 refresh_token_error_key=refresh_token_error_key, 2940 refresh_token_error_values=refresh_token_error_values, 2941 ) 2942 # ignore type error because fixing it would have a lot of dependencies, revisit later 2943 return DeclarativeOauth2Authenticator( # type: ignore 2944 access_token_name=model.access_token_name or "access_token", 2945 access_token_value=model.access_token_value, 2946 client_id_name=model.client_id_name or "client_id", 2947 client_id=model.client_id, 2948 client_secret_name=model.client_secret_name or "client_secret", 2949 client_secret=model.client_secret, 2950 expires_in_name=model.expires_in_name or "expires_in", 2951 grant_type_name=model.grant_type_name or "grant_type", 2952 grant_type=model.grant_type or "refresh_token", 2953 refresh_request_body=model.refresh_request_body, 2954 refresh_request_headers=model.refresh_request_headers, 2955 send_refresh_request_as_query_params=bool(model.send_refresh_request_as_query_params), 2956 refresh_token_name=model.refresh_token_name or "refresh_token", 2957 refresh_token=model.refresh_token, 2958 scopes=model.scopes, 2959 token_expiry_date=model.token_expiry_date, 2960 token_expiry_date_format=model.token_expiry_date_format, 2961 token_expiry_is_time_of_expiration=bool(model.token_expiry_date_format), 2962 token_refresh_endpoint=model.token_refresh_endpoint, 2963 config=config, 2964 parameters=model.parameters or {}, 2965 message_repository=self._message_repository, 2966 profile_assertion=profile_assertion, 2967 use_profile_assertion=model.use_profile_assertion, 2968 refresh_token_error_status_codes=refresh_token_error_status_codes, 2969 refresh_token_error_key=refresh_token_error_key, 2970 refresh_token_error_values=refresh_token_error_values, 2971 ) 2972 2973 @staticmethod 2974 def _get_refresh_token_error_information( 2975 model: OAuthAuthenticatorModel, 2976 ) -> Tuple[Tuple[int, ...], str, Tuple[str, ...]]: 2977 """ 2978 In a previous version of the CDK, the auth error as config_error was only done if a refresh token updater was 2979 defined. As a transition, we added those fields on the OAuthAuthenticatorModel. This method ensures that the 2980 information is defined only once and return the right fields. 2981 """ 2982 refresh_token_updater = model.refresh_token_updater 2983 is_defined_on_refresh_token_updated = refresh_token_updater and ( 2984 refresh_token_updater.refresh_token_error_status_codes 2985 or refresh_token_updater.refresh_token_error_key 2986 or refresh_token_updater.refresh_token_error_values 2987 ) 2988 is_defined_on_oauth_authenticator = ( 2989 model.refresh_token_error_status_codes 2990 or model.refresh_token_error_key 2991 or model.refresh_token_error_values 2992 ) 2993 if is_defined_on_refresh_token_updated and is_defined_on_oauth_authenticator: 2994 raise ValueError( 2995 "refresh_token_error should either be defined on the OAuthAuthenticatorModel or the RefreshTokenUpdaterModel, not both" 2996 ) 2997 2998 if is_defined_on_refresh_token_updated: 2999 not_optional_refresh_token_updater: RefreshTokenUpdaterModel = refresh_token_updater # type: ignore # we know from the condition that this is not None 3000 return ( 3001 tuple(not_optional_refresh_token_updater.refresh_token_error_status_codes) 3002 if not_optional_refresh_token_updater.refresh_token_error_status_codes 3003 else (), 3004 not_optional_refresh_token_updater.refresh_token_error_key or "", 3005 tuple(not_optional_refresh_token_updater.refresh_token_error_values) 3006 if not_optional_refresh_token_updater.refresh_token_error_values 3007 else (), 3008 ) 3009 elif is_defined_on_oauth_authenticator: 3010 return ( 3011 tuple(model.refresh_token_error_status_codes) 3012 if model.refresh_token_error_status_codes 3013 else (), 3014 model.refresh_token_error_key or "", 3015 tuple(model.refresh_token_error_values) if model.refresh_token_error_values else (), 3016 ) 3017 3018 # returning default values we think cover most cases 3019 return (400,), "error", ("invalid_grant", "invalid_permissions") 3020 3021 def create_offset_increment( 3022 self, 3023 model: OffsetIncrementModel, 3024 config: Config, 3025 decoder: Decoder, 3026 extractor_model: Optional[Union[CustomRecordExtractorModel, DpathExtractorModel]] = None, 3027 **kwargs: Any, 3028 ) -> OffsetIncrement: 3029 if isinstance(decoder, PaginationDecoderDecorator): 3030 inner_decoder = decoder.decoder 3031 else: 3032 inner_decoder = decoder 3033 decoder = PaginationDecoderDecorator(decoder=decoder) 3034 3035 if self._is_supported_decoder_for_pagination(inner_decoder): 3036 decoder_to_use = decoder 3037 else: 3038 raise ValueError( 3039 self._UNSUPPORTED_DECODER_ERROR.format(decoder_type=type(inner_decoder)) 3040 ) 3041 3042 # Ideally we would instantiate the runtime extractor from highest most level (in this case the SimpleRetriever) 3043 # so that it can be shared by OffSetIncrement and RecordSelector. However, due to how we instantiate the 3044 # decoder with various decorators here, but not in create_record_selector, it is simpler to retain existing 3045 # behavior by having two separate extractors with identical behavior since they use the same extractor model. 3046 # When we have more time to investigate we can look into reusing the same component. 3047 extractor = ( 3048 self._create_component_from_model( 3049 model=extractor_model, config=config, decoder=decoder_to_use 3050 ) 3051 if extractor_model 3052 else None 3053 ) 3054 3055 # Pydantic v1 Union type coercion can convert int to string depending on Union order. 3056 # If page_size is a string that represents an integer (not an interpolation), convert it back. 3057 page_size = model.page_size 3058 if isinstance(page_size, str) and page_size.isdigit(): 3059 page_size = int(page_size) 3060 3061 return OffsetIncrement( 3062 page_size=page_size, 3063 config=config, 3064 decoder=decoder_to_use, 3065 extractor=extractor, 3066 inject_on_first_request=model.inject_on_first_request or False, 3067 parameters=model.parameters or {}, 3068 ) 3069 3070 def create_page_increment( 3071 self, 3072 model: PageIncrementModel, 3073 config: Config, 3074 decoder: Optional[Decoder] = None, 3075 extractor_model: Optional[Union[CustomRecordExtractorModel, DpathExtractorModel]] = None, 3076 **kwargs: Any, 3077 ) -> PageIncrement: 3078 # Like OffsetIncrement, we instantiate a separate extractor with identical behavior to the 3079 # RecordSelector's so the strategy can count the raw records in the response. This ensures 3080 # pagination is driven by the API's page size, not the post-filter record count. 3081 extractor = ( 3082 self._create_component_from_model(model=extractor_model, config=config, decoder=decoder) 3083 if extractor_model 3084 else None 3085 ) 3086 3087 # Pydantic v1 Union type coercion can convert int to string depending on Union order. 3088 # If page_size is a string that represents an integer (not an interpolation), convert it back. 3089 page_size = model.page_size 3090 if isinstance(page_size, str) and page_size.isdigit(): 3091 page_size = int(page_size) 3092 3093 return PageIncrement( 3094 page_size=page_size, 3095 config=config, 3096 start_from_page=model.start_from_page or 0, 3097 inject_on_first_request=model.inject_on_first_request or False, 3098 extractor=extractor, 3099 parameters=model.parameters or {}, 3100 ) 3101 3102 def create_parent_stream_config( 3103 self, model: ParentStreamConfigModel, config: Config, *, stream_name: str, **kwargs: Any 3104 ) -> ParentStreamConfig: 3105 declarative_stream = self._create_component_from_model( 3106 model.stream, 3107 config=config, 3108 is_parent=True, 3109 **kwargs, 3110 ) 3111 request_option = ( 3112 self._create_component_from_model(model.request_option, config=config) 3113 if model.request_option 3114 else None 3115 ) 3116 3117 if model.lazy_read_pointer and any("*" in pointer for pointer in model.lazy_read_pointer): 3118 raise ValueError( 3119 "The '*' wildcard in 'lazy_read_pointer' is not supported — only direct paths are allowed." 3120 ) 3121 3122 model_lazy_read_pointer: List[Union[InterpolatedString, str]] = ( 3123 [x for x in model.lazy_read_pointer] if model.lazy_read_pointer else [] 3124 ) 3125 3126 return ParentStreamConfig( 3127 parent_key=model.parent_key, 3128 request_option=request_option, 3129 stream=declarative_stream, 3130 partition_field=model.partition_field, 3131 config=config, 3132 incremental_dependency=model.incremental_dependency or False, 3133 parameters=model.parameters or {}, 3134 extra_fields=model.extra_fields, 3135 lazy_read_pointer=model_lazy_read_pointer, 3136 ) 3137 3138 def create_properties_from_endpoint( 3139 self, model: PropertiesFromEndpointModel, config: Config, **kwargs: Any 3140 ) -> PropertiesFromEndpoint: 3141 retriever = self._create_component_from_model( 3142 model=model.retriever, 3143 config=config, 3144 name="dynamic_properties", 3145 primary_key=None, 3146 stream_slicer=None, 3147 transformations=[], 3148 use_cache=True, # Enable caching on the HttpRequester/HttpClient because the properties endpoint will be called for every slice being processed, and it is highly unlikely for the response to different 3149 ) 3150 return PropertiesFromEndpoint( 3151 property_field_path=model.property_field_path, 3152 retriever=retriever, 3153 config=config, 3154 parameters=model.parameters or {}, 3155 ) 3156 3157 def create_property_chunking( 3158 self, model: PropertyChunkingModel, config: Config, **kwargs: Any 3159 ) -> PropertyChunking: 3160 record_merge_strategy = ( 3161 self._create_component_from_model( 3162 model=model.record_merge_strategy, config=config, **kwargs 3163 ) 3164 if model.record_merge_strategy 3165 else None 3166 ) 3167 3168 property_limit_type: PropertyLimitType 3169 match model.property_limit_type: 3170 case PropertyLimitTypeModel.property_count: 3171 property_limit_type = PropertyLimitType.property_count 3172 case PropertyLimitTypeModel.characters: 3173 property_limit_type = PropertyLimitType.characters 3174 case _: 3175 raise ValueError(f"Invalid PropertyLimitType {property_limit_type}") 3176 3177 return PropertyChunking( 3178 property_limit_type=property_limit_type, 3179 property_limit=model.property_limit, 3180 record_merge_strategy=record_merge_strategy, 3181 config=config, 3182 parameters=model.parameters or {}, 3183 ) 3184 3185 def create_query_properties( 3186 self, model: QueryPropertiesModel, config: Config, *, stream_name: str, **kwargs: Any 3187 ) -> QueryProperties: 3188 if isinstance(model.property_list, list): 3189 property_list = model.property_list 3190 else: 3191 property_list = self._create_component_from_model( 3192 model=model.property_list, config=config, **kwargs 3193 ) 3194 3195 property_chunking = ( 3196 self._create_component_from_model( 3197 model=model.property_chunking, config=config, **kwargs 3198 ) 3199 if model.property_chunking 3200 else None 3201 ) 3202 3203 property_selector = ( 3204 self._create_component_from_model( 3205 model=model.property_selector, config=config, stream_name=stream_name, **kwargs 3206 ) 3207 if model.property_selector 3208 else None 3209 ) 3210 3211 return QueryProperties( 3212 property_list=property_list, 3213 always_include_properties=model.always_include_properties, 3214 property_chunking=property_chunking, 3215 property_selector=property_selector, 3216 config=config, 3217 parameters=model.parameters or {}, 3218 ) 3219 3220 def create_json_schema_property_selector( 3221 self, 3222 model: JsonSchemaPropertySelectorModel, 3223 config: Config, 3224 *, 3225 stream_name: str, 3226 **kwargs: Any, 3227 ) -> JsonSchemaPropertySelector: 3228 configured_stream = self._stream_name_to_configured_stream.get(stream_name) 3229 3230 transformations = [] 3231 if model.transformations: 3232 for transformation_model in model.transformations: 3233 transformations.append( 3234 self._create_component_from_model(model=transformation_model, config=config) 3235 ) 3236 3237 return JsonSchemaPropertySelector( 3238 configured_stream=configured_stream, 3239 properties_transformations=transformations, 3240 config=config, 3241 parameters=model.parameters or {}, 3242 ) 3243 3244 @staticmethod 3245 def create_record_filter( 3246 model: RecordFilterModel, config: Config, **kwargs: Any 3247 ) -> RecordFilter: 3248 return RecordFilter( 3249 condition=model.condition or "", config=config, parameters=model.parameters or {} 3250 ) 3251 3252 @staticmethod 3253 def create_request_path(model: RequestPathModel, config: Config, **kwargs: Any) -> RequestPath: 3254 return RequestPath(parameters={}) 3255 3256 @staticmethod 3257 def create_request_option( 3258 model: RequestOptionModel, config: Config, **kwargs: Any 3259 ) -> RequestOption: 3260 inject_into = RequestOptionType(model.inject_into.value) 3261 field_path: Optional[List[Union[InterpolatedString, str]]] = ( 3262 [ 3263 InterpolatedString.create(segment, parameters=kwargs.get("parameters", {})) 3264 for segment in model.field_path 3265 ] 3266 if model.field_path 3267 else None 3268 ) 3269 field_name = ( 3270 InterpolatedString.create(model.field_name, parameters=kwargs.get("parameters", {})) 3271 if model.field_name 3272 else None 3273 ) 3274 return RequestOption( 3275 field_name=field_name, 3276 field_path=field_path, 3277 inject_into=inject_into, 3278 parameters=kwargs.get("parameters", {}), 3279 ) 3280 3281 def create_record_selector( 3282 self, 3283 model: RecordSelectorModel, 3284 config: Config, 3285 *, 3286 name: str, 3287 transformations: List[RecordTransformation] | None = None, 3288 decoder: Decoder | None = None, 3289 client_side_incremental_sync_cursor: Optional[Cursor] = None, 3290 file_uploader: Optional[DefaultFileUploader] = None, 3291 **kwargs: Any, 3292 ) -> RecordSelector: 3293 extractor = self._create_component_from_model( 3294 model=model.extractor, decoder=decoder, config=config 3295 ) 3296 record_filter = ( 3297 self._create_component_from_model(model.record_filter, config=config) 3298 if model.record_filter 3299 else None 3300 ) 3301 3302 transform_before_filtering = ( 3303 False if model.transform_before_filtering is None else model.transform_before_filtering 3304 ) 3305 if client_side_incremental_sync_cursor: 3306 record_filter = ClientSideIncrementalRecordFilterDecorator( 3307 config=config, 3308 parameters=model.parameters, 3309 condition=model.record_filter.condition 3310 if (model.record_filter and hasattr(model.record_filter, "condition")) 3311 else None, 3312 cursor=client_side_incremental_sync_cursor, 3313 ) 3314 transform_before_filtering = ( 3315 True 3316 if model.transform_before_filtering is None 3317 else model.transform_before_filtering 3318 ) 3319 3320 if model.schema_normalization is None: 3321 # default to no schema normalization if not set 3322 model.schema_normalization = SchemaNormalizationModel.None_ 3323 3324 schema_normalization = ( 3325 TypeTransformer(SCHEMA_TRANSFORMER_TYPE_MAPPING[model.schema_normalization]) 3326 if isinstance(model.schema_normalization, SchemaNormalizationModel) 3327 else self._create_component_from_model(model.schema_normalization, config=config) # type: ignore[arg-type] # custom normalization model expected here 3328 ) 3329 3330 return RecordSelector( 3331 extractor=extractor, 3332 name=name, 3333 config=config, 3334 record_filter=record_filter, 3335 transformations=transformations or [], 3336 file_uploader=file_uploader, 3337 schema_normalization=schema_normalization, 3338 parameters=model.parameters or {}, 3339 transform_before_filtering=transform_before_filtering, 3340 ) 3341 3342 @staticmethod 3343 def create_remove_fields( 3344 model: RemoveFieldsModel, config: Config, **kwargs: Any 3345 ) -> RemoveFields: 3346 return RemoveFields( 3347 field_pointers=model.field_pointers, condition=model.condition or "", parameters={} 3348 ) 3349 3350 def create_selective_authenticator( 3351 self, model: SelectiveAuthenticatorModel, config: Config, **kwargs: Any 3352 ) -> DeclarativeAuthenticator: 3353 authenticators = { 3354 name: self._create_component_from_model(model=auth, config=config) 3355 for name, auth in model.authenticators.items() 3356 } 3357 # SelectiveAuthenticator will return instance of DeclarativeAuthenticator or raise ValueError error 3358 return SelectiveAuthenticator( # type: ignore[abstract] 3359 config=config, 3360 authenticators=authenticators, 3361 authenticator_selection_path=model.authenticator_selection_path, 3362 **kwargs, 3363 ) 3364 3365 @staticmethod 3366 def create_legacy_session_token_authenticator( 3367 model: LegacySessionTokenAuthenticatorModel, config: Config, *, url_base: str, **kwargs: Any 3368 ) -> LegacySessionTokenAuthenticator: 3369 return LegacySessionTokenAuthenticator( 3370 api_url=url_base, 3371 header=model.header, 3372 login_url=model.login_url, 3373 password=model.password or "", 3374 session_token=model.session_token or "", 3375 session_token_response_key=model.session_token_response_key or "", 3376 username=model.username or "", 3377 validate_session_url=model.validate_session_url, 3378 config=config, 3379 parameters=model.parameters or {}, 3380 ) 3381 3382 def create_simple_retriever( 3383 self, 3384 model: SimpleRetrieverModel, 3385 config: Config, 3386 *, 3387 name: str, 3388 primary_key: Optional[Union[str, List[str], List[List[str]]]], 3389 request_options_provider: Optional[RequestOptionsProvider] = None, 3390 cursor: Optional[Cursor] = None, 3391 has_stop_condition_cursor: bool = False, 3392 is_client_side_incremental_sync: bool = False, 3393 transformations: List[RecordTransformation], 3394 file_uploader: Optional[DefaultFileUploader] = None, 3395 incremental_sync: Optional[ 3396 Union[IncrementingCountCursorModel, DatetimeBasedCursorModel] 3397 ] = None, 3398 use_cache: Optional[bool] = None, 3399 log_formatter: Optional[Callable[[Response], Any]] = None, 3400 partition_router: Optional[PartitionRouter] = None, 3401 **kwargs: Any, 3402 ) -> SimpleRetriever: 3403 def _get_url(req: Requester) -> str: 3404 """ 3405 Closure to get the URL from the requester. This is used to get the URL in the case of a lazy retriever. 3406 This is needed because the URL is not set until the requester is created. 3407 """ 3408 3409 _url: str = ( 3410 model.requester.url 3411 if hasattr(model.requester, "url") and model.requester.url is not None 3412 else req.get_url(stream_state=None, stream_slice=None, next_page_token=None) 3413 ) 3414 _url_base: str = ( 3415 model.requester.url_base 3416 if hasattr(model.requester, "url_base") and model.requester.url_base is not None 3417 else req.get_url_base(stream_state=None, stream_slice=None, next_page_token=None) 3418 ) 3419 3420 return _url or _url_base 3421 3422 if cursor is None: 3423 cursor = FinalStateCursor(name, None, self._message_repository) 3424 3425 decoder = ( 3426 self._create_component_from_model(model=model.decoder, config=config) 3427 if model.decoder 3428 else JsonDecoder(parameters={}) 3429 ) 3430 record_selector = self._create_component_from_model( 3431 model=model.record_selector, 3432 name=name, 3433 config=config, 3434 decoder=decoder, 3435 transformations=transformations, 3436 client_side_incremental_sync_cursor=cursor if is_client_side_incremental_sync else None, 3437 file_uploader=file_uploader, 3438 ) 3439 3440 query_properties: Optional[QueryProperties] = None 3441 query_properties_key: Optional[str] = None 3442 self._ensure_query_properties_to_model(model.requester) 3443 if self._has_query_properties_in_request_parameters(model.requester): 3444 # It is better to be explicit about an error if PropertiesFromEndpoint is defined in multiple 3445 # places instead of default to request_parameters which isn't clearly documented 3446 if ( 3447 hasattr(model.requester, "fetch_properties_from_endpoint") 3448 and model.requester.fetch_properties_from_endpoint 3449 ): 3450 raise ValueError( 3451 f"PropertiesFromEndpoint should only be specified once per stream, but found in {model.requester.type}.fetch_properties_from_endpoint and {model.requester.type}.request_parameters" 3452 ) 3453 3454 query_properties_definitions = [] 3455 for key, request_parameter in model.requester.request_parameters.items(): # type: ignore # request_parameters is already validated to be a Mapping using _has_query_properties_in_request_parameters() 3456 if isinstance(request_parameter, QueryPropertiesModel): 3457 query_properties_key = key 3458 query_properties_definitions.append(request_parameter) 3459 3460 if len(query_properties_definitions) > 1: 3461 raise ValueError( 3462 f"request_parameters only supports defining one QueryProperties field, but found {len(query_properties_definitions)} usages" 3463 ) 3464 3465 if len(query_properties_definitions) == 1: 3466 query_properties = self._create_component_from_model( 3467 model=query_properties_definitions[0], stream_name=name, config=config 3468 ) 3469 3470 # Removes QueryProperties components from the interpolated mappings because it has been designed 3471 # to be used by the SimpleRetriever and will be resolved from the provider from the slice directly 3472 # instead of through jinja interpolation 3473 if hasattr(model.requester, "request_parameters") and isinstance( 3474 model.requester.request_parameters, Mapping 3475 ): 3476 model.requester.request_parameters = self._remove_query_properties( 3477 model.requester.request_parameters 3478 ) 3479 elif ( 3480 hasattr(model.requester, "fetch_properties_from_endpoint") 3481 and model.requester.fetch_properties_from_endpoint 3482 ): 3483 # todo: Deprecate this condition once dependent connectors migrate to query_properties 3484 query_properties_definition = QueryPropertiesModel( 3485 type="QueryProperties", 3486 property_list=model.requester.fetch_properties_from_endpoint, 3487 always_include_properties=None, 3488 property_chunking=None, 3489 ) # type: ignore # $parameters has a default value 3490 3491 query_properties = self.create_query_properties( 3492 model=query_properties_definition, 3493 stream_name=name, 3494 config=config, 3495 ) 3496 elif hasattr(model.requester, "query_properties") and model.requester.query_properties: 3497 query_properties = self.create_query_properties( 3498 model=model.requester.query_properties, 3499 stream_name=name, 3500 config=config, 3501 ) 3502 3503 requester = self._create_component_from_model( 3504 model=model.requester, 3505 decoder=decoder, 3506 name=name, 3507 query_properties_key=query_properties_key, 3508 use_cache=use_cache, 3509 config=config, 3510 ) 3511 3512 if not request_options_provider: 3513 request_options_provider = DefaultRequestOptionsProvider(parameters={}) 3514 if isinstance(request_options_provider, DefaultRequestOptionsProvider) and isinstance( 3515 partition_router, PartitionRouter 3516 ): 3517 request_options_provider = partition_router 3518 3519 paginator = ( 3520 self._create_component_from_model( 3521 model=model.paginator, 3522 config=config, 3523 url_base=_get_url(requester), 3524 extractor_model=model.record_selector.extractor, 3525 decoder=decoder, 3526 cursor_used_for_stop_condition=cursor if has_stop_condition_cursor else None, 3527 ) 3528 if model.paginator 3529 else NoPagination(parameters={}) 3530 ) 3531 3532 ignore_stream_slicer_parameters_on_paginated_requests = ( 3533 model.ignore_stream_slicer_parameters_on_paginated_requests or False 3534 ) 3535 3536 if ( 3537 model.partition_router 3538 and isinstance(model.partition_router, SubstreamPartitionRouterModel) 3539 and not bool(self._connector_state_manager.get_stream_state(name, None)) 3540 and any( 3541 parent_stream_config.lazy_read_pointer 3542 for parent_stream_config in model.partition_router.parent_stream_configs 3543 ) 3544 ): 3545 if incremental_sync: 3546 if incremental_sync.type != "DatetimeBasedCursor": 3547 raise ValueError( 3548 f"LazySimpleRetriever only supports DatetimeBasedCursor. Found: {incremental_sync.type}." 3549 ) 3550 3551 elif incremental_sync.step or incremental_sync.cursor_granularity: 3552 raise ValueError( 3553 f"Found more that one slice per parent. LazySimpleRetriever only supports single slice read for stream - {name}." 3554 ) 3555 3556 if model.decoder and model.decoder.type != "JsonDecoder": 3557 raise ValueError( 3558 f"LazySimpleRetriever only supports JsonDecoder. Found: {model.decoder.type}." 3559 ) 3560 3561 return LazySimpleRetriever( 3562 name=name, 3563 paginator=paginator, 3564 primary_key=primary_key, 3565 requester=requester, 3566 record_selector=record_selector, 3567 stream_slicer=_NO_STREAM_SLICING, 3568 request_option_provider=request_options_provider, 3569 config=config, 3570 ignore_stream_slicer_parameters_on_paginated_requests=ignore_stream_slicer_parameters_on_paginated_requests, 3571 parameters=model.parameters or {}, 3572 ) 3573 3574 if ( 3575 model.record_selector.record_filter 3576 and model.pagination_reset 3577 and model.pagination_reset.limits 3578 ): 3579 raise ValueError("PaginationResetLimits are not supported while having record filter.") 3580 3581 return SimpleRetriever( 3582 name=name, 3583 paginator=paginator, 3584 primary_key=primary_key, 3585 requester=requester, 3586 record_selector=record_selector, 3587 stream_slicer=_NO_STREAM_SLICING, 3588 request_option_provider=request_options_provider, 3589 config=config, 3590 ignore_stream_slicer_parameters_on_paginated_requests=ignore_stream_slicer_parameters_on_paginated_requests, 3591 additional_query_properties=query_properties, 3592 log_formatter=self._get_log_formatter(log_formatter, name), 3593 pagination_tracker_factory=self._create_pagination_tracker_factory( 3594 model.pagination_reset, cursor 3595 ), 3596 parameters=model.parameters or {}, 3597 ) 3598 3599 def _create_pagination_tracker_factory( 3600 self, model: Optional[PaginationResetModel], cursor: Cursor 3601 ) -> Callable[[], PaginationTracker]: 3602 if model is None: 3603 return lambda: PaginationTracker() 3604 3605 # Until we figure out a way to use any cursor for PaginationTracker, we will have to have this cursor selector logic 3606 cursor_factory: Callable[[], Optional[ConcurrentCursor]] = lambda: None 3607 if model.action == PaginationResetActionModel.RESET: 3608 # in that case, we will let cursor_factory to return None even if the stream has a cursor 3609 pass 3610 elif model.action == PaginationResetActionModel.SPLIT_USING_CURSOR: 3611 if isinstance(cursor, ConcurrentCursor): 3612 cursor_factory = lambda: cursor.copy_without_state() # type: ignore # the if condition validates that it is a ConcurrentCursor 3613 elif isinstance(cursor, ConcurrentPerPartitionCursor): 3614 cursor_factory = lambda: cursor._cursor_factory.create( # type: ignore # if this becomes a problem, we would need to extract the cursor_factory instantiation logic and make it accessible here 3615 {}, datetime.timedelta(0) 3616 ) 3617 elif not isinstance(cursor, FinalStateCursor): 3618 LOGGER.warning( 3619 "Unknown cursor for PaginationTracker. Pagination resets might not work properly" 3620 ) 3621 else: 3622 raise ValueError(f"Unknown PaginationReset action: {model.action}") 3623 3624 limit = model.limits.number_of_records if model and model.limits else None 3625 return lambda: PaginationTracker(cursor_factory(), limit) 3626 3627 def _get_log_formatter( 3628 self, log_formatter: Callable[[Response], Any] | None, name: str 3629 ) -> Callable[[Response], Any] | None: 3630 if self._should_limit_slices_fetched(): 3631 return ( 3632 ( 3633 lambda response: format_http_message( 3634 response, 3635 f"Stream '{name}' request", 3636 f"Request performed in order to extract records for stream '{name}'", 3637 name, 3638 ) 3639 ) 3640 if not log_formatter 3641 else log_formatter 3642 ) 3643 return None 3644 3645 def _should_limit_slices_fetched(self) -> bool: 3646 """ 3647 Returns True if the number of slices fetched should be limited, False otherwise. 3648 This is used to limit the number of slices fetched during tests. 3649 """ 3650 return bool(self._limit_slices_fetched or self._emit_connector_builder_messages) 3651 3652 @staticmethod 3653 def _has_query_properties_in_request_parameters( 3654 requester: Union[HttpRequesterModel, CustomRequesterModel], 3655 ) -> bool: 3656 if not hasattr(requester, "request_parameters"): 3657 return False 3658 request_parameters = requester.request_parameters 3659 if request_parameters and isinstance(request_parameters, Mapping): 3660 for request_parameter in request_parameters.values(): 3661 if isinstance(request_parameter, QueryPropertiesModel): 3662 return True 3663 return False 3664 3665 @staticmethod 3666 def _remove_query_properties( 3667 request_parameters: Mapping[str, Union[str, QueryPropertiesModel]], 3668 ) -> Mapping[str, str]: 3669 return { 3670 parameter_field: request_parameter 3671 for parameter_field, request_parameter in request_parameters.items() 3672 if not isinstance(request_parameter, QueryPropertiesModel) 3673 } 3674 3675 def create_state_delegating_stream( 3676 self, 3677 model: StateDelegatingStreamModel, 3678 config: Config, 3679 **kwargs: Any, 3680 ) -> DefaultStream: 3681 if ( 3682 model.full_refresh_stream.name != model.name 3683 or model.name != model.incremental_stream.name 3684 ): 3685 raise ValueError( 3686 f"state_delegating_stream, full_refresh_stream name and incremental_stream must have equal names. Instead has {model.name}, {model.full_refresh_stream.name} and {model.incremental_stream.name}." 3687 ) 3688 3689 # Resolve api_retention_period with config context (supports Jinja2 interpolation) 3690 resolved_retention_period: Optional[str] = None 3691 if model.api_retention_period: 3692 interpolated_retention = InterpolatedString.create( 3693 model.api_retention_period, parameters=model.parameters or {} 3694 ) 3695 resolved_value = interpolated_retention.eval(config=config) 3696 if resolved_value: 3697 resolved_retention_period = str(resolved_value) 3698 3699 if resolved_retention_period: 3700 for stream_model in (model.full_refresh_stream, model.incremental_stream): 3701 if isinstance(stream_model.incremental_sync, IncrementingCountCursorModel): 3702 raise ValueError( 3703 f"Stream '{model.name}' uses IncrementingCountCursor which is not supported " 3704 f"with api_retention_period. IncrementingCountCursor does not use datetime-based " 3705 f"cursors, so cursor age validation cannot be performed." 3706 ) 3707 3708 stream_state = self._connector_state_manager.get_stream_state(model.name, None) 3709 3710 if not stream_state: 3711 return self._create_component_from_model( # type: ignore[no-any-return] 3712 model.full_refresh_stream, config=config, **kwargs 3713 ) 3714 3715 incremental_stream: DefaultStream = self._create_component_from_model( 3716 model.incremental_stream, config=config, **kwargs 3717 ) # type: ignore[assignment] 3718 3719 # Only run cursor age validation for streams that are in the configured 3720 # catalog (or when no catalog was provided, e.g. during discover / connector 3721 # builder). Streams not selected by the user but instantiated as parent-stream 3722 # dependencies must not go through this path because it emits state messages 3723 # that the destination does not know about, causing "Stream not found" crashes. 3724 stream_is_in_catalog = ( 3725 not self._stream_name_to_configured_stream # no catalog → validate by default 3726 or model.name in self._stream_name_to_configured_stream 3727 ) 3728 if resolved_retention_period and stream_is_in_catalog: 3729 full_refresh_stream: DefaultStream = self._create_component_from_model( 3730 model.full_refresh_stream, config=config, **kwargs 3731 ) # type: ignore[assignment] 3732 if self._is_cursor_older_than_retention_period( 3733 stream_state, 3734 full_refresh_stream.cursor, 3735 incremental_stream.cursor, 3736 resolved_retention_period, 3737 model.name, 3738 ): 3739 # Clear state BEFORE constructing the full_refresh_stream so that 3740 # its cursor starts from start_date instead of the stale cursor. 3741 self._connector_state_manager.update_state_for_stream(model.name, None, {}) 3742 state_message = self._connector_state_manager.create_state_message(model.name, None) 3743 self._message_repository.emit_message(state_message) 3744 return self._create_component_from_model( # type: ignore[no-any-return] 3745 model.full_refresh_stream, config=config, **kwargs 3746 ) 3747 3748 return incremental_stream 3749 3750 @staticmethod 3751 def _is_cursor_older_than_retention_period( 3752 stream_state: Mapping[str, Any], 3753 full_refresh_cursor: Cursor, 3754 incremental_cursor: Cursor, 3755 api_retention_period: str, 3756 stream_name: str, 3757 ) -> bool: 3758 """Check if the cursor value in the state is older than the API's retention period. 3759 3760 Checks cursors in sequence: full refresh cursor first, then incremental cursor. 3761 FinalStateCursor returns now() for completed full refresh state (NO_CURSOR_STATE_KEY), 3762 which is always within retention, so we use incremental. For other states, it returns 3763 None and we fall back to checking the incremental cursor. 3764 3765 Returns True if the cursor is older than the retention period (should use full refresh). 3766 Returns False if the cursor is within the retention period (safe to use incremental). 3767 """ 3768 retention_duration = parse_duration(api_retention_period) 3769 retention_cutoff = datetime.datetime.now(datetime.timezone.utc) - retention_duration 3770 3771 # Check full refresh cursor first 3772 cursor_datetime = full_refresh_cursor.get_cursor_datetime_from_state(stream_state) 3773 3774 # If full refresh cursor returns None, check incremental cursor 3775 if cursor_datetime is None: 3776 cursor_datetime = incremental_cursor.get_cursor_datetime_from_state(stream_state) 3777 3778 if cursor_datetime is None: 3779 # Neither cursor could parse the state - fall back to full refresh to be safe 3780 return True 3781 3782 if cursor_datetime < retention_cutoff: 3783 logging.warning( 3784 f"Stream '{stream_name}' has a cursor value older than " 3785 f"the API's retention period of {api_retention_period} " 3786 f"(cutoff: {retention_cutoff.isoformat()}). " 3787 f"Falling back to full refresh to avoid data loss." 3788 ) 3789 return True 3790 3791 return False 3792 3793 def _get_state_delegating_stream_model( 3794 self, 3795 model: StateDelegatingStreamModel, 3796 parent_state: Optional[Mapping[str, Any]] = None, 3797 ) -> DeclarativeStreamModel: 3798 """Return the appropriate underlying stream model based on state.""" 3799 return ( 3800 model.incremental_stream 3801 if self._connector_state_manager.get_stream_state(model.name, None) or parent_state 3802 else model.full_refresh_stream 3803 ) 3804 3805 _OPTIONAL_ASYNC_STATUS_FIELDS = {"skipped"} 3806 3807 def _create_async_job_status_mapping( 3808 self, model: AsyncJobStatusMapModel, config: Config, **kwargs: Any 3809 ) -> Mapping[str, AsyncJobStatus]: 3810 api_status_to_cdk_status = {} 3811 for cdk_status, api_statuses in model.dict().items(): 3812 if cdk_status == "type": 3813 # This is an element of the dict because of the typing of the CDK but it is not a CDK status 3814 continue 3815 3816 if api_statuses is None: 3817 if cdk_status in self._OPTIONAL_ASYNC_STATUS_FIELDS: 3818 continue 3819 raise ValueError( 3820 f"Required CDK status '{cdk_status}' has no API statuses mapped. " 3821 f"Please provide at least an empty list for required status fields." 3822 ) 3823 3824 for status in api_statuses: 3825 if status in api_status_to_cdk_status: 3826 raise ValueError( 3827 f"API status {status} is already set for CDK status {cdk_status}. Please ensure API statuses are only provided once" 3828 ) 3829 api_status_to_cdk_status[status] = self._get_async_job_status(cdk_status) 3830 return api_status_to_cdk_status 3831 3832 def _get_async_job_status(self, status: str) -> AsyncJobStatus: 3833 match status: 3834 case "running": 3835 return AsyncJobStatus.RUNNING 3836 case "completed": 3837 return AsyncJobStatus.COMPLETED 3838 case "failed": 3839 return AsyncJobStatus.FAILED 3840 case "timeout": 3841 return AsyncJobStatus.TIMED_OUT 3842 case "skipped": 3843 return AsyncJobStatus.SKIPPED 3844 case _: 3845 raise ValueError(f"Unsupported CDK status {status}") 3846 3847 def create_async_retriever( 3848 self, 3849 model: AsyncRetrieverModel, 3850 config: Config, 3851 *, 3852 name: str, 3853 primary_key: Optional[ 3854 Union[str, List[str], List[List[str]]] 3855 ], # this seems to be needed to match create_simple_retriever 3856 stream_slicer: Optional[StreamSlicer], 3857 client_side_incremental_sync: Optional[Dict[str, Any]] = None, 3858 transformations: List[RecordTransformation], 3859 **kwargs: Any, 3860 ) -> AsyncRetriever: 3861 if model.download_target_requester and not model.download_target_extractor: 3862 raise ValueError( 3863 f"`download_target_extractor` required if using a `download_target_requester`" 3864 ) 3865 3866 def _get_download_retriever( 3867 requester: Requester, extractor: RecordExtractor, _decoder: Decoder 3868 ) -> SimpleRetriever: 3869 # We create a record selector for the download retriever 3870 # with no schema normalization and no transformations, neither record filter 3871 # as all this occurs in the record_selector of the AsyncRetriever 3872 record_selector = RecordSelector( 3873 extractor=extractor, 3874 name=name, 3875 record_filter=None, 3876 transformations=[], 3877 schema_normalization=TypeTransformer(TransformConfig.NoTransform), 3878 config=config, 3879 parameters={}, 3880 ) 3881 paginator = ( 3882 self._create_component_from_model( 3883 model=model.download_paginator, 3884 decoder=_decoder, 3885 config=config, 3886 url_base="", 3887 ) 3888 if model.download_paginator 3889 else NoPagination(parameters={}) 3890 ) 3891 3892 return SimpleRetriever( 3893 requester=requester, 3894 record_selector=record_selector, 3895 primary_key=None, 3896 name=name, 3897 paginator=paginator, 3898 config=config, 3899 parameters={}, 3900 log_formatter=self._get_log_formatter(None, name), 3901 ) 3902 3903 def _get_job_timeout() -> datetime.timedelta: 3904 user_defined_timeout: Optional[int] = ( 3905 int( 3906 InterpolatedString.create( 3907 str(model.polling_job_timeout), 3908 parameters={}, 3909 ).eval(config) 3910 ) 3911 if model.polling_job_timeout 3912 else None 3913 ) 3914 3915 # check for user defined timeout during the test read or 15 minutes 3916 test_read_timeout = datetime.timedelta(minutes=user_defined_timeout or 15) 3917 # default value for non-connector builder is 60 minutes. 3918 default_sync_timeout = datetime.timedelta(minutes=user_defined_timeout or 60) 3919 3920 return ( 3921 test_read_timeout if self._emit_connector_builder_messages else default_sync_timeout 3922 ) 3923 3924 decoder = ( 3925 self._create_component_from_model(model=model.decoder, config=config) 3926 if model.decoder 3927 else JsonDecoder(parameters={}) 3928 ) 3929 record_selector = self._create_component_from_model( 3930 model=model.record_selector, 3931 config=config, 3932 decoder=decoder, 3933 name=name, 3934 transformations=transformations, 3935 client_side_incremental_sync=client_side_incremental_sync, 3936 ) 3937 3938 stream_slicer = stream_slicer or SinglePartitionRouter(parameters={}) 3939 if self._should_limit_slices_fetched(): 3940 stream_slicer = cast( 3941 StreamSlicer, 3942 StreamSlicerTestReadDecorator( 3943 wrapped_slicer=stream_slicer, 3944 maximum_number_of_slices=self._limit_slices_fetched or 5, 3945 ), 3946 ) 3947 3948 creation_requester = self._create_component_from_model( 3949 model=model.creation_requester, 3950 decoder=decoder, 3951 config=config, 3952 name=f"job creation - {name}", 3953 ) 3954 polling_requester = self._create_component_from_model( 3955 model=model.polling_requester, 3956 decoder=decoder, 3957 config=config, 3958 name=f"job polling - {name}", 3959 ) 3960 job_download_components_name = f"job download - {name}" 3961 download_decoder = ( 3962 self._create_component_from_model(model=model.download_decoder, config=config) 3963 if model.download_decoder 3964 else JsonDecoder(parameters={}) 3965 ) 3966 download_extractor = ( 3967 self._create_component_from_model( 3968 model=model.download_extractor, 3969 config=config, 3970 decoder=download_decoder, 3971 parameters=model.parameters, 3972 ) 3973 if model.download_extractor 3974 else DpathExtractor( 3975 [], 3976 config=config, 3977 decoder=download_decoder, 3978 parameters=model.parameters or {}, 3979 ) 3980 ) 3981 download_requester = self._create_component_from_model( 3982 model=model.download_requester, 3983 decoder=download_decoder, 3984 config=config, 3985 name=job_download_components_name, 3986 ) 3987 download_retriever = _get_download_retriever( 3988 download_requester, download_extractor, download_decoder 3989 ) 3990 abort_requester = ( 3991 self._create_component_from_model( 3992 model=model.abort_requester, 3993 decoder=decoder, 3994 config=config, 3995 name=f"job abort - {name}", 3996 ) 3997 if model.abort_requester 3998 else None 3999 ) 4000 delete_requester = ( 4001 self._create_component_from_model( 4002 model=model.delete_requester, 4003 decoder=decoder, 4004 config=config, 4005 name=f"job delete - {name}", 4006 ) 4007 if model.delete_requester 4008 else None 4009 ) 4010 download_target_requester = ( 4011 self._create_component_from_model( 4012 model=model.download_target_requester, 4013 decoder=decoder, 4014 config=config, 4015 name=f"job extract_url - {name}", 4016 ) 4017 if model.download_target_requester 4018 else None 4019 ) 4020 status_extractor = self._create_component_from_model( 4021 model=model.status_extractor, decoder=decoder, config=config, name=name 4022 ) 4023 download_target_extractor = ( 4024 self._create_component_from_model( 4025 model=model.download_target_extractor, 4026 decoder=decoder, 4027 config=config, 4028 name=name, 4029 ) 4030 if model.download_target_extractor 4031 else None 4032 ) 4033 4034 job_repository: AsyncJobRepository = AsyncHttpJobRepository( 4035 creation_requester=creation_requester, 4036 polling_requester=polling_requester, 4037 download_retriever=download_retriever, 4038 download_target_requester=download_target_requester, 4039 abort_requester=abort_requester, 4040 delete_requester=delete_requester, 4041 status_extractor=status_extractor, 4042 status_mapping=self._create_async_job_status_mapping(model.status_mapping, config), 4043 download_target_extractor=download_target_extractor, 4044 job_timeout=_get_job_timeout(), 4045 ) 4046 4047 failed_retry_wait_time_in_seconds: Optional[int] = ( 4048 int( 4049 InterpolatedString.create( 4050 str(model.failed_retry_wait_time_in_seconds), 4051 parameters={}, 4052 ).eval(config) 4053 ) 4054 if model.failed_retry_wait_time_in_seconds 4055 else None 4056 ) 4057 4058 async_job_partition_router = AsyncJobPartitionRouter( 4059 job_orchestrator_factory=lambda stream_slices: AsyncJobOrchestrator( 4060 job_repository, 4061 stream_slices, 4062 self._job_tracker, 4063 self._message_repository, 4064 # FIXME work would need to be done here in order to detect if a stream as a parent stream that is bulk 4065 has_bulk_parent=False, 4066 # set the `job_max_retry` to 1 for the `Connector Builder`` use-case. 4067 # `None` == default retry is set to 3 attempts, under the hood. 4068 job_max_retry=1 if self._emit_connector_builder_messages else None, 4069 failed_retry_wait_time_in_seconds=failed_retry_wait_time_in_seconds, 4070 ), 4071 stream_slicer=stream_slicer, 4072 config=config, 4073 parameters=model.parameters or {}, 4074 ) 4075 4076 return AsyncRetriever( 4077 record_selector=record_selector, 4078 stream_slicer=async_job_partition_router, 4079 config=config, 4080 parameters=model.parameters or {}, 4081 ) 4082 4083 def create_spec(self, model: SpecModel, config: Config, **kwargs: Any) -> Spec: 4084 config_migrations = [ 4085 self._create_component_from_model(migration, config) 4086 for migration in ( 4087 model.config_normalization_rules.config_migrations 4088 if ( 4089 model.config_normalization_rules 4090 and model.config_normalization_rules.config_migrations 4091 ) 4092 else [] 4093 ) 4094 ] 4095 config_transformations = [ 4096 self._create_component_from_model(transformation, config) 4097 for transformation in ( 4098 model.config_normalization_rules.transformations 4099 if ( 4100 model.config_normalization_rules 4101 and model.config_normalization_rules.transformations 4102 ) 4103 else [] 4104 ) 4105 ] 4106 config_validations = [ 4107 self._create_component_from_model(validation, config) 4108 for validation in ( 4109 model.config_normalization_rules.validations 4110 if ( 4111 model.config_normalization_rules 4112 and model.config_normalization_rules.validations 4113 ) 4114 else [] 4115 ) 4116 ] 4117 4118 return Spec( 4119 connection_specification=model.connection_specification, 4120 documentation_url=model.documentation_url, 4121 advanced_auth=model.advanced_auth, 4122 parameters={}, 4123 config_migrations=config_migrations, 4124 config_transformations=config_transformations, 4125 config_validations=config_validations, 4126 ) 4127 4128 def create_substream_partition_router( 4129 self, 4130 model: SubstreamPartitionRouterModel, 4131 config: Config, 4132 *, 4133 stream_name: str, 4134 **kwargs: Any, 4135 ) -> SubstreamPartitionRouter: 4136 parent_stream_configs = [] 4137 if model.parent_stream_configs: 4138 parent_stream_configs.extend( 4139 [ 4140 self.create_parent_stream_config_with_substream_wrapper( 4141 model=parent_stream_config, config=config, stream_name=stream_name, **kwargs 4142 ) 4143 for parent_stream_config in model.parent_stream_configs 4144 ] 4145 ) 4146 4147 return SubstreamPartitionRouter( 4148 parent_stream_configs=parent_stream_configs, 4149 parameters=model.parameters or {}, 4150 config=config, 4151 ) 4152 4153 def create_parent_stream_config_with_substream_wrapper( 4154 self, model: ParentStreamConfigModel, config: Config, *, stream_name: str, **kwargs: Any 4155 ) -> Any: 4156 child_state = self._connector_state_manager.get_stream_state(stream_name, None) 4157 4158 parent_state: Optional[Mapping[str, Any]] = ( 4159 child_state if model.incremental_dependency and child_state else None 4160 ) 4161 connector_state_manager = self._instantiate_parent_stream_state_manager( 4162 child_state, config, model, parent_state 4163 ) 4164 4165 substream_factory = ModelToComponentFactory( 4166 custom_components_trusted=self._custom_components_trusted, 4167 connector_state_manager=connector_state_manager, 4168 limit_pages_fetched_per_slice=self._limit_pages_fetched_per_slice, 4169 limit_slices_fetched=self._limit_slices_fetched, 4170 emit_connector_builder_messages=self._emit_connector_builder_messages, 4171 disable_retries=self._disable_retries, 4172 disable_cache=self._disable_cache, 4173 message_repository=StateFilteringMessageRepository( 4174 LogAppenderMessageRepositoryDecorator( 4175 { 4176 "airbyte_cdk": {"stream": {"is_substream": True}}, 4177 "http": {"is_auxiliary": True}, 4178 }, 4179 self._message_repository, 4180 self._evaluate_log_level(self._emit_connector_builder_messages), 4181 ), 4182 ), 4183 api_budget=self._api_budget, 4184 # Share the authenticator registry so parent and child streams draw from the 4185 # same token quota counters 4186 rate_limited_authenticators=self._rate_limited_authenticators, 4187 ) 4188 4189 return substream_factory.create_parent_stream_config( 4190 model=model, config=config, stream_name=stream_name, **kwargs 4191 ) 4192 4193 def _instantiate_parent_stream_state_manager( 4194 self, 4195 child_state: MutableMapping[str, Any], 4196 config: Config, 4197 model: ParentStreamConfigModel, 4198 parent_state: Optional[Mapping[str, Any]] = None, 4199 ) -> ConnectorStateManager: 4200 """ 4201 With DefaultStream, the state needs to be provided during __init__ of the cursor as opposed to the 4202 `set_initial_state` flow that existed for the declarative cursors. This state is taken from 4203 self._connector_state_manager.get_stream_state (`self` being a newly created ModelToComponentFactory to account 4204 for the MessageRepository being different). So we need to pass a ConnectorStateManager to the 4205 ModelToComponentFactory that has the parent states. This method populates this if there is a child state and if 4206 incremental_dependency is set. 4207 """ 4208 if model.incremental_dependency and child_state: 4209 parent_stream_name = model.stream.name or "" 4210 extracted_parent_state = ConcurrentPerPartitionCursor.get_parent_state( 4211 child_state, parent_stream_name 4212 ) 4213 4214 if not extracted_parent_state: 4215 extracted_parent_state = ConcurrentPerPartitionCursor.get_global_state( 4216 child_state, parent_stream_name 4217 ) 4218 4219 if not extracted_parent_state and not isinstance(extracted_parent_state, dict): 4220 cursor_values = child_state.values() 4221 if cursor_values and len(cursor_values) == 1: 4222 incremental_sync_model: Union[ 4223 DatetimeBasedCursorModel, 4224 IncrementingCountCursorModel, 4225 ] = ( 4226 model.stream.incremental_sync # type: ignore # if we are there, it is because there is incremental_dependency and therefore there is an incremental_sync on the parent stream 4227 if isinstance(model.stream, DeclarativeStreamModel) 4228 else self._get_state_delegating_stream_model( 4229 model.stream, parent_state=parent_state 4230 ).incremental_sync 4231 ) 4232 cursor_field = InterpolatedString.create( 4233 incremental_sync_model.cursor_field, 4234 parameters=incremental_sync_model.parameters or {}, 4235 ).eval(config) 4236 extracted_parent_state = AirbyteStateMessage( 4237 type=AirbyteStateType.STREAM, 4238 stream=AirbyteStreamState( 4239 stream_descriptor=StreamDescriptor( 4240 name=parent_stream_name, namespace=None 4241 ), 4242 stream_state=AirbyteStateBlob( 4243 {cursor_field: list(cursor_values)[0]} 4244 ), 4245 ), 4246 ) 4247 return ConnectorStateManager([extracted_parent_state] if extracted_parent_state else []) 4248 4249 return ConnectorStateManager([]) 4250 4251 @staticmethod 4252 def create_wait_time_from_header( 4253 model: WaitTimeFromHeaderModel, config: Config, **kwargs: Any 4254 ) -> WaitTimeFromHeaderBackoffStrategy: 4255 return WaitTimeFromHeaderBackoffStrategy( 4256 header=model.header, 4257 parameters=model.parameters or {}, 4258 config=config, 4259 regex=model.regex, 4260 max_waiting_time_in_seconds=model.max_waiting_time_in_seconds 4261 if model.max_waiting_time_in_seconds is not None 4262 else None, 4263 ) 4264 4265 @staticmethod 4266 def create_wait_until_time_from_header( 4267 model: WaitUntilTimeFromHeaderModel, config: Config, **kwargs: Any 4268 ) -> WaitUntilTimeFromHeaderBackoffStrategy: 4269 return WaitUntilTimeFromHeaderBackoffStrategy( 4270 header=model.header, 4271 parameters=model.parameters or {}, 4272 config=config, 4273 min_wait=model.min_wait, 4274 regex=model.regex, 4275 ) 4276 4277 def get_message_repository(self) -> MessageRepository: 4278 return self._message_repository 4279 4280 def _evaluate_log_level(self, emit_connector_builder_messages: bool) -> Level: 4281 return Level.DEBUG if emit_connector_builder_messages else Level.INFO 4282 4283 @staticmethod 4284 def create_components_mapping_definition( 4285 model: ComponentMappingDefinitionModel, config: Config, **kwargs: Any 4286 ) -> ComponentMappingDefinition: 4287 interpolated_value = InterpolatedString.create( 4288 model.value, parameters=model.parameters or {} 4289 ) 4290 field_path = [ 4291 InterpolatedString.create(path, parameters=model.parameters or {}) 4292 for path in model.field_path 4293 ] 4294 return ComponentMappingDefinition( 4295 field_path=field_path, # type: ignore[arg-type] # field_path can be str and InterpolatedString 4296 value=interpolated_value, 4297 value_type=ModelToComponentFactory._json_schema_type_name_to_type(model.value_type), 4298 create_or_update=model.create_or_update, 4299 condition=model.condition, 4300 parameters=model.parameters or {}, 4301 ) 4302 4303 def create_http_components_resolver( 4304 self, model: HttpComponentsResolverModel, config: Config, stream_name: Optional[str] = None 4305 ) -> Any: 4306 retriever = self._create_component_from_model( 4307 model=model.retriever, 4308 config=config, 4309 name=f"{stream_name if stream_name else '__http_components_resolver'}", 4310 primary_key=None, 4311 stream_slicer=self._build_stream_slicer_from_partition_router(model.retriever, config), 4312 transformations=[], 4313 ) 4314 4315 components_mapping = [] 4316 for component_mapping_definition_model in model.components_mapping: 4317 if component_mapping_definition_model.condition: 4318 raise ValueError("`condition` is only supported for `ConfigComponentsResolver`") 4319 components_mapping.append( 4320 self._create_component_from_model( 4321 model=component_mapping_definition_model, 4322 value_type=ModelToComponentFactory._json_schema_type_name_to_type( 4323 component_mapping_definition_model.value_type 4324 ), 4325 config=config, 4326 ) 4327 ) 4328 4329 return HttpComponentsResolver( 4330 retriever=retriever, 4331 stream_slicer=self._build_stream_slicer_from_partition_router(model.retriever, config), 4332 config=config, 4333 components_mapping=components_mapping, 4334 parameters=model.parameters or {}, 4335 ) 4336 4337 @staticmethod 4338 def create_stream_config( 4339 model: StreamConfigModel, config: Config, **kwargs: Any 4340 ) -> StreamConfig: 4341 model_configs_pointer: List[Union[InterpolatedString, str]] = ( 4342 [x for x in model.configs_pointer] if model.configs_pointer else [] 4343 ) 4344 4345 return StreamConfig( 4346 configs_pointer=model_configs_pointer, 4347 default_values=model.default_values, 4348 parameters=model.parameters or {}, 4349 ) 4350 4351 def create_config_components_resolver( 4352 self, 4353 model: ConfigComponentsResolverModel, 4354 config: Config, 4355 ) -> Any: 4356 model_stream_configs = ( 4357 model.stream_config if isinstance(model.stream_config, list) else [model.stream_config] 4358 ) 4359 4360 stream_configs = [ 4361 self._create_component_from_model( 4362 stream_config, config=config, parameters=model.parameters or {} 4363 ) 4364 for stream_config in model_stream_configs 4365 ] 4366 4367 components_mapping = [ 4368 self._create_component_from_model( 4369 model=components_mapping_definition_model, 4370 value_type=ModelToComponentFactory._json_schema_type_name_to_type( 4371 components_mapping_definition_model.value_type 4372 ), 4373 config=config, 4374 parameters=model.parameters, 4375 ) 4376 for components_mapping_definition_model in model.components_mapping 4377 ] 4378 4379 return ConfigComponentsResolver( 4380 stream_configs=stream_configs, 4381 config=config, 4382 components_mapping=components_mapping, 4383 parameters=model.parameters or {}, 4384 ) 4385 4386 def create_parametrized_components_resolver( 4387 self, 4388 model: ParametrizedComponentsResolverModel, 4389 config: Config, 4390 ) -> ParametrizedComponentsResolver: 4391 stream_parameters = StreamParametersDefinition( 4392 list_of_parameters_for_stream=model.stream_parameters.list_of_parameters_for_stream 4393 ) 4394 4395 components_mapping = [] 4396 for components_mapping_definition_model in model.components_mapping: 4397 if components_mapping_definition_model.condition: 4398 raise ValueError("`condition` is only supported for `ConfigComponentsResolver`") 4399 components_mapping.append( 4400 self._create_component_from_model( 4401 model=components_mapping_definition_model, 4402 value_type=ModelToComponentFactory._json_schema_type_name_to_type( 4403 components_mapping_definition_model.value_type 4404 ), 4405 config=config, 4406 ) 4407 ) 4408 return ParametrizedComponentsResolver( 4409 stream_parameters=stream_parameters, 4410 config=config, 4411 components_mapping=components_mapping, 4412 parameters=model.parameters or {}, 4413 ) 4414 4415 _UNSUPPORTED_DECODER_ERROR = ( 4416 "Specified decoder of {decoder_type} is not supported for pagination." 4417 "Please set as `JsonDecoder`, `XmlDecoder`, or a `CompositeRawDecoder` with an inner_parser of `JsonParser` or `GzipParser` instead." 4418 "If using `GzipParser`, please ensure that the lowest level inner_parser is a `JsonParser`." 4419 ) 4420 4421 def _is_supported_decoder_for_pagination(self, decoder: Decoder) -> bool: 4422 if isinstance(decoder, (JsonDecoder, XmlDecoder)): 4423 return True 4424 elif isinstance(decoder, CompositeRawDecoder): 4425 return self._is_supported_parser_for_pagination(decoder.parser) 4426 else: 4427 return False 4428 4429 def _is_supported_parser_for_pagination(self, parser: Parser) -> bool: 4430 if isinstance(parser, JsonParser): 4431 return True 4432 elif isinstance(parser, GzipParser): 4433 return isinstance(parser.inner_parser, JsonParser) 4434 else: 4435 return False 4436 4437 def create_http_api_budget( 4438 self, model: HTTPAPIBudgetModel, config: Config, **kwargs: Any 4439 ) -> HttpAPIBudget: 4440 policies = [ 4441 self._create_component_from_model(model=policy, config=config) 4442 for policy in model.policies 4443 ] 4444 4445 return HttpAPIBudget( 4446 policies=policies, 4447 ratelimit_reset_header=model.ratelimit_reset_header or "ratelimit-reset", 4448 ratelimit_remaining_header=model.ratelimit_remaining_header or "ratelimit-remaining", 4449 status_codes_for_ratelimit_hit=model.status_codes_for_ratelimit_hit or [429], 4450 ) 4451 4452 def create_fixed_window_call_rate_policy( 4453 self, model: FixedWindowCallRatePolicyModel, config: Config, **kwargs: Any 4454 ) -> FixedWindowCallRatePolicy: 4455 matchers = [ 4456 self._create_component_from_model(model=matcher, config=config) 4457 for matcher in model.matchers 4458 ] 4459 4460 # Set the initial reset timestamp to 10 days from now. 4461 # This value will be updated by the first request. 4462 return FixedWindowCallRatePolicy( 4463 next_reset_ts=datetime.datetime.now() + datetime.timedelta(days=10), 4464 period=parse_duration(model.period), 4465 call_limit=model.call_limit, 4466 matchers=matchers, 4467 ) 4468 4469 def create_file_uploader( 4470 self, model: FileUploaderModel, config: Config, **kwargs: Any 4471 ) -> FileUploader: 4472 name = "File Uploader" 4473 requester = self._create_component_from_model( 4474 model=model.requester, 4475 config=config, 4476 name=name, 4477 **kwargs, 4478 ) 4479 download_target_extractor = self._create_component_from_model( 4480 model=model.download_target_extractor, 4481 config=config, 4482 name=name, 4483 **kwargs, 4484 ) 4485 emit_connector_builder_messages = self._emit_connector_builder_messages 4486 file_uploader = DefaultFileUploader( 4487 requester=requester, 4488 download_target_extractor=download_target_extractor, 4489 config=config, 4490 file_writer=NoopFileWriter() 4491 if emit_connector_builder_messages 4492 else LocalFileSystemFileWriter(), 4493 parameters=model.parameters or {}, 4494 filename_extractor=model.filename_extractor if model.filename_extractor else None, 4495 ) 4496 4497 return ( 4498 ConnectorBuilderFileUploader(file_uploader) 4499 if emit_connector_builder_messages 4500 else file_uploader 4501 ) 4502 4503 def create_moving_window_call_rate_policy( 4504 self, model: MovingWindowCallRatePolicyModel, config: Config, **kwargs: Any 4505 ) -> MovingWindowCallRatePolicy: 4506 rates = [ 4507 self._create_component_from_model(model=rate, config=config) for rate in model.rates 4508 ] 4509 matchers = [ 4510 self._create_component_from_model(model=matcher, config=config) 4511 for matcher in model.matchers 4512 ] 4513 return MovingWindowCallRatePolicy( 4514 rates=rates, 4515 matchers=matchers, 4516 ) 4517 4518 def create_unlimited_call_rate_policy( 4519 self, model: UnlimitedCallRatePolicyModel, config: Config, **kwargs: Any 4520 ) -> UnlimitedCallRatePolicy: 4521 matchers = [ 4522 self._create_component_from_model(model=matcher, config=config) 4523 for matcher in model.matchers 4524 ] 4525 4526 return UnlimitedCallRatePolicy( 4527 matchers=matchers, 4528 ) 4529 4530 def create_rate(self, model: RateModel, config: Config, **kwargs: Any) -> Rate: 4531 interpolated_limit = InterpolatedString.create(str(model.limit), parameters={}) 4532 return Rate( 4533 limit=int(interpolated_limit.eval(config=config)), 4534 interval=parse_duration(model.interval), 4535 ) 4536 4537 def create_http_request_matcher( 4538 self, model: HttpRequestRegexMatcherModel, config: Config, **kwargs: Any 4539 ) -> HttpRequestRegexMatcher: 4540 weight = model.weight 4541 if weight is not None: 4542 if isinstance(weight, str): 4543 weight = int(InterpolatedString.create(weight, parameters={}).eval(config)) 4544 else: 4545 weight = int(weight) 4546 if weight < 1: 4547 raise ValueError(f"weight must be >= 1, got {weight}") 4548 return HttpRequestRegexMatcher( 4549 method=model.method, 4550 url_base=model.url_base, 4551 url_path_pattern=model.url_path_pattern, 4552 params=model.params, 4553 headers=model.headers, 4554 weight=weight, 4555 ) 4556 4557 def create_rate_limited_multiple_token_authenticator( 4558 self, 4559 model: RateLimitedMultipleTokenAuthenticatorModel, 4560 config: Config, 4561 **kwargs: Any, 4562 ) -> RateLimitedMultipleTokenAuthenticator: 4563 if isinstance(model.tokens, str): 4564 tokens_value = InterpolatedString.create(model.tokens, parameters={}).eval(config) 4565 delimiter = model.token_delimiter or "," 4566 tokens = [ 4567 token.strip() for token in str(tokens_value).split(delimiter) if token.strip() 4568 ] 4569 else: 4570 tokens = [ 4571 token_value 4572 for token in model.tokens 4573 if ( 4574 token_value := str( 4575 InterpolatedString.create(token, parameters={}).eval(config) 4576 ).strip() 4577 ) 4578 ] 4579 4580 quota_specs = [ 4581 { 4582 "name": quota_model.name, 4583 "remaining_path": quota_model.remaining_path, 4584 "reset_path": quota_model.reset_path, 4585 "limit_path": quota_model.limit_path, 4586 "matchers": [ 4587 { 4588 "method": matcher_model.method, 4589 "url_base": matcher_model.url_base, 4590 "url_path_pattern": matcher_model.url_path_pattern, 4591 "params": matcher_model.params, 4592 "headers": matcher_model.headers, 4593 "weight": matcher_model.weight, 4594 } 4595 for matcher_model in quota_model.matchers or [] 4596 ], 4597 } 4598 for quota_model in model.quotas 4599 ] 4600 4601 quota_status_url = str( 4602 InterpolatedString.create(model.quota_status_source.url, parameters={}).eval(config) 4603 ) 4604 quota_status_http_method = ( 4605 model.quota_status_source.http_method.value 4606 if model.quota_status_source.http_method 4607 else "GET" 4608 ) 4609 quota_status_headers = { 4610 key: str(InterpolatedString.create(value, parameters={}).eval(config)) 4611 for key, value in (model.quota_status_source.request_headers or {}).items() 4612 } 4613 auth_method = model.auth_method or "Bearer" 4614 header = model.header or "Authorization" 4615 max_wait_time_str = str( 4616 InterpolatedString.create(model.max_wait_time or "PT2H", parameters={}).eval(config) 4617 ) 4618 max_wait_time = parse_duration(max_wait_time_str) 4619 if not isinstance(max_wait_time, datetime.timedelta): 4620 raise ValueError( 4621 f"max_wait_time must be a fixed-length ISO 8601 duration (e.g. 'PT2H'); " 4622 f"calendar-unit durations like '{max_wait_time_str}' are not supported" 4623 ) 4624 budget_reserve_fraction = ( 4625 model.budget_reserve_fraction if model.budget_reserve_fraction is not None else 0.1 4626 ) 4627 budget_min_reserve = ( 4628 model.budget_min_reserve if model.budget_min_reserve is not None else 50 4629 ) 4630 4631 # Reuse the same instance for identical definitions so that all streams share the 4632 # same token quota counters (similar to how api_budget is shared). The key is built 4633 # from the resolved constructor arguments rather than the raw model so that 4634 # stream-specific `$parameters` propagated onto the model (and its nested components) 4635 # cannot break instance sharing. 4636 cache_key = json.dumps( 4637 { 4638 "tokens": tokens, 4639 "quotas": quota_specs, 4640 "quota_status_url": quota_status_url, 4641 "quota_status_http_method": quota_status_http_method, 4642 "quota_status_headers": quota_status_headers, 4643 "auth_method": auth_method, 4644 "header": header, 4645 "max_wait_time": max_wait_time.total_seconds(), 4646 "budget_reserve_fraction": budget_reserve_fraction, 4647 "budget_min_reserve": budget_min_reserve, 4648 }, 4649 sort_keys=True, 4650 ) 4651 if cache_key in self._rate_limited_authenticators: 4652 return self._rate_limited_authenticators[cache_key] 4653 4654 quotas = [ 4655 TokenQuota( 4656 name=quota_model.name, 4657 remaining_path=quota_model.remaining_path, 4658 reset_path=quota_model.reset_path, 4659 limit_path=quota_model.limit_path, 4660 matchers=[ 4661 self.create_http_request_matcher(matcher_model, config) 4662 for matcher_model in quota_model.matchers or [] 4663 ], 4664 ) 4665 for quota_model in model.quotas 4666 ] 4667 4668 authenticator = RateLimitedMultipleTokenAuthenticator( 4669 tokens=tokens, 4670 quotas=quotas, 4671 quota_status_url=quota_status_url, 4672 quota_status_http_method=quota_status_http_method, 4673 quota_status_headers=quota_status_headers, 4674 auth_method=auth_method, 4675 header=header, 4676 max_wait_time=max_wait_time, 4677 budget_reserve_fraction=budget_reserve_fraction, 4678 budget_min_reserve=budget_min_reserve, 4679 ) 4680 self._rate_limited_authenticators[cache_key] = authenticator 4681 return authenticator 4682 4683 def set_api_budget(self, component_definition: ComponentDefinition, config: Config) -> None: 4684 self._api_budget = self.create_component( 4685 model_type=HTTPAPIBudgetModel, component_definition=component_definition, config=config 4686 ) 4687 4688 def create_grouping_partition_router( 4689 self, 4690 model: GroupingPartitionRouterModel, 4691 config: Config, 4692 *, 4693 stream_name: str, 4694 **kwargs: Any, 4695 ) -> GroupingPartitionRouter: 4696 underlying_router = self._create_component_from_model( 4697 model=model.underlying_partition_router, 4698 config=config, 4699 stream_name=stream_name, 4700 **kwargs, 4701 ) 4702 if model.group_size < 1: 4703 raise ValueError(f"Group size must be greater than 0, got {model.group_size}") 4704 4705 # Request options in underlying partition routers are not supported for GroupingPartitionRouter 4706 # because they are specific to individual partitions and cannot be aggregated or handled 4707 # when grouping, potentially leading to incorrect API calls. Any request customization 4708 # should be managed at the stream level through the requester's configuration. 4709 if isinstance(underlying_router, SubstreamPartitionRouter): 4710 if any( 4711 parent_config.request_option 4712 for parent_config in underlying_router.parent_stream_configs 4713 ): 4714 raise ValueError("Request options are not supported for GroupingPartitionRouter.") 4715 4716 if isinstance(underlying_router, ListPartitionRouter): 4717 if underlying_router.request_option: 4718 raise ValueError("Request options are not supported for GroupingPartitionRouter.") 4719 4720 return GroupingPartitionRouter( 4721 group_size=model.group_size, 4722 underlying_partition_router=underlying_router, 4723 deduplicate=model.deduplicate if model.deduplicate is not None else True, 4724 config=config, 4725 ) 4726 4727 def _ensure_query_properties_to_model( 4728 self, requester: Union[HttpRequesterModel, CustomRequesterModel] 4729 ) -> None: 4730 """ 4731 For some reason, it seems like CustomRequesterModel request_parameters stays as dictionaries which means that 4732 the other conditions relying on it being QueryPropertiesModel instead of a dict fail. Here, we migrate them to 4733 proper model. 4734 """ 4735 if not hasattr(requester, "request_parameters"): 4736 return 4737 4738 request_parameters = requester.request_parameters 4739 if request_parameters and isinstance(request_parameters, Dict): 4740 for request_parameter_key in request_parameters.keys(): 4741 request_parameter = request_parameters[request_parameter_key] 4742 if ( 4743 isinstance(request_parameter, Dict) 4744 and request_parameter.get("type") == "QueryProperties" 4745 ): 4746 request_parameters[request_parameter_key] = QueryPropertiesModel.parse_obj( 4747 request_parameter 4748 ) 4749 4750 def _get_catalog_defined_cursor_field( 4751 self, stream_name: str, allow_catalog_defined_cursor_field: bool 4752 ) -> Optional[CursorField]: 4753 if not allow_catalog_defined_cursor_field: 4754 return None 4755 4756 configured_stream = self._stream_name_to_configured_stream.get(stream_name) 4757 4758 # Depending on the operation is being performed, there may not be a configured stream yet. In this 4759 # case we return None which will then use the default cursor field defined on the cursor model. 4760 # We also treat cursor_field: [""] (list with empty string) as no cursor field, since this can 4761 # occur when the platform serializes "no cursor configured" streams incorrectly. 4762 if ( 4763 not configured_stream 4764 or not configured_stream.cursor_field 4765 or not configured_stream.cursor_field[0] 4766 ): 4767 return None 4768 elif len(configured_stream.cursor_field) > 1: 4769 raise ValueError( 4770 f"The `{stream_name}` stream does not support nested cursor_field. Please specify only a single cursor_field for the stream in the configured catalog." 4771 ) 4772 else: 4773 return CursorField( 4774 cursor_field_key=configured_stream.cursor_field[0], 4775 supports_catalog_defined_cursor_field=allow_catalog_defined_cursor_field, 4776 )
ComponentDefinition =
typing.Mapping[str, typing.Any]
SCHEMA_TRANSFORMER_TYPE_MAPPING =
{<SchemaNormalization.None_: 'None'>: <TransformConfig.NoTransform: 1>, <SchemaNormalization.Default: 'Default'>: <TransformConfig.DefaultSchemaNormalization: 2>}
MAX_SLICES =
5
LOGGER =
<Logger airbyte.model_to_component_factory (INFO)>
class
ModelToComponentFactory:
702class ModelToComponentFactory: 703 EPOCH_DATETIME_FORMAT = "%s" 704 705 def __init__( 706 self, 707 limit_pages_fetched_per_slice: Optional[int] = None, 708 limit_slices_fetched: Optional[int] = None, 709 emit_connector_builder_messages: bool = False, 710 disable_retries: bool = False, 711 disable_cache: bool = False, 712 message_repository: Optional[MessageRepository] = None, 713 connector_state_manager: Optional[ConnectorStateManager] = None, 714 max_concurrent_async_job_count: Optional[int] = None, 715 configured_catalog: Optional[ConfiguredAirbyteCatalog] = None, 716 api_budget: Optional[APIBudget] = None, 717 rate_limited_authenticators: Optional[ 718 Dict[str, RateLimitedMultipleTokenAuthenticator] 719 ] = None, 720 custom_components_trusted: bool = True, 721 ): 722 self._init_mappings() 723 self._custom_components_trusted = custom_components_trusted 724 self._limit_pages_fetched_per_slice = limit_pages_fetched_per_slice 725 self._limit_slices_fetched = limit_slices_fetched 726 self._emit_connector_builder_messages = emit_connector_builder_messages 727 self._disable_retries = disable_retries 728 self._disable_cache = disable_cache 729 self._message_repository = message_repository or InMemoryMessageRepository( 730 self._evaluate_log_level(emit_connector_builder_messages) 731 ) 732 self._stream_name_to_configured_stream = self._create_stream_name_to_configured_stream( 733 configured_catalog 734 ) 735 self._connector_state_manager = connector_state_manager or ConnectorStateManager() 736 self._api_budget: Optional[Union[APIBudget]] = api_budget 737 # Shared instances so all streams see the same token quota counters (like api_budget) 738 self._rate_limited_authenticators: Dict[str, RateLimitedMultipleTokenAuthenticator] = ( 739 rate_limited_authenticators if rate_limited_authenticators is not None else {} 740 ) 741 self._job_tracker: JobTracker = JobTracker(max_concurrent_async_job_count or 1) 742 # placeholder for deprecation warnings 743 self._collected_deprecation_logs: List[ConnectorBuilderLogMessage] = [] 744 745 def _init_mappings(self) -> None: 746 self.PYDANTIC_MODEL_TO_CONSTRUCTOR: Mapping[Type[BaseModel], Callable[..., Any]] = { 747 AddedFieldDefinitionModel: self.create_added_field_definition, 748 AddFieldsModel: self.create_add_fields, 749 ApiKeyAuthenticatorModel: self.create_api_key_authenticator, 750 BasicHttpAuthenticatorModel: self.create_basic_http_authenticator, 751 BearerAuthenticatorModel: self.create_bearer_authenticator, 752 CheckStreamModel: self.create_check_stream, 753 DynamicStreamCheckConfigModel: self.create_dynamic_stream_check_config, 754 CheckDynamicStreamModel: self.create_check_dynamic_stream, 755 CompositeErrorHandlerModel: self.create_composite_error_handler, 756 ConcurrencyLevelModel: self.create_concurrency_level, 757 ConfigMigrationModel: self.create_config_migration, 758 ConfigAddFieldsModel: self.create_config_add_fields, 759 ConfigRemapFieldModel: self.create_config_remap_field, 760 ConfigRemoveFieldsModel: self.create_config_remove_fields, 761 ConstantBackoffStrategyModel: self.create_constant_backoff_strategy, 762 CsvDecoderModel: self.create_csv_decoder, 763 CursorPaginationModel: self.create_cursor_pagination, 764 CustomAuthenticatorModel: self.create_custom_component, 765 CustomBackoffStrategyModel: self.create_custom_component, 766 CustomDecoderModel: self.create_custom_component, 767 CustomErrorHandlerModel: self.create_custom_component, 768 CustomRecordExtractorModel: self.create_custom_component, 769 CustomRecordFilterModel: self.create_custom_component, 770 CustomRequesterModel: self.create_custom_component, 771 CustomRetrieverModel: self.create_custom_component, 772 CustomSchemaLoader: self.create_custom_component, 773 CustomSchemaNormalizationModel: self.create_custom_component, 774 CustomStateMigration: self.create_custom_component, 775 CustomPaginationStrategyModel: self.create_custom_component, 776 CustomPartitionRouterModel: self.create_custom_component, 777 CustomTransformationModel: self.create_custom_component, 778 CustomValidationStrategyModel: self.create_custom_component, 779 CustomConfigTransformationModel: self.create_custom_component, 780 DeclarativeStreamModel: self.create_default_stream, 781 DefaultErrorHandlerModel: self.create_default_error_handler, 782 DefaultPaginatorModel: self.create_default_paginator, 783 DpathExtractorModel: self.create_dpath_extractor, 784 DpathValidatorModel: self.create_dpath_validator, 785 ResponseToFileExtractorModel: self.create_response_to_file_extractor, 786 ExponentialBackoffStrategyModel: self.create_exponential_backoff_strategy, 787 SessionTokenAuthenticatorModel: self.create_session_token_authenticator, 788 GroupByKeyMergeStrategyModel: self.create_group_by_key, 789 HttpRequesterModel: self.create_http_requester, 790 HttpResponseFilterModel: self.create_http_response_filter, 791 InlineSchemaLoaderModel: self.create_inline_schema_loader, 792 JsonDecoderModel: self.create_json_decoder, 793 JsonItemsDecoderModel: self.create_json_items_decoder, 794 JsonlDecoderModel: self.create_jsonl_decoder, 795 JsonSchemaPropertySelectorModel: self.create_json_schema_property_selector, 796 GzipDecoderModel: self.create_gzip_decoder, 797 KeysToLowerModel: self.create_keys_to_lower_transformation, 798 KeysToSnakeCaseModel: self.create_keys_to_snake_transformation, 799 KeysReplaceModel: self.create_keys_replace_transformation, 800 FlattenFieldsModel: self.create_flatten_fields, 801 DpathFlattenFieldsModel: self.create_dpath_flatten_fields, 802 IterableDecoderModel: self.create_iterable_decoder, 803 XmlDecoderModel: self.create_xml_decoder, 804 JsonFileSchemaLoaderModel: self.create_json_file_schema_loader, 805 DynamicSchemaLoaderModel: self.create_dynamic_schema_loader, 806 SchemaTypeIdentifierModel: self.create_schema_type_identifier, 807 TypesMapModel: self.create_types_map, 808 ComplexFieldTypeModel: self.create_complex_field_type, 809 JwtAuthenticatorModel: self.create_jwt_authenticator, 810 LegacyToPerPartitionStateMigrationModel: self.create_legacy_to_per_partition_state_migration, 811 ListPartitionRouterModel: self.create_list_partition_router, 812 MinMaxDatetimeModel: self.create_min_max_datetime, 813 NoAuthModel: self.create_no_auth, 814 NoPaginationModel: self.create_no_pagination, 815 OAuthAuthenticatorModel: self.create_oauth_authenticator, 816 OffsetIncrementModel: self.create_offset_increment, 817 PageIncrementModel: self.create_page_increment, 818 ParentStreamConfigModel: self.create_parent_stream_config_with_substream_wrapper, 819 PredicateValidatorModel: self.create_predicate_validator, 820 PropertiesFromEndpointModel: self.create_properties_from_endpoint, 821 PropertyChunkingModel: self.create_property_chunking, 822 QueryPropertiesModel: self.create_query_properties, 823 RecordExpanderModel: self.create_record_expander, 824 RecordFilterModel: self.create_record_filter, 825 RecordSelectorModel: self.create_record_selector, 826 RemoveFieldsModel: self.create_remove_fields, 827 RequestPathModel: self.create_request_path, 828 RequestOptionModel: self.create_request_option, 829 LegacySessionTokenAuthenticatorModel: self.create_legacy_session_token_authenticator, 830 SelectiveAuthenticatorModel: self.create_selective_authenticator, 831 SimpleRetrieverModel: self.create_simple_retriever, 832 StateDelegatingStreamModel: self.create_state_delegating_stream, 833 SpecModel: self.create_spec, 834 SubstreamPartitionRouterModel: self.create_substream_partition_router, 835 ValidateAdheresToSchemaModel: self.create_validate_adheres_to_schema, 836 WaitTimeFromHeaderModel: self.create_wait_time_from_header, 837 WaitUntilTimeFromHeaderModel: self.create_wait_until_time_from_header, 838 AsyncRetrieverModel: self.create_async_retriever, 839 HttpComponentsResolverModel: self.create_http_components_resolver, 840 ConfigComponentsResolverModel: self.create_config_components_resolver, 841 ParametrizedComponentsResolverModel: self.create_parametrized_components_resolver, 842 StreamConfigModel: self.create_stream_config, 843 ComponentMappingDefinitionModel: self.create_components_mapping_definition, 844 ZipfileDecoderModel: self.create_zipfile_decoder, 845 HTTPAPIBudgetModel: self.create_http_api_budget, 846 FileUploaderModel: self.create_file_uploader, 847 FixedWindowCallRatePolicyModel: self.create_fixed_window_call_rate_policy, 848 MovingWindowCallRatePolicyModel: self.create_moving_window_call_rate_policy, 849 UnlimitedCallRatePolicyModel: self.create_unlimited_call_rate_policy, 850 RateModel: self.create_rate, 851 HttpRequestRegexMatcherModel: self.create_http_request_matcher, 852 RateLimitedMultipleTokenAuthenticatorModel: self.create_rate_limited_multiple_token_authenticator, 853 GroupingPartitionRouterModel: self.create_grouping_partition_router, 854 } 855 856 # Needed for the case where we need to perform a second parse on the fields of a custom component 857 self.TYPE_NAME_TO_MODEL = {cls.__name__: cls for cls in self.PYDANTIC_MODEL_TO_CONSTRUCTOR} 858 859 @staticmethod 860 def _create_stream_name_to_configured_stream( 861 configured_catalog: Optional[ConfiguredAirbyteCatalog], 862 ) -> Mapping[str, ConfiguredAirbyteStream]: 863 return ( 864 {stream.stream.name: stream for stream in configured_catalog.streams} 865 if configured_catalog 866 else {} 867 ) 868 869 def create_component( 870 self, 871 model_type: Type[BaseModel], 872 component_definition: ComponentDefinition, 873 config: Config, 874 **kwargs: Any, 875 ) -> Any: 876 """ 877 Takes a given Pydantic model type and Mapping representing a component definition and creates a declarative component and 878 subcomponents which will be used at runtime. This is done by first parsing the mapping into a Pydantic model and then creating 879 creating declarative components from that model. 880 881 :param model_type: The type of declarative component that is being initialized 882 :param component_definition: The mapping that represents a declarative component 883 :param config: The connector config that is provided by the customer 884 :return: The declarative component to be used at runtime 885 """ 886 887 component_type = component_definition.get("type") 888 if component_definition.get("type") != model_type.__name__: 889 raise ValueError( 890 f"Expected manifest component of type {model_type.__name__}, but received {component_type} instead" 891 ) 892 893 declarative_component_model = model_type.parse_obj(component_definition) 894 895 if not isinstance(declarative_component_model, model_type): 896 raise ValueError( 897 f"Expected {model_type.__name__} component, but received {declarative_component_model.__class__.__name__}" 898 ) 899 900 return self._create_component_from_model( 901 model=declarative_component_model, config=config, **kwargs 902 ) 903 904 def _create_component_from_model(self, model: BaseModel, config: Config, **kwargs: Any) -> Any: 905 if model.__class__ not in self.PYDANTIC_MODEL_TO_CONSTRUCTOR: 906 raise ValueError( 907 f"{model.__class__} with attributes {model} is not a valid component type" 908 ) 909 component_constructor = self.PYDANTIC_MODEL_TO_CONSTRUCTOR.get(model.__class__) 910 if not component_constructor: 911 raise ValueError(f"Could not find constructor for {model.__class__}") 912 913 # collect deprecation warnings for supported models. 914 if isinstance(model, BaseModelWithDeprecations): 915 self._collect_model_deprecations(model) 916 917 return component_constructor(model=model, config=config, **kwargs) 918 919 def get_model_deprecations(self) -> List[ConnectorBuilderLogMessage]: 920 """ 921 Returns the deprecation warnings that were collected during the creation of components. 922 """ 923 return self._collected_deprecation_logs 924 925 def _collect_model_deprecations(self, model: BaseModelWithDeprecations) -> None: 926 """ 927 Collects deprecation logs from the given model and appends any new logs to the internal collection. 928 929 This method checks if the provided model has deprecation logs (identified by the presence of the DEPRECATION_LOGS_TAG attribute and a non-None `_deprecation_logs` property). It iterates through each deprecation log in the model and appends it to the `_collected_deprecation_logs` list if it has not already been collected, ensuring that duplicate logs are avoided. 930 931 Args: 932 model (BaseModelWithDeprecations): The model instance from which to collect deprecation logs. 933 """ 934 if hasattr(model, DEPRECATION_LOGS_TAG) and model._deprecation_logs is not None: 935 for log in model._deprecation_logs: 936 # avoid duplicates for deprecation logs observed. 937 if log not in self._collected_deprecation_logs: 938 self._collected_deprecation_logs.append(log) 939 940 def create_config_migration( 941 self, model: ConfigMigrationModel, config: Config 942 ) -> ConfigMigration: 943 transformations: List[ConfigTransformation] = [ 944 self._create_component_from_model(transformation, config) 945 for transformation in model.transformations 946 ] 947 948 return ConfigMigration( 949 description=model.description, 950 transformations=transformations, 951 ) 952 953 def create_config_add_fields( 954 self, model: ConfigAddFieldsModel, config: Config, **kwargs: Any 955 ) -> ConfigAddFields: 956 fields = [self._create_component_from_model(field, config) for field in model.fields] 957 return ConfigAddFields( 958 fields=fields, 959 condition=model.condition or "", 960 ) 961 962 @staticmethod 963 def create_config_remove_fields( 964 model: ConfigRemoveFieldsModel, config: Config, **kwargs: Any 965 ) -> ConfigRemoveFields: 966 return ConfigRemoveFields( 967 field_pointers=model.field_pointers, 968 condition=model.condition or "", 969 ) 970 971 @staticmethod 972 def create_config_remap_field( 973 model: ConfigRemapFieldModel, config: Config, **kwargs: Any 974 ) -> ConfigRemapField: 975 mapping = cast(Mapping[str, Any], model.map) 976 return ConfigRemapField( 977 map=mapping, 978 field_path=model.field_path, 979 config=config, 980 ) 981 982 def create_dpath_validator(self, model: DpathValidatorModel, config: Config) -> DpathValidator: 983 strategy = self._create_component_from_model(model.validation_strategy, config) 984 985 return DpathValidator( 986 field_path=model.field_path, 987 strategy=strategy, 988 ) 989 990 def create_predicate_validator( 991 self, model: PredicateValidatorModel, config: Config 992 ) -> PredicateValidator: 993 strategy = self._create_component_from_model(model.validation_strategy, config) 994 995 return PredicateValidator( 996 value=model.value, 997 strategy=strategy, 998 ) 999 1000 @staticmethod 1001 def create_validate_adheres_to_schema( 1002 model: ValidateAdheresToSchemaModel, config: Config, **kwargs: Any 1003 ) -> ValidateAdheresToSchema: 1004 base_schema = cast(Mapping[str, Any], model.base_schema) 1005 return ValidateAdheresToSchema( 1006 schema=base_schema, 1007 ) 1008 1009 @staticmethod 1010 def create_added_field_definition( 1011 model: AddedFieldDefinitionModel, config: Config, **kwargs: Any 1012 ) -> AddedFieldDefinition: 1013 interpolated_value = InterpolatedString.create( 1014 model.value, parameters=model.parameters or {} 1015 ) 1016 return AddedFieldDefinition( 1017 path=model.path, 1018 value=interpolated_value, 1019 value_type=ModelToComponentFactory._json_schema_type_name_to_type(model.value_type), 1020 parameters=model.parameters or {}, 1021 ) 1022 1023 def create_add_fields(self, model: AddFieldsModel, config: Config, **kwargs: Any) -> AddFields: 1024 added_field_definitions = [ 1025 self._create_component_from_model( 1026 model=added_field_definition_model, 1027 value_type=ModelToComponentFactory._json_schema_type_name_to_type( 1028 added_field_definition_model.value_type 1029 ), 1030 config=config, 1031 ) 1032 for added_field_definition_model in model.fields 1033 ] 1034 return AddFields( 1035 fields=added_field_definitions, 1036 condition=model.condition or "", 1037 parameters=model.parameters or {}, 1038 ) 1039 1040 def create_keys_to_lower_transformation( 1041 self, model: KeysToLowerModel, config: Config, **kwargs: Any 1042 ) -> KeysToLowerTransformation: 1043 return KeysToLowerTransformation() 1044 1045 def create_keys_to_snake_transformation( 1046 self, model: KeysToSnakeCaseModel, config: Config, **kwargs: Any 1047 ) -> KeysToSnakeCaseTransformation: 1048 return KeysToSnakeCaseTransformation() 1049 1050 def create_keys_replace_transformation( 1051 self, model: KeysReplaceModel, config: Config, **kwargs: Any 1052 ) -> KeysReplaceTransformation: 1053 return KeysReplaceTransformation( 1054 old=model.old, new=model.new, parameters=model.parameters or {} 1055 ) 1056 1057 def create_flatten_fields( 1058 self, model: FlattenFieldsModel, config: Config, **kwargs: Any 1059 ) -> FlattenFields: 1060 return FlattenFields( 1061 flatten_lists=model.flatten_lists if model.flatten_lists is not None else True 1062 ) 1063 1064 def create_dpath_flatten_fields( 1065 self, model: DpathFlattenFieldsModel, config: Config, **kwargs: Any 1066 ) -> DpathFlattenFields: 1067 model_field_path: List[Union[InterpolatedString, str]] = [x for x in model.field_path] 1068 key_transformation = ( 1069 KeyTransformation( 1070 config=config, 1071 prefix=model.key_transformation.prefix, 1072 suffix=model.key_transformation.suffix, 1073 parameters=model.parameters or {}, 1074 ) 1075 if model.key_transformation is not None 1076 else None 1077 ) 1078 return DpathFlattenFields( 1079 config=config, 1080 field_path=model_field_path, 1081 delete_origin_value=model.delete_origin_value 1082 if model.delete_origin_value is not None 1083 else False, 1084 replace_record=model.replace_record if model.replace_record is not None else False, 1085 key_transformation=key_transformation, 1086 parameters=model.parameters or {}, 1087 ) 1088 1089 @staticmethod 1090 def _json_schema_type_name_to_type(value_type: Optional[ValueType]) -> Optional[Type[Any]]: 1091 if not value_type: 1092 return None 1093 names_to_types = { 1094 ValueType.string: str, 1095 ValueType.number: float, 1096 ValueType.integer: int, 1097 ValueType.boolean: bool, 1098 } 1099 return names_to_types[value_type] 1100 1101 def create_api_key_authenticator( 1102 self, 1103 model: ApiKeyAuthenticatorModel, 1104 config: Config, 1105 token_provider: Optional[TokenProvider] = None, 1106 **kwargs: Any, 1107 ) -> ApiKeyAuthenticator: 1108 if model.inject_into is None and model.header is None: 1109 raise ValueError( 1110 "Expected either inject_into or header to be set for ApiKeyAuthenticator" 1111 ) 1112 1113 if model.inject_into is not None and model.header is not None: 1114 raise ValueError( 1115 "inject_into and header cannot be set both for ApiKeyAuthenticator - remove the deprecated header option" 1116 ) 1117 1118 if token_provider is not None and model.api_token != "": 1119 raise ValueError( 1120 "If token_provider is set, api_token is ignored and has to be set to empty string." 1121 ) 1122 1123 request_option = ( 1124 self._create_component_from_model( 1125 model.inject_into, config, parameters=model.parameters or {} 1126 ) 1127 if model.inject_into 1128 else RequestOption( 1129 inject_into=RequestOptionType.header, 1130 field_name=model.header or "", 1131 parameters=model.parameters or {}, 1132 ) 1133 ) 1134 1135 return ApiKeyAuthenticator( 1136 token_provider=( 1137 token_provider 1138 if token_provider is not None 1139 else InterpolatedStringTokenProvider( 1140 api_token=model.api_token or "", 1141 config=config, 1142 parameters=model.parameters or {}, 1143 ) 1144 ), 1145 request_option=request_option, 1146 config=config, 1147 parameters=model.parameters or {}, 1148 ) 1149 1150 def create_legacy_to_per_partition_state_migration( 1151 self, 1152 model: LegacyToPerPartitionStateMigrationModel, 1153 config: Mapping[str, Any], 1154 declarative_stream: DeclarativeStreamModel, 1155 ) -> LegacyToPerPartitionStateMigration: 1156 retriever = declarative_stream.retriever 1157 if not isinstance(retriever, (SimpleRetrieverModel, AsyncRetrieverModel)): 1158 raise ValueError( 1159 f"LegacyToPerPartitionStateMigrations can only be applied on a DeclarativeStream with a SimpleRetriever or AsyncRetriever. Got {type(retriever)}" 1160 ) 1161 partition_router = retriever.partition_router 1162 if not isinstance( 1163 partition_router, (SubstreamPartitionRouterModel, CustomPartitionRouterModel) 1164 ): 1165 raise ValueError( 1166 f"LegacyToPerPartitionStateMigrations can only be applied on a SimpleRetriever with a Substream partition router. Got {type(partition_router)}" 1167 ) 1168 if not hasattr(partition_router, "parent_stream_configs"): 1169 raise ValueError( 1170 "LegacyToPerPartitionStateMigrations can only be applied with a parent stream configuration." 1171 ) 1172 1173 if not hasattr(declarative_stream, "incremental_sync"): 1174 raise ValueError( 1175 "LegacyToPerPartitionStateMigrations can only be applied with an incremental_sync configuration." 1176 ) 1177 1178 return LegacyToPerPartitionStateMigration( 1179 partition_router, # type: ignore # was already checked above 1180 declarative_stream.incremental_sync, # type: ignore # was already checked. Migration can be applied only to incremental streams. 1181 config, 1182 declarative_stream.parameters, # type: ignore # different type is expected here Mapping[str, Any], got Dict[str, Any] 1183 ) 1184 1185 def create_session_token_authenticator( 1186 self, model: SessionTokenAuthenticatorModel, config: Config, name: str, **kwargs: Any 1187 ) -> Union[ApiKeyAuthenticator, BearerAuthenticator]: 1188 decoder = ( 1189 self._create_component_from_model(model=model.decoder, config=config) 1190 if model.decoder 1191 else JsonDecoder(parameters={}) 1192 ) 1193 login_requester = self._create_component_from_model( 1194 model=model.login_requester, 1195 config=config, 1196 name=f"{name}_login_requester", 1197 decoder=decoder, 1198 ) 1199 token_provider = SessionTokenProvider( 1200 login_requester=login_requester, 1201 session_token_path=model.session_token_path, 1202 expiration_duration=parse_duration(model.expiration_duration) 1203 if model.expiration_duration 1204 else None, 1205 parameters=model.parameters or {}, 1206 message_repository=self._message_repository, 1207 decoder=decoder, 1208 ) 1209 if model.request_authentication.type == "Bearer": 1210 return ModelToComponentFactory.create_bearer_authenticator( 1211 BearerAuthenticatorModel(type="BearerAuthenticator", api_token=""), # type: ignore # $parameters has a default value 1212 config, 1213 token_provider=token_provider, 1214 ) 1215 else: 1216 # Get the api_token template if specified, default to just the session token 1217 api_token_template = ( 1218 getattr(model.request_authentication, "api_token", None) or "{{ session_token }}" 1219 ) 1220 final_token_provider: TokenProvider = InterpolatedSessionTokenProvider( 1221 config=config, 1222 api_token=api_token_template, 1223 session_token_provider=token_provider, 1224 parameters=model.parameters or {}, 1225 ) 1226 return self.create_api_key_authenticator( 1227 ApiKeyAuthenticatorModel( 1228 type="ApiKeyAuthenticator", 1229 api_token="", 1230 inject_into=model.request_authentication.inject_into, 1231 ), # type: ignore # $parameters and headers default to None 1232 config=config, 1233 token_provider=final_token_provider, 1234 ) 1235 1236 @staticmethod 1237 def create_basic_http_authenticator( 1238 model: BasicHttpAuthenticatorModel, config: Config, **kwargs: Any 1239 ) -> BasicHttpAuthenticator: 1240 return BasicHttpAuthenticator( 1241 password=model.password or "", 1242 username=model.username, 1243 config=config, 1244 parameters=model.parameters or {}, 1245 ) 1246 1247 @staticmethod 1248 def create_bearer_authenticator( 1249 model: BearerAuthenticatorModel, 1250 config: Config, 1251 token_provider: Optional[TokenProvider] = None, 1252 **kwargs: Any, 1253 ) -> BearerAuthenticator: 1254 if token_provider is not None and model.api_token != "": 1255 raise ValueError( 1256 "If token_provider is set, api_token is ignored and has to be set to empty string." 1257 ) 1258 return BearerAuthenticator( 1259 token_provider=( 1260 token_provider 1261 if token_provider is not None 1262 else InterpolatedStringTokenProvider( 1263 api_token=model.api_token or "", 1264 config=config, 1265 parameters=model.parameters or {}, 1266 ) 1267 ), 1268 config=config, 1269 parameters=model.parameters or {}, 1270 ) 1271 1272 @staticmethod 1273 def create_dynamic_stream_check_config( 1274 model: DynamicStreamCheckConfigModel, config: Config, **kwargs: Any 1275 ) -> DynamicStreamCheckConfig: 1276 return DynamicStreamCheckConfig( 1277 dynamic_stream_name=model.dynamic_stream_name, 1278 stream_count=model.stream_count, 1279 ) 1280 1281 def create_check_stream( 1282 self, model: CheckStreamModel, config: Config, **kwargs: Any 1283 ) -> CheckStream: 1284 if model.dynamic_streams_check_configs is None and model.stream_names is None: 1285 raise ValueError( 1286 "Expected either stream_names or dynamic_streams_check_configs to be set for CheckStream" 1287 ) 1288 1289 dynamic_streams_check_configs = ( 1290 [ 1291 self._create_component_from_model(model=dynamic_stream_check_config, config=config) 1292 for dynamic_stream_check_config in model.dynamic_streams_check_configs 1293 ] 1294 if model.dynamic_streams_check_configs 1295 else [] 1296 ) 1297 1298 return CheckStream( 1299 stream_names=model.stream_names or [], 1300 dynamic_streams_check_configs=dynamic_streams_check_configs, 1301 parameters={}, 1302 ) 1303 1304 @staticmethod 1305 def create_check_dynamic_stream( 1306 model: CheckDynamicStreamModel, config: Config, **kwargs: Any 1307 ) -> CheckDynamicStream: 1308 assert model.use_check_availability is not None # for mypy 1309 1310 use_check_availability = model.use_check_availability 1311 1312 return CheckDynamicStream( 1313 stream_count=model.stream_count, 1314 use_check_availability=use_check_availability, 1315 parameters={}, 1316 ) 1317 1318 def create_composite_error_handler( 1319 self, model: CompositeErrorHandlerModel, config: Config, **kwargs: Any 1320 ) -> CompositeErrorHandler: 1321 error_handlers = [ 1322 self._create_component_from_model(model=error_handler_model, config=config) 1323 for error_handler_model in model.error_handlers 1324 ] 1325 return CompositeErrorHandler( 1326 error_handlers=error_handlers, parameters=model.parameters or {} 1327 ) 1328 1329 @staticmethod 1330 def create_concurrency_level( 1331 model: ConcurrencyLevelModel, config: Config, **kwargs: Any 1332 ) -> ConcurrencyLevel: 1333 return ConcurrencyLevel( 1334 default_concurrency=model.default_concurrency, 1335 max_concurrency=model.max_concurrency, 1336 config=config, 1337 parameters={}, 1338 ) 1339 1340 @staticmethod 1341 def apply_stream_state_migrations( 1342 stream_state_migrations: List[Any] | None, stream_state: MutableMapping[str, Any] 1343 ) -> MutableMapping[str, Any]: 1344 if stream_state_migrations: 1345 for state_migration in stream_state_migrations: 1346 if state_migration.should_migrate(stream_state): 1347 # The state variable is expected to be mutable but the migrate method returns an immutable mapping. 1348 stream_state = dict(state_migration.migrate(stream_state)) 1349 return stream_state 1350 1351 def create_concurrent_cursor_from_datetime_based_cursor( 1352 self, 1353 model_type: Type[BaseModel], 1354 component_definition: ComponentDefinition, 1355 stream_name: str, 1356 stream_namespace: Optional[str], 1357 stream_state: MutableMapping[str, Any], 1358 config: Config, 1359 message_repository: Optional[MessageRepository] = None, 1360 runtime_lookback_window: Optional[datetime.timedelta] = None, 1361 **kwargs: Any, 1362 ) -> ConcurrentCursor: 1363 component_type = component_definition.get("type") 1364 if component_definition.get("type") != model_type.__name__: 1365 raise ValueError( 1366 f"Expected manifest component of type {model_type.__name__}, but received {component_type} instead" 1367 ) 1368 1369 # FIXME the interfaces of the concurrent cursor are kind of annoying as they take a `ComponentDefinition` instead of the actual model. This was done because the ConcurrentDeclarativeSource didn't have access to the models [here for example](https://github.com/airbytehq/airbyte-python-cdk/blob/f525803b3fec9329e4cc8478996a92bf884bfde9/airbyte_cdk/sources/declarative/concurrent_declarative_source.py#L354C54-L354C91). So now we have two cases: 1370 # * The ComponentDefinition comes from model.__dict__ in which case we have `parameters` 1371 # * The ComponentDefinition comes from the manifest as a dict in which case we have `$parameters` 1372 # We should change those interfaces to use the model once we clean up the code in CDS at which point the parameter propagation should happen as part of the ModelToComponentFactory. 1373 if "$parameters" not in component_definition and "parameters" in component_definition: 1374 component_definition["$parameters"] = component_definition.get("parameters") # type: ignore # This is a dict 1375 datetime_based_cursor_model = model_type.parse_obj(component_definition) 1376 1377 if not isinstance(datetime_based_cursor_model, DatetimeBasedCursorModel): 1378 raise ValueError( 1379 f"Expected {model_type.__name__} component, but received {datetime_based_cursor_model.__class__.__name__}" 1380 ) 1381 1382 model_parameters = datetime_based_cursor_model.parameters or {} 1383 1384 cursor_field = self._get_catalog_defined_cursor_field( 1385 stream_name=stream_name, 1386 allow_catalog_defined_cursor_field=datetime_based_cursor_model.allow_catalog_defined_cursor_field 1387 or False, 1388 ) 1389 1390 if not cursor_field: 1391 interpolated_cursor_field = InterpolatedString.create( 1392 datetime_based_cursor_model.cursor_field, 1393 parameters=model_parameters, 1394 ) 1395 cursor_field = CursorField( 1396 cursor_field_key=interpolated_cursor_field.eval(config=config), 1397 supports_catalog_defined_cursor_field=datetime_based_cursor_model.allow_catalog_defined_cursor_field 1398 or False, 1399 ) 1400 1401 interpolated_partition_field_start = InterpolatedString.create( 1402 datetime_based_cursor_model.partition_field_start or "start_time", 1403 parameters=model_parameters, 1404 ) 1405 interpolated_partition_field_end = InterpolatedString.create( 1406 datetime_based_cursor_model.partition_field_end or "end_time", 1407 parameters=model_parameters, 1408 ) 1409 1410 slice_boundary_fields = ( 1411 interpolated_partition_field_start.eval(config=config), 1412 interpolated_partition_field_end.eval(config=config), 1413 ) 1414 1415 datetime_format = datetime_based_cursor_model.datetime_format 1416 1417 cursor_granularity = ( 1418 parse_duration(datetime_based_cursor_model.cursor_granularity) 1419 if datetime_based_cursor_model.cursor_granularity 1420 else None 1421 ) 1422 1423 lookback_window = None 1424 interpolated_lookback_window = ( 1425 InterpolatedString.create( 1426 datetime_based_cursor_model.lookback_window, 1427 parameters=model_parameters, 1428 ) 1429 if datetime_based_cursor_model.lookback_window 1430 else None 1431 ) 1432 if interpolated_lookback_window: 1433 evaluated_lookback_window = interpolated_lookback_window.eval(config=config) 1434 if evaluated_lookback_window: 1435 lookback_window = parse_duration(evaluated_lookback_window) 1436 1437 connector_state_converter: DateTimeStreamStateConverter 1438 connector_state_converter = CustomFormatConcurrentStreamStateConverter( 1439 datetime_format=datetime_format, 1440 input_datetime_formats=datetime_based_cursor_model.cursor_datetime_formats, 1441 is_sequential_state=True, # ConcurrentPerPartitionCursor only works with sequential state 1442 cursor_granularity=cursor_granularity, 1443 ) 1444 1445 # Adjusts the stream state by applying the runtime lookback window. 1446 # This is used to ensure correct state handling in case of failed partitions. 1447 stream_state_value = stream_state.get(cursor_field.cursor_field_key) 1448 if runtime_lookback_window and stream_state_value: 1449 new_stream_state = ( 1450 connector_state_converter.parse_timestamp(stream_state_value) 1451 - runtime_lookback_window 1452 ) 1453 stream_state[cursor_field.cursor_field_key] = connector_state_converter.output_format( 1454 new_stream_state 1455 ) 1456 1457 start_date_runtime_value: Union[InterpolatedString, str, MinMaxDatetime] 1458 if isinstance(datetime_based_cursor_model.start_datetime, MinMaxDatetimeModel): 1459 start_date_runtime_value = self.create_min_max_datetime( 1460 model=datetime_based_cursor_model.start_datetime, config=config 1461 ) 1462 else: 1463 start_date_runtime_value = datetime_based_cursor_model.start_datetime 1464 1465 end_date_runtime_value: Optional[Union[InterpolatedString, str, MinMaxDatetime]] 1466 if isinstance(datetime_based_cursor_model.end_datetime, MinMaxDatetimeModel): 1467 end_date_runtime_value = self.create_min_max_datetime( 1468 model=datetime_based_cursor_model.end_datetime, config=config 1469 ) 1470 else: 1471 end_date_runtime_value = datetime_based_cursor_model.end_datetime 1472 1473 interpolated_start_date = MinMaxDatetime.create( 1474 interpolated_string_or_min_max_datetime=start_date_runtime_value, 1475 parameters=datetime_based_cursor_model.parameters, 1476 ) 1477 interpolated_end_date = ( 1478 None 1479 if not end_date_runtime_value 1480 else MinMaxDatetime.create( 1481 end_date_runtime_value, datetime_based_cursor_model.parameters 1482 ) 1483 ) 1484 1485 # If datetime format is not specified then start/end datetime should inherit it from the stream slicer 1486 if not interpolated_start_date.datetime_format: 1487 interpolated_start_date.datetime_format = datetime_format 1488 if interpolated_end_date and not interpolated_end_date.datetime_format: 1489 interpolated_end_date.datetime_format = datetime_format 1490 1491 start_date = interpolated_start_date.get_datetime(config=config) 1492 end_date_provider = ( 1493 partial(interpolated_end_date.get_datetime, config) 1494 if interpolated_end_date 1495 else connector_state_converter.get_end_provider() 1496 ) 1497 1498 if ( 1499 datetime_based_cursor_model.step and not datetime_based_cursor_model.cursor_granularity 1500 ) or ( 1501 not datetime_based_cursor_model.step and datetime_based_cursor_model.cursor_granularity 1502 ): 1503 raise ValueError( 1504 f"If step is defined, cursor_granularity should be as well and vice-versa. " 1505 f"Right now, step is `{datetime_based_cursor_model.step}` and cursor_granularity is `{datetime_based_cursor_model.cursor_granularity}`" 1506 ) 1507 1508 # When step is not defined, default to a step size from the starting date to the present moment 1509 step_length = datetime.timedelta.max 1510 interpolated_step = ( 1511 InterpolatedString.create( 1512 datetime_based_cursor_model.step, 1513 parameters=model_parameters, 1514 ) 1515 if datetime_based_cursor_model.step 1516 else None 1517 ) 1518 if interpolated_step: 1519 evaluated_step = interpolated_step.eval(config) 1520 if evaluated_step: 1521 step_length = parse_duration(evaluated_step) 1522 1523 clamping_strategy: ClampingStrategy = NoClamping() 1524 if datetime_based_cursor_model.clamping: 1525 # While it is undesirable to interpolate within the model factory (as opposed to at runtime), 1526 # it is still better than shifting interpolation low-code concept into the ConcurrentCursor runtime 1527 # object which we want to keep agnostic of being low-code 1528 target = InterpolatedString( 1529 string=datetime_based_cursor_model.clamping.target, 1530 parameters=model_parameters, 1531 ) 1532 evaluated_target = target.eval(config=config) 1533 match evaluated_target: 1534 case "DAY": 1535 clamping_strategy = DayClampingStrategy() 1536 end_date_provider = ClampingEndProvider( 1537 DayClampingStrategy(is_ceiling=False), 1538 end_date_provider, # type: ignore # Having issues w/ inspection for GapType and CursorValueType as shown in existing tests. Confirmed functionality is working in practice 1539 granularity=cursor_granularity or datetime.timedelta(seconds=1), 1540 ) 1541 case "WEEK": 1542 if ( 1543 not datetime_based_cursor_model.clamping.target_details 1544 or "weekday" not in datetime_based_cursor_model.clamping.target_details 1545 ): 1546 raise ValueError( 1547 "Given WEEK clamping, weekday needs to be provided as target_details" 1548 ) 1549 weekday = self._assemble_weekday( 1550 datetime_based_cursor_model.clamping.target_details["weekday"] 1551 ) 1552 clamping_strategy = WeekClampingStrategy(weekday) 1553 end_date_provider = ClampingEndProvider( 1554 WeekClampingStrategy(weekday, is_ceiling=False), 1555 end_date_provider, # type: ignore # Having issues w/ inspection for GapType and CursorValueType as shown in existing tests. Confirmed functionality is working in practice 1556 granularity=cursor_granularity or datetime.timedelta(days=1), 1557 ) 1558 case "MONTH": 1559 clamping_strategy = MonthClampingStrategy() 1560 end_date_provider = ClampingEndProvider( 1561 MonthClampingStrategy(is_ceiling=False), 1562 end_date_provider, # type: ignore # Having issues w/ inspection for GapType and CursorValueType as shown in existing tests. Confirmed functionality is working in practice 1563 granularity=cursor_granularity or datetime.timedelta(days=1), 1564 ) 1565 case _: 1566 raise ValueError( 1567 f"Invalid clamping target {evaluated_target}, expected DAY, WEEK, MONTH" 1568 ) 1569 1570 return ConcurrentCursor( 1571 stream_name=stream_name, 1572 stream_namespace=stream_namespace, 1573 stream_state=stream_state, 1574 message_repository=message_repository or self._message_repository, 1575 connector_state_manager=self._connector_state_manager, 1576 connector_state_converter=connector_state_converter, 1577 cursor_field=cursor_field, 1578 slice_boundary_fields=slice_boundary_fields, 1579 start=start_date, # type: ignore # Having issues w/ inspection for GapType and CursorValueType as shown in existing tests. Confirmed functionality is working in practice 1580 end_provider=end_date_provider, # type: ignore # Having issues w/ inspection for GapType and CursorValueType as shown in existing tests. Confirmed functionality is working in practice 1581 lookback_window=lookback_window, 1582 slice_range=step_length, 1583 cursor_granularity=cursor_granularity, 1584 clamping_strategy=clamping_strategy, 1585 ) 1586 1587 def create_concurrent_cursor_from_incrementing_count_cursor( 1588 self, 1589 model_type: Type[BaseModel], 1590 component_definition: ComponentDefinition, 1591 stream_name: str, 1592 stream_namespace: Optional[str], 1593 stream_state: MutableMapping[str, Any], 1594 config: Config, 1595 message_repository: Optional[MessageRepository] = None, 1596 **kwargs: Any, 1597 ) -> ConcurrentCursor: 1598 component_type = component_definition.get("type") 1599 if component_definition.get("type") != model_type.__name__: 1600 raise ValueError( 1601 f"Expected manifest component of type {model_type.__name__}, but received {component_type} instead" 1602 ) 1603 1604 incrementing_count_cursor_model = model_type.parse_obj(component_definition) 1605 1606 if not isinstance(incrementing_count_cursor_model, IncrementingCountCursorModel): 1607 raise ValueError( 1608 f"Expected {model_type.__name__} component, but received {incrementing_count_cursor_model.__class__.__name__}" 1609 ) 1610 1611 start_value: Union[int, str, None] = incrementing_count_cursor_model.start_value 1612 # Pydantic Union type coercion can convert int 0 to string '0' depending on Union order. 1613 # We need to handle both int and str representations of numeric values. 1614 # Evaluate the InterpolatedString and convert to int for the ConcurrentCursor. 1615 if start_value is not None: 1616 interpolated_start_value = InterpolatedString.create( 1617 str(start_value), # Ensure we pass a string to InterpolatedString.create 1618 parameters=incrementing_count_cursor_model.parameters or {}, 1619 ) 1620 evaluated_start_value: int = int(interpolated_start_value.eval(config=config)) 1621 else: 1622 evaluated_start_value = 0 1623 1624 cursor_field = self._get_catalog_defined_cursor_field( 1625 stream_name=stream_name, 1626 allow_catalog_defined_cursor_field=incrementing_count_cursor_model.allow_catalog_defined_cursor_field 1627 or False, 1628 ) 1629 1630 if not cursor_field: 1631 interpolated_cursor_field = InterpolatedString.create( 1632 incrementing_count_cursor_model.cursor_field, 1633 parameters=incrementing_count_cursor_model.parameters or {}, 1634 ) 1635 cursor_field = CursorField( 1636 cursor_field_key=interpolated_cursor_field.eval(config=config), 1637 supports_catalog_defined_cursor_field=incrementing_count_cursor_model.allow_catalog_defined_cursor_field 1638 or False, 1639 ) 1640 1641 connector_state_converter = IncrementingCountStreamStateConverter( 1642 is_sequential_state=True, # ConcurrentPerPartitionCursor only works with sequential state 1643 ) 1644 1645 return ConcurrentCursor( 1646 stream_name=stream_name, 1647 stream_namespace=stream_namespace, 1648 stream_state=stream_state, 1649 message_repository=message_repository or self._message_repository, 1650 connector_state_manager=self._connector_state_manager, 1651 connector_state_converter=connector_state_converter, 1652 cursor_field=cursor_field, 1653 slice_boundary_fields=None, 1654 start=evaluated_start_value, # type: ignore # Having issues w/ inspection for GapType and CursorValueType as shown in existing tests. Confirmed functionality is working in practice 1655 end_provider=connector_state_converter.get_end_provider(), # type: ignore # Having issues w/ inspection for GapType and CursorValueType as shown in existing tests. Confirmed functionality is working in practice 1656 ) 1657 1658 def _assemble_weekday(self, weekday: str) -> Weekday: 1659 match weekday: 1660 case "MONDAY": 1661 return Weekday.MONDAY 1662 case "TUESDAY": 1663 return Weekday.TUESDAY 1664 case "WEDNESDAY": 1665 return Weekday.WEDNESDAY 1666 case "THURSDAY": 1667 return Weekday.THURSDAY 1668 case "FRIDAY": 1669 return Weekday.FRIDAY 1670 case "SATURDAY": 1671 return Weekday.SATURDAY 1672 case "SUNDAY": 1673 return Weekday.SUNDAY 1674 case _: 1675 raise ValueError(f"Unknown weekday {weekday}") 1676 1677 def create_concurrent_cursor_from_perpartition_cursor( 1678 self, 1679 state_manager: ConnectorStateManager, 1680 model_type: Type[BaseModel], 1681 component_definition: ComponentDefinition, 1682 stream_name: str, 1683 stream_namespace: Optional[str], 1684 config: Config, 1685 stream_state: MutableMapping[str, Any], 1686 partition_router: PartitionRouter, 1687 attempt_to_create_cursor_if_not_provided: bool = False, 1688 **kwargs: Any, 1689 ) -> ConcurrentPerPartitionCursor: 1690 component_type = component_definition.get("type") 1691 if component_definition.get("type") != model_type.__name__: 1692 raise ValueError( 1693 f"Expected manifest component of type {model_type.__name__}, but received {component_type} instead" 1694 ) 1695 1696 # FIXME the interfaces of the concurrent cursor are kind of annoying as they take a `ComponentDefinition` instead of the actual model. This was done because the ConcurrentDeclarativeSource didn't have access to the models [here for example](https://github.com/airbytehq/airbyte-python-cdk/blob/f525803b3fec9329e4cc8478996a92bf884bfde9/airbyte_cdk/sources/declarative/concurrent_declarative_source.py#L354C54-L354C91). So now we have two cases: 1697 # * The ComponentDefinition comes from model.__dict__ in which case we have `parameters` 1698 # * The ComponentDefinition comes from the manifest as a dict in which case we have `$parameters` 1699 # We should change those interfaces to use the model once we clean up the code in CDS at which point the parameter propagation should happen as part of the ModelToComponentFactory. 1700 if "$parameters" not in component_definition and "parameters" in component_definition: 1701 component_definition["$parameters"] = component_definition.get("parameters") # type: ignore # This is a dict 1702 datetime_based_cursor_model = model_type.parse_obj(component_definition) 1703 1704 if not isinstance(datetime_based_cursor_model, DatetimeBasedCursorModel): 1705 raise ValueError( 1706 f"Expected {model_type.__name__} component, but received {datetime_based_cursor_model.__class__.__name__}" 1707 ) 1708 1709 cursor_field = self._get_catalog_defined_cursor_field( 1710 stream_name=stream_name, 1711 allow_catalog_defined_cursor_field=datetime_based_cursor_model.allow_catalog_defined_cursor_field 1712 or False, 1713 ) 1714 1715 if not cursor_field: 1716 interpolated_cursor_field = InterpolatedString.create( 1717 datetime_based_cursor_model.cursor_field, 1718 # FIXME the interfaces of the concurrent cursor are kind of annoying as they take a `ComponentDefinition` instead of the actual model. This was done because the ConcurrentDeclarativeSource didn't have access to the models [here for example](https://github.com/airbytehq/airbyte-python-cdk/blob/f525803b3fec9329e4cc8478996a92bf884bfde9/airbyte_cdk/sources/declarative/concurrent_declarative_source.py#L354C54-L354C91). So now we have two cases: 1719 # * The ComponentDefinition comes from model.__dict__ in which case we have `parameters` 1720 # * The ComponentDefinition comes from the manifest as a dict in which case we have `$parameters` 1721 # We should change those interfaces to use the model once we clean up the code in CDS at which point the parameter propagation should happen as part of the ModelToComponentFactory. 1722 parameters=datetime_based_cursor_model.parameters or {}, 1723 ) 1724 cursor_field = CursorField( 1725 cursor_field_key=interpolated_cursor_field.eval(config=config), 1726 supports_catalog_defined_cursor_field=datetime_based_cursor_model.allow_catalog_defined_cursor_field 1727 or False, 1728 ) 1729 1730 datetime_format = datetime_based_cursor_model.datetime_format 1731 1732 cursor_granularity = ( 1733 parse_duration(datetime_based_cursor_model.cursor_granularity) 1734 if datetime_based_cursor_model.cursor_granularity 1735 else None 1736 ) 1737 1738 connector_state_converter: DateTimeStreamStateConverter 1739 connector_state_converter = CustomFormatConcurrentStreamStateConverter( 1740 datetime_format=datetime_format, 1741 input_datetime_formats=datetime_based_cursor_model.cursor_datetime_formats, 1742 is_sequential_state=True, # ConcurrentPerPartitionCursor only works with sequential state 1743 cursor_granularity=cursor_granularity, 1744 ) 1745 1746 # Create the cursor factory 1747 cursor_factory = ConcurrentCursorFactory( 1748 partial( 1749 self.create_concurrent_cursor_from_datetime_based_cursor, 1750 state_manager=state_manager, 1751 model_type=model_type, 1752 component_definition=component_definition, 1753 stream_name=stream_name, 1754 stream_namespace=stream_namespace, 1755 config=config, 1756 message_repository=NoopMessageRepository(), 1757 ) 1758 ) 1759 1760 # Per-partition state doesn't make sense for GroupingPartitionRouter, so force the global state 1761 use_global_cursor = isinstance( 1762 partition_router, GroupingPartitionRouter 1763 ) or component_definition.get("global_substream_cursor", False) 1764 1765 # Return the concurrent cursor and state converter 1766 return ConcurrentPerPartitionCursor( 1767 cursor_factory=cursor_factory, 1768 partition_router=partition_router, 1769 stream_name=stream_name, 1770 stream_namespace=stream_namespace, 1771 stream_state=stream_state, 1772 message_repository=self._message_repository, # type: ignore 1773 connector_state_manager=state_manager, 1774 connector_state_converter=connector_state_converter, 1775 cursor_field=cursor_field, 1776 use_global_cursor=use_global_cursor, 1777 attempt_to_create_cursor_if_not_provided=attempt_to_create_cursor_if_not_provided, 1778 ) 1779 1780 @staticmethod 1781 def create_constant_backoff_strategy( 1782 model: ConstantBackoffStrategyModel, config: Config, **kwargs: Any 1783 ) -> ConstantBackoffStrategy: 1784 ModelToComponentFactory._validate_jitter_range(model.jitter_range_in_seconds) 1785 return ConstantBackoffStrategy( 1786 backoff_time_in_seconds=model.backoff_time_in_seconds, 1787 jitter_range_in_seconds=model.jitter_range_in_seconds, 1788 config=config, 1789 parameters=model.parameters or {}, 1790 ) 1791 1792 @staticmethod 1793 def _validate_jitter_range(jitter_range_in_seconds: Optional[float]) -> None: 1794 if jitter_range_in_seconds is not None and jitter_range_in_seconds < 0: 1795 raise ValueError("jitter_range_in_seconds must be greater than or equal to 0") 1796 1797 def create_cursor_pagination( 1798 self, model: CursorPaginationModel, config: Config, decoder: Decoder, **kwargs: Any 1799 ) -> CursorPaginationStrategy: 1800 if isinstance(decoder, PaginationDecoderDecorator): 1801 inner_decoder = decoder.decoder 1802 else: 1803 inner_decoder = decoder 1804 decoder = PaginationDecoderDecorator(decoder=decoder) 1805 1806 if self._is_supported_decoder_for_pagination(inner_decoder): 1807 decoder_to_use = decoder 1808 else: 1809 raise ValueError( 1810 self._UNSUPPORTED_DECODER_ERROR.format(decoder_type=type(inner_decoder)) 1811 ) 1812 1813 # Pydantic v1 Union type coercion can convert int to string depending on Union order. 1814 # If page_size is a string that represents an integer (not an interpolation), convert it back. 1815 page_size = model.page_size 1816 if isinstance(page_size, str) and page_size.isdigit(): 1817 page_size = int(page_size) 1818 1819 return CursorPaginationStrategy( 1820 cursor_value=model.cursor_value, 1821 decoder=decoder_to_use, 1822 page_size=page_size, 1823 stop_condition=model.stop_condition, 1824 config=config, 1825 parameters=model.parameters or {}, 1826 ) 1827 1828 def create_custom_component(self, model: Any, config: Config, **kwargs: Any) -> Any: 1829 """ 1830 Generically creates a custom component based on the model type and a class_name reference to the custom Python class being 1831 instantiated. Only the model's additional properties that match the custom class definition are passed to the constructor 1832 :param model: The Pydantic model of the custom component being created 1833 :param config: The custom defined connector config 1834 :return: The declarative component built from the Pydantic model to be used at runtime 1835 """ 1836 # Instantiating a custom component means importing and executing arbitrary code referenced 1837 # by `class_name`. Manifests supplied by a caller, whether through the config or directly to 1838 # the manifest server, are untrusted input and could point `class_name` at any importable 1839 # callable, so they honor the same `AIRBYTE_ENABLE_UNSAFE_CODE` gate as injected 1840 # `components.py` code. Manifests bundled in a published connector image are trusted and may 1841 # always use their bundled custom components. 1842 manifest_is_untrusted = not self._custom_components_trusted or bool( 1843 config.get(INJECTED_MANIFEST) 1844 ) 1845 if manifest_is_untrusted and not custom_code_execution_permitted(): 1846 raise AirbyteCustomCodeNotPermittedError 1847 1848 custom_component_class = self._get_class_from_fully_qualified_class_name(model.class_name) 1849 component_fields = get_type_hints(custom_component_class) 1850 model_args = model.dict() 1851 model_args["config"] = config 1852 1853 # There are cases where a parent component will pass arguments to a child component via kwargs. When there are field collisions 1854 # we defer to these arguments over the component's definition 1855 for key, arg in kwargs.items(): 1856 model_args[key] = arg 1857 1858 # Pydantic is unable to parse a custom component's fields that are subcomponents into models because their fields and types are not 1859 # defined in the schema. The fields and types are defined within the Python class implementation. Pydantic can only parse down to 1860 # the custom component and this code performs a second parse to convert the sub-fields first into models, then declarative components 1861 for model_field, model_value in model_args.items(): 1862 # If a custom component field doesn't have a type set, we try to use the type hints to infer the type 1863 if ( 1864 isinstance(model_value, dict) 1865 and "type" not in model_value 1866 and model_field in component_fields 1867 ): 1868 derived_type = self._derive_component_type_from_type_hints( 1869 component_fields.get(model_field) 1870 ) 1871 if derived_type: 1872 model_value["type"] = derived_type 1873 1874 if self._is_component(model_value): 1875 model_args[model_field] = self._create_nested_component( 1876 model, 1877 model_field, 1878 model_value, 1879 config, 1880 **kwargs, 1881 ) 1882 elif isinstance(model_value, list): 1883 vals = [] 1884 for v in model_value: 1885 if isinstance(v, dict) and "type" not in v and model_field in component_fields: 1886 derived_type = self._derive_component_type_from_type_hints( 1887 component_fields.get(model_field) 1888 ) 1889 if derived_type: 1890 v["type"] = derived_type 1891 if self._is_component(v): 1892 vals.append( 1893 self._create_nested_component( 1894 model, 1895 model_field, 1896 v, 1897 config, 1898 **kwargs, 1899 ) 1900 ) 1901 else: 1902 vals.append(v) 1903 model_args[model_field] = vals 1904 1905 kwargs = { 1906 class_field: model_args[class_field] 1907 for class_field in component_fields.keys() 1908 if class_field in model_args 1909 } 1910 1911 if "api_budget" in component_fields and kwargs.get("api_budget") is None: 1912 kwargs["api_budget"] = self._api_budget 1913 1914 return custom_component_class(**kwargs) 1915 1916 @staticmethod 1917 def _get_class_from_fully_qualified_class_name( 1918 full_qualified_class_name: str, 1919 ) -> Any: 1920 """Get a class from its fully qualified name. 1921 1922 If a custom components module is needed, we assume it is already registered - probably 1923 as `source_declarative_manifest.components` or `components`. 1924 1925 Args: 1926 full_qualified_class_name (str): The fully qualified name of the class (e.g., "module.ClassName"). 1927 1928 Returns: 1929 Any: The class object. 1930 1931 Raises: 1932 ValueError: If the class cannot be loaded. 1933 """ 1934 split = full_qualified_class_name.split(".") 1935 module_name_full = ".".join(split[:-1]) 1936 class_name = split[-1] 1937 1938 try: 1939 module_ref = importlib.import_module(module_name_full) 1940 except ModuleNotFoundError as e: 1941 if split[0] == "source_declarative_manifest": 1942 # During testing, the modules containing the custom components are not moved to source_declarative_manifest. In order to run the test, add the source folder to your PYTHONPATH or add it runtime using sys.path.append 1943 try: 1944 import os 1945 1946 module_name_with_source_declarative_manifest = ".".join(split[1:-1]) 1947 module_ref = importlib.import_module( 1948 module_name_with_source_declarative_manifest 1949 ) 1950 except ModuleNotFoundError: 1951 raise ValueError(f"Could not load module `{module_name_full}`.") from e 1952 else: 1953 raise ValueError(f"Could not load module `{module_name_full}`.") from e 1954 1955 try: 1956 return getattr(module_ref, class_name) 1957 except AttributeError as e: 1958 raise ValueError( 1959 f"Could not load class `{class_name}` from module `{module_name_full}`.", 1960 ) from e 1961 1962 @staticmethod 1963 def _derive_component_type_from_type_hints(field_type: Any) -> Optional[str]: 1964 interface = field_type 1965 while True: 1966 origin = get_origin(interface) 1967 if origin: 1968 # Unnest types until we reach the raw type 1969 # List[T] -> T 1970 # Optional[List[T]] -> T 1971 args = get_args(interface) 1972 interface = args[0] 1973 else: 1974 break 1975 if isinstance(interface, type) and not ModelToComponentFactory.is_builtin_type(interface): 1976 return interface.__name__ 1977 return None 1978 1979 @staticmethod 1980 def is_builtin_type(cls: Optional[Type[Any]]) -> bool: 1981 if not cls: 1982 return False 1983 return cls.__module__ == "builtins" 1984 1985 @staticmethod 1986 def _extract_missing_parameters(error: TypeError) -> List[str]: 1987 parameter_search = re.search(r"keyword-only.*:\s(.*)", str(error)) 1988 if parameter_search: 1989 return re.findall(r"\'(.+?)\'", parameter_search.group(1)) 1990 else: 1991 return [] 1992 1993 def _create_nested_component( 1994 self, model: Any, model_field: str, model_value: Any, config: Config, **kwargs: Any 1995 ) -> Any: 1996 type_name = model_value.get("type", None) 1997 if not type_name: 1998 # If no type is specified, we can assume this is a dictionary object which can be returned instead of a subcomponent 1999 return model_value 2000 2001 model_type = self.TYPE_NAME_TO_MODEL.get(type_name, None) 2002 if model_type: 2003 parsed_model = model_type.parse_obj(model_value) 2004 try: 2005 # To improve usability of the language, certain fields are shared between components. This can come in the form of 2006 # a parent component passing some of its fields to a child component or the parent extracting fields from other child 2007 # components and passing it to others. One example is the DefaultPaginator referencing the HttpRequester url_base 2008 # while constructing a SimpleRetriever. However, custom components don't support this behavior because they are created 2009 # generically in create_custom_component(). This block allows developers to specify extra arguments in $parameters that 2010 # are needed by a component and could not be shared. 2011 model_constructor = self.PYDANTIC_MODEL_TO_CONSTRUCTOR.get(parsed_model.__class__) 2012 constructor_kwargs = inspect.getfullargspec(model_constructor).kwonlyargs 2013 model_parameters = model_value.get("$parameters", {}) 2014 matching_parameters = { 2015 kwarg: model_parameters[kwarg] 2016 for kwarg in constructor_kwargs 2017 if kwarg in model_parameters 2018 } 2019 matching_kwargs = { 2020 kwarg: kwargs[kwarg] for kwarg in constructor_kwargs if kwarg in kwargs 2021 } 2022 return self._create_component_from_model( 2023 model=parsed_model, config=config, **(matching_parameters | matching_kwargs) 2024 ) 2025 except TypeError as error: 2026 missing_parameters = self._extract_missing_parameters(error) 2027 if missing_parameters: 2028 raise ValueError( 2029 f"Error creating component '{type_name}' with parent custom component {model.class_name}: Please provide " 2030 + ", ".join( 2031 ( 2032 f"{type_name}.$parameters.{parameter}" 2033 for parameter in missing_parameters 2034 ) 2035 ) 2036 ) 2037 raise TypeError( 2038 f"Error creating component '{type_name}' with parent custom component {model.class_name}: {error}" 2039 ) 2040 else: 2041 raise ValueError( 2042 f"Error creating custom component {model.class_name}. Subcomponent creation has not been implemented for '{type_name}'" 2043 ) 2044 2045 @staticmethod 2046 def _is_component(model_value: Any) -> bool: 2047 return isinstance(model_value, dict) and model_value.get("type") is not None 2048 2049 def create_default_stream( 2050 self, model: DeclarativeStreamModel, config: Config, is_parent: bool = False, **kwargs: Any 2051 ) -> AbstractStream: 2052 primary_key = model.primary_key.__root__ if model.primary_key else None 2053 self._migrate_state(model, config) 2054 2055 partition_router = self._build_stream_slicer_from_partition_router( 2056 model.retriever, 2057 config, 2058 stream_name=model.name, 2059 **kwargs, 2060 ) 2061 concurrent_cursor = self._build_concurrent_cursor(model, partition_router, config) 2062 if model.incremental_sync and isinstance(model.incremental_sync, DatetimeBasedCursorModel): 2063 cursor_model: DatetimeBasedCursorModel = model.incremental_sync 2064 2065 end_time_option = ( 2066 self._create_component_from_model( 2067 cursor_model.end_time_option, config, parameters=cursor_model.parameters or {} 2068 ) 2069 if cursor_model.end_time_option 2070 else None 2071 ) 2072 start_time_option = ( 2073 self._create_component_from_model( 2074 cursor_model.start_time_option, config, parameters=cursor_model.parameters or {} 2075 ) 2076 if cursor_model.start_time_option 2077 else None 2078 ) 2079 2080 datetime_request_options_provider = DatetimeBasedRequestOptionsProvider( 2081 start_time_option=start_time_option, 2082 end_time_option=end_time_option, 2083 partition_field_start=cursor_model.partition_field_start, 2084 partition_field_end=cursor_model.partition_field_end, 2085 config=config, 2086 parameters=model.parameters or {}, 2087 ) 2088 request_options_provider = ( 2089 datetime_request_options_provider 2090 if not isinstance(concurrent_cursor, ConcurrentPerPartitionCursor) 2091 else PerPartitionRequestOptionsProvider( 2092 partition_router, datetime_request_options_provider 2093 ) 2094 ) 2095 elif model.incremental_sync and isinstance( 2096 model.incremental_sync, IncrementingCountCursorModel 2097 ): 2098 if isinstance(concurrent_cursor, ConcurrentPerPartitionCursor): 2099 raise ValueError( 2100 "PerPartition does not support per partition states because switching to global state is time based" 2101 ) 2102 2103 cursor_model: IncrementingCountCursorModel = model.incremental_sync # type: ignore 2104 2105 start_time_option = ( 2106 self._create_component_from_model( 2107 cursor_model.start_value_option, # type: ignore # mypy still thinks cursor_model of type DatetimeBasedCursor 2108 config, 2109 parameters=cursor_model.parameters or {}, 2110 ) 2111 if cursor_model.start_value_option # type: ignore # mypy still thinks cursor_model of type DatetimeBasedCursor 2112 else None 2113 ) 2114 2115 # The concurrent engine defaults the start/end fields on the slice to "start" and "end", but 2116 # the default DatetimeBasedRequestOptionsProvider() sets them to start_time/end_time 2117 partition_field_start = "start" 2118 2119 request_options_provider = DatetimeBasedRequestOptionsProvider( 2120 start_time_option=start_time_option, 2121 partition_field_start=partition_field_start, 2122 config=config, 2123 parameters=model.parameters or {}, 2124 ) 2125 else: 2126 request_options_provider = None 2127 2128 transformations = [] 2129 if model.transformations: 2130 for transformation_model in model.transformations: 2131 transformations.append( 2132 self._create_component_from_model(model=transformation_model, config=config) 2133 ) 2134 file_uploader = None 2135 if model.file_uploader: 2136 file_uploader = self._create_component_from_model( 2137 model=model.file_uploader, config=config 2138 ) 2139 2140 stream_slicer: ConcurrentStreamSlicer = ( 2141 partition_router 2142 if isinstance(concurrent_cursor, FinalStateCursor) 2143 else concurrent_cursor 2144 ) 2145 2146 retriever = self._create_component_from_model( 2147 model=model.retriever, 2148 config=config, 2149 name=model.name, 2150 primary_key=primary_key, 2151 request_options_provider=request_options_provider, 2152 stream_slicer=stream_slicer, 2153 partition_router=partition_router, 2154 has_stop_condition_cursor=self._is_stop_condition_on_cursor(model), 2155 is_client_side_incremental_sync=self._is_client_side_filtering_enabled(model), 2156 cursor=concurrent_cursor, 2157 transformations=transformations, 2158 file_uploader=file_uploader, 2159 incremental_sync=model.incremental_sync, 2160 ) 2161 if isinstance(retriever, AsyncRetriever): 2162 stream_slicer = retriever.stream_slicer 2163 2164 schema_loader: SchemaLoader 2165 if model.schema_loader and isinstance(model.schema_loader, list): 2166 nested_schema_loaders = [ 2167 self._create_component_from_model(model=nested_schema_loader, config=config) 2168 for nested_schema_loader in model.schema_loader 2169 ] 2170 schema_loader = CompositeSchemaLoader( 2171 schema_loaders=nested_schema_loaders, parameters={} 2172 ) 2173 elif model.schema_loader: 2174 schema_loader = self._create_component_from_model( 2175 model=model.schema_loader, # type: ignore # If defined, schema_loader is guaranteed not to be a list and will be one of the existing base models 2176 config=config, 2177 ) 2178 else: 2179 options = model.parameters or {} 2180 if "name" not in options: 2181 options["name"] = model.name 2182 schema_loader = DefaultSchemaLoader(config=config, parameters=options) 2183 schema_loader = CachingSchemaLoaderDecorator(schema_loader) 2184 2185 stream_name = model.name or "" 2186 return DefaultStream( 2187 partition_generator=StreamSlicerPartitionGenerator( 2188 DeclarativePartitionFactory( 2189 stream_name, 2190 schema_loader, 2191 retriever, 2192 self._message_repository, 2193 ), 2194 stream_slicer, 2195 slice_limit=self._limit_slices_fetched, 2196 ), 2197 name=stream_name, 2198 json_schema=schema_loader.get_json_schema, 2199 primary_key=get_primary_key_from_stream(primary_key), 2200 cursor_field=( 2201 concurrent_cursor.cursor_field 2202 if hasattr(concurrent_cursor, "cursor_field") 2203 else None 2204 ), 2205 logger=logging.getLogger(f"airbyte.{stream_name}"), 2206 cursor=concurrent_cursor, 2207 supports_file_transfer=hasattr(model, "file_uploader") and bool(model.file_uploader), 2208 ) 2209 2210 def _migrate_state(self, model: DeclarativeStreamModel, config: Config) -> None: 2211 stream_name = model.name or "" 2212 stream_state = self._connector_state_manager.get_stream_state( 2213 stream_name=stream_name, namespace=None 2214 ) 2215 if model.state_migrations: 2216 state_transformations = [ 2217 self._create_component_from_model(state_migration, config, declarative_stream=model) 2218 for state_migration in model.state_migrations 2219 ] 2220 else: 2221 state_transformations = [] 2222 stream_state = self.apply_stream_state_migrations(state_transformations, stream_state) 2223 self._connector_state_manager.update_state_for_stream( 2224 stream_name=stream_name, namespace=None, value=stream_state 2225 ) 2226 2227 def _is_stop_condition_on_cursor(self, model: DeclarativeStreamModel) -> bool: 2228 return bool( 2229 model.incremental_sync 2230 and hasattr(model.incremental_sync, "is_data_feed") 2231 and model.incremental_sync.is_data_feed 2232 ) 2233 2234 def _is_client_side_filtering_enabled(self, model: DeclarativeStreamModel) -> bool: 2235 return bool( 2236 model.incremental_sync 2237 and hasattr(model.incremental_sync, "is_client_side_incremental") 2238 and model.incremental_sync.is_client_side_incremental 2239 ) 2240 2241 def _build_stream_slicer_from_partition_router( 2242 self, 2243 model: Union[ 2244 AsyncRetrieverModel, 2245 CustomRetrieverModel, 2246 SimpleRetrieverModel, 2247 ], 2248 config: Config, 2249 stream_name: Optional[str] = None, 2250 **kwargs: Any, 2251 ) -> PartitionRouter: 2252 if ( 2253 hasattr(model, "partition_router") 2254 and isinstance(model, (SimpleRetrieverModel, AsyncRetrieverModel, CustomRetrieverModel)) 2255 and model.partition_router 2256 ): 2257 stream_slicer_model = model.partition_router 2258 if isinstance(stream_slicer_model, list): 2259 return CartesianProductStreamSlicer( 2260 [ 2261 self._create_component_from_model( 2262 model=slicer, config=config, stream_name=stream_name or "" 2263 ) 2264 for slicer in stream_slicer_model 2265 ], 2266 parameters={}, 2267 ) 2268 elif isinstance(stream_slicer_model, dict): 2269 # partition router comes from CustomRetrieverModel therefore has not been parsed as a model 2270 params = stream_slicer_model.get("$parameters") 2271 if not isinstance(params, dict): 2272 params = {} 2273 stream_slicer_model["$parameters"] = params 2274 2275 if stream_name is not None: 2276 params["stream_name"] = stream_name 2277 2278 return self._create_nested_component( # type: ignore[no-any-return] # There is no guarantee that this will return a stream slicer. If not, we expect an AttributeError during the call to `stream_slices` 2279 model, 2280 "partition_router", 2281 stream_slicer_model, 2282 config, 2283 **kwargs, 2284 ) 2285 else: 2286 return self._create_component_from_model( # type: ignore[no-any-return] # Will be created PartitionRouter as stream_slicer_model is model.partition_router 2287 model=stream_slicer_model, config=config, stream_name=stream_name or "" 2288 ) 2289 return SinglePartitionRouter(parameters={}) 2290 2291 def _build_concurrent_cursor( 2292 self, 2293 model: DeclarativeStreamModel, 2294 stream_slicer: Optional[PartitionRouter], 2295 config: Config, 2296 ) -> Cursor: 2297 stream_name = model.name or "" 2298 stream_state = self._connector_state_manager.get_stream_state(stream_name, None) 2299 2300 if ( 2301 model.incremental_sync 2302 and stream_slicer 2303 and not isinstance(stream_slicer, SinglePartitionRouter) 2304 ): 2305 if isinstance(model.incremental_sync, IncrementingCountCursorModel): 2306 # We don't currently support usage of partition routing and IncrementingCountCursor at the 2307 # same time because we didn't solve for design questions like what the lookback window would 2308 # be as well as global cursor fall backs. We have not seen customers that have needed both 2309 # at the same time yet and are currently punting on this until we need to solve it. 2310 raise ValueError( 2311 f"The low-code framework does not currently support usage of a PartitionRouter and an IncrementingCountCursor at the same time. Please specify only one of these options for stream {stream_name}." 2312 ) 2313 return self.create_concurrent_cursor_from_perpartition_cursor( # type: ignore # This is a known issue that we are creating and returning a ConcurrentCursor which does not technically implement the (low-code) StreamSlicer. However, (low-code) StreamSlicer and ConcurrentCursor both implement StreamSlicer.stream_slices() which is the primary method needed for checkpointing 2314 state_manager=self._connector_state_manager, 2315 model_type=DatetimeBasedCursorModel, 2316 component_definition=model.incremental_sync.__dict__, 2317 stream_name=stream_name, 2318 stream_state=stream_state, 2319 stream_namespace=None, 2320 config=config or {}, 2321 partition_router=stream_slicer, 2322 attempt_to_create_cursor_if_not_provided=True, # FIXME can we remove that now? 2323 ) 2324 elif model.incremental_sync: 2325 if type(model.incremental_sync) == IncrementingCountCursorModel: 2326 return self.create_concurrent_cursor_from_incrementing_count_cursor( # type: ignore # This is a known issue that we are creating and returning a ConcurrentCursor which does not technically implement the (low-code) StreamSlicer. However, (low-code) StreamSlicer and ConcurrentCursor both implement StreamSlicer.stream_slices() which is the primary method needed for checkpointing 2327 model_type=IncrementingCountCursorModel, 2328 component_definition=model.incremental_sync.__dict__, 2329 stream_name=stream_name, 2330 stream_namespace=None, 2331 stream_state=stream_state, 2332 config=config or {}, 2333 ) 2334 elif type(model.incremental_sync) == DatetimeBasedCursorModel: 2335 return self.create_concurrent_cursor_from_datetime_based_cursor( # type: ignore # This is a known issue that we are creating and returning a ConcurrentCursor which does not technically implement the (low-code) StreamSlicer. However, (low-code) StreamSlicer and ConcurrentCursor both implement StreamSlicer.stream_slices() which is the primary method needed for checkpointing 2336 model_type=type(model.incremental_sync), 2337 component_definition=model.incremental_sync.__dict__, 2338 stream_name=stream_name, 2339 stream_namespace=None, 2340 stream_state=stream_state, 2341 config=config or {}, 2342 attempt_to_create_cursor_if_not_provided=True, 2343 ) 2344 else: 2345 raise ValueError( 2346 f"Incremental sync of type {type(model.incremental_sync)} is not supported" 2347 ) 2348 return FinalStateCursor(stream_name, None, self._message_repository) 2349 2350 def create_default_error_handler( 2351 self, model: DefaultErrorHandlerModel, config: Config, **kwargs: Any 2352 ) -> DefaultErrorHandler: 2353 backoff_strategies = [] 2354 if model.backoff_strategies: 2355 for backoff_strategy_model in model.backoff_strategies: 2356 backoff_strategies.append( 2357 self._create_component_from_model(model=backoff_strategy_model, config=config) 2358 ) 2359 2360 response_filters = [] 2361 if model.response_filters: 2362 for response_filter_model in model.response_filters: 2363 response_filters.append( 2364 self._create_component_from_model(model=response_filter_model, config=config) 2365 ) 2366 response_filters.append( 2367 HttpResponseFilter(config=config, parameters=model.parameters or {}) 2368 ) 2369 2370 return DefaultErrorHandler( 2371 backoff_strategies=backoff_strategies, 2372 max_retries=model.max_retries, 2373 response_filters=response_filters, 2374 config=config, 2375 parameters=model.parameters or {}, 2376 ) 2377 2378 def create_default_paginator( 2379 self, 2380 model: DefaultPaginatorModel, 2381 config: Config, 2382 *, 2383 url_base: str, 2384 extractor_model: Optional[Union[CustomRecordExtractorModel, DpathExtractorModel]] = None, 2385 decoder: Optional[Decoder] = None, 2386 cursor_used_for_stop_condition: Optional[Cursor] = None, 2387 ) -> Union[DefaultPaginator, PaginatorTestReadDecorator]: 2388 if decoder: 2389 if self._is_supported_decoder_for_pagination(decoder): 2390 decoder_to_use = PaginationDecoderDecorator(decoder=decoder) 2391 else: 2392 raise ValueError(self._UNSUPPORTED_DECODER_ERROR.format(decoder_type=type(decoder))) 2393 else: 2394 decoder_to_use = PaginationDecoderDecorator(decoder=JsonDecoder(parameters={})) 2395 page_size_option = ( 2396 self._create_component_from_model(model=model.page_size_option, config=config) 2397 if model.page_size_option 2398 else None 2399 ) 2400 page_token_option = ( 2401 self._create_component_from_model(model=model.page_token_option, config=config) 2402 if model.page_token_option 2403 else None 2404 ) 2405 pagination_strategy = self._create_component_from_model( 2406 model=model.pagination_strategy, 2407 config=config, 2408 decoder=decoder_to_use, 2409 extractor_model=extractor_model, 2410 ) 2411 if cursor_used_for_stop_condition: 2412 pagination_strategy = StopConditionPaginationStrategyDecorator( 2413 pagination_strategy, CursorStopCondition(cursor_used_for_stop_condition) 2414 ) 2415 paginator = DefaultPaginator( 2416 decoder=decoder_to_use, 2417 page_size_option=page_size_option, 2418 page_token_option=page_token_option, 2419 pagination_strategy=pagination_strategy, 2420 url_base=url_base, 2421 config=config, 2422 parameters=model.parameters or {}, 2423 ) 2424 if self._limit_pages_fetched_per_slice: 2425 return PaginatorTestReadDecorator(paginator, self._limit_pages_fetched_per_slice) 2426 return paginator 2427 2428 def create_dpath_extractor( 2429 self, 2430 model: DpathExtractorModel, 2431 config: Config, 2432 decoder: Optional[Decoder] = None, 2433 **kwargs: Any, 2434 ) -> DpathExtractor: 2435 if decoder: 2436 decoder_to_use = decoder 2437 else: 2438 decoder_to_use = JsonDecoder(parameters={}) 2439 model_field_path: List[Union[InterpolatedString, str]] = [x for x in model.field_path] 2440 2441 record_expander = None 2442 if model.record_expander: 2443 record_expander = self._create_component_from_model( 2444 model=model.record_expander, 2445 config=config, 2446 ) 2447 2448 return DpathExtractor( 2449 decoder=decoder_to_use, 2450 field_path=model_field_path, 2451 config=config, 2452 parameters=model.parameters or {}, 2453 record_expander=record_expander, 2454 ) 2455 2456 def create_record_expander( 2457 self, 2458 model: RecordExpanderModel, 2459 config: Config, 2460 **kwargs: Any, 2461 ) -> RecordExpander: 2462 return RecordExpander( 2463 expand_records_from_field=model.expand_records_from_field, 2464 config=config, 2465 parameters=model.parameters or {}, 2466 remain_original_record=model.remain_original_record or False, 2467 on_no_records=OnNoRecords(model.on_no_records.value) 2468 if model.on_no_records 2469 else OnNoRecords.skip, 2470 ) 2471 2472 @staticmethod 2473 def create_response_to_file_extractor( 2474 model: ResponseToFileExtractorModel, 2475 **kwargs: Any, 2476 ) -> ResponseToFileExtractor: 2477 return ResponseToFileExtractor( 2478 parameters=model.parameters or {}, 2479 preserve_na_values=model.preserve_na_values or False, 2480 ) 2481 2482 @staticmethod 2483 def create_exponential_backoff_strategy( 2484 model: ExponentialBackoffStrategyModel, config: Config 2485 ) -> ExponentialBackoffStrategy: 2486 ModelToComponentFactory._validate_jitter_range(model.jitter_range_in_seconds) 2487 return ExponentialBackoffStrategy( 2488 factor=model.factor or 5, 2489 jitter_range_in_seconds=model.jitter_range_in_seconds, 2490 parameters=model.parameters or {}, 2491 config=config, 2492 ) 2493 2494 @staticmethod 2495 def create_group_by_key(model: GroupByKeyMergeStrategyModel, config: Config) -> GroupByKey: 2496 return GroupByKey(model.key, config=config, parameters=model.parameters or {}) 2497 2498 def create_http_requester( 2499 self, 2500 model: HttpRequesterModel, 2501 config: Config, 2502 decoder: Decoder = JsonDecoder(parameters={}), 2503 query_properties_key: Optional[str] = None, 2504 use_cache: Optional[bool] = None, 2505 *, 2506 name: str, 2507 ) -> HttpRequester: 2508 authenticator = ( 2509 self._create_component_from_model( 2510 model=model.authenticator, 2511 config=config, 2512 url_base=model.url or model.url_base, 2513 name=name, 2514 decoder=decoder, 2515 ) 2516 if model.authenticator 2517 else None 2518 ) 2519 error_handler = ( 2520 self._create_component_from_model(model=model.error_handler, config=config) 2521 if model.error_handler 2522 else DefaultErrorHandler( 2523 backoff_strategies=[], 2524 response_filters=[], 2525 config=config, 2526 parameters=model.parameters or {}, 2527 ) 2528 ) 2529 2530 api_budget = self._api_budget 2531 2532 request_options_provider = InterpolatedRequestOptionsProvider( 2533 request_body=model.request_body, 2534 request_body_data=model.request_body_data, 2535 request_body_json=model.request_body_json, 2536 request_headers=model.request_headers, 2537 request_parameters=model.request_parameters, # type: ignore # QueryProperties have been removed in `create_simple_retriever` 2538 query_properties_key=query_properties_key, 2539 config=config, 2540 parameters=model.parameters or {}, 2541 ) 2542 2543 assert model.use_cache is not None # for mypy 2544 assert model.http_method is not None # for mypy 2545 2546 should_use_cache = (model.use_cache or bool(use_cache)) and not self._disable_cache 2547 2548 return HttpRequester( 2549 name=name, 2550 url=model.url, 2551 url_base=model.url_base, 2552 path=model.path, 2553 authenticator=authenticator, 2554 error_handler=error_handler, 2555 api_budget=api_budget, 2556 http_method=HttpMethod[model.http_method.value], 2557 request_options_provider=request_options_provider, 2558 config=config, 2559 disable_retries=self._disable_retries, 2560 parameters=model.parameters or {}, 2561 message_repository=self._message_repository, 2562 use_cache=should_use_cache, 2563 decoder=decoder, 2564 stream_response=decoder.is_stream_response() if decoder else False, 2565 ) 2566 2567 @staticmethod 2568 def create_http_response_filter( 2569 model: HttpResponseFilterModel, config: Config, **kwargs: Any 2570 ) -> HttpResponseFilter: 2571 if model.action: 2572 action = ResponseAction(model.action.value) 2573 else: 2574 action = None 2575 2576 failure_type = FailureType(model.failure_type.value) if model.failure_type else None 2577 2578 http_codes = ( 2579 set(model.http_codes) if model.http_codes else set() 2580 ) # JSON schema notation has no set data type. The schema enforces an array of unique elements 2581 2582 return HttpResponseFilter( 2583 action=action, 2584 failure_type=failure_type, 2585 error_message=model.error_message or "", 2586 error_message_contains=model.error_message_contains or "", 2587 http_codes=http_codes, 2588 predicate=model.predicate or "", 2589 config=config, 2590 parameters=model.parameters or {}, 2591 ) 2592 2593 @staticmethod 2594 def create_inline_schema_loader( 2595 model: InlineSchemaLoaderModel, config: Config, **kwargs: Any 2596 ) -> InlineSchemaLoader: 2597 return InlineSchemaLoader(schema=model.schema_ or {}, parameters={}) 2598 2599 def create_complex_field_type( 2600 self, model: ComplexFieldTypeModel, config: Config, **kwargs: Any 2601 ) -> ComplexFieldType: 2602 items = ( 2603 self._create_component_from_model(model=model.items, config=config) 2604 if isinstance(model.items, ComplexFieldTypeModel) 2605 else model.items 2606 ) 2607 2608 return ComplexFieldType(field_type=model.field_type, items=items) 2609 2610 def create_types_map(self, model: TypesMapModel, config: Config, **kwargs: Any) -> TypesMap: 2611 target_type = ( 2612 self._create_component_from_model(model=model.target_type, config=config) 2613 if isinstance(model.target_type, ComplexFieldTypeModel) 2614 else model.target_type 2615 ) 2616 2617 return TypesMap( 2618 target_type=target_type, 2619 current_type=model.current_type, 2620 condition=model.condition if model.condition is not None else "True", 2621 ) 2622 2623 def create_schema_type_identifier( 2624 self, model: SchemaTypeIdentifierModel, config: Config, **kwargs: Any 2625 ) -> SchemaTypeIdentifier: 2626 types_mapping = [] 2627 if model.types_mapping: 2628 types_mapping.extend( 2629 [ 2630 self._create_component_from_model(types_map, config=config) 2631 for types_map in model.types_mapping 2632 ] 2633 ) 2634 model_schema_pointer: List[Union[InterpolatedString, str]] = ( 2635 [x for x in model.schema_pointer] if model.schema_pointer else [] 2636 ) 2637 model_key_pointer: List[Union[InterpolatedString, str]] = [x for x in model.key_pointer] 2638 model_type_pointer: Optional[List[Union[InterpolatedString, str]]] = ( 2639 [x for x in model.type_pointer] if model.type_pointer else None 2640 ) 2641 2642 return SchemaTypeIdentifier( 2643 schema_pointer=model_schema_pointer, 2644 key_pointer=model_key_pointer, 2645 type_pointer=model_type_pointer, 2646 types_mapping=types_mapping, 2647 parameters=model.parameters or {}, 2648 ) 2649 2650 def create_dynamic_schema_loader( 2651 self, model: DynamicSchemaLoaderModel, config: Config, **kwargs: Any 2652 ) -> DynamicSchemaLoader: 2653 schema_transformations = [] 2654 if model.schema_transformations: 2655 for transformation_model in model.schema_transformations: 2656 schema_transformations.append( 2657 self._create_component_from_model(model=transformation_model, config=config) 2658 ) 2659 name = "dynamic_properties" 2660 retriever = self._create_component_from_model( 2661 model=model.retriever, 2662 config=config, 2663 name=name, 2664 primary_key=None, 2665 partition_router=self._build_stream_slicer_from_partition_router( 2666 model.retriever, config 2667 ), 2668 transformations=[], 2669 use_cache=True, 2670 log_formatter=( 2671 lambda response: format_http_message( 2672 response, 2673 f"Schema loader '{name}' request", 2674 f"Request performed in order to extract schema.", 2675 name, 2676 is_auxiliary=True, 2677 ) 2678 ), 2679 ) 2680 schema_type_identifier = self._create_component_from_model( 2681 model.schema_type_identifier, config=config, parameters=model.parameters or {} 2682 ) 2683 schema_filter = ( 2684 self._create_component_from_model( 2685 model.schema_filter, config=config, parameters=model.parameters or {} 2686 ) 2687 if model.schema_filter is not None 2688 else None 2689 ) 2690 2691 return DynamicSchemaLoader( 2692 retriever=retriever, 2693 config=config, 2694 schema_transformations=schema_transformations, 2695 schema_filter=schema_filter, 2696 schema_type_identifier=schema_type_identifier, 2697 parameters=model.parameters or {}, 2698 ) 2699 2700 @staticmethod 2701 def create_json_decoder(model: JsonDecoderModel, config: Config, **kwargs: Any) -> Decoder: 2702 return JsonDecoder(parameters={}) 2703 2704 def create_csv_decoder(self, model: CsvDecoderModel, config: Config, **kwargs: Any) -> Decoder: 2705 return CompositeRawDecoder( 2706 parser=ModelToComponentFactory._get_parser(model, config), 2707 stream_response=False if self._emit_connector_builder_messages else True, 2708 ) 2709 2710 def create_jsonl_decoder( 2711 self, model: JsonlDecoderModel, config: Config, **kwargs: Any 2712 ) -> Decoder: 2713 return CompositeRawDecoder( 2714 parser=ModelToComponentFactory._get_parser(model, config), 2715 stream_response=False if self._emit_connector_builder_messages else True, 2716 ) 2717 2718 def create_json_items_decoder( 2719 self, model: JsonItemsDecoderModel, config: Config, **kwargs: Any 2720 ) -> Decoder: 2721 return CompositeRawDecoder( 2722 parser=ModelToComponentFactory._get_parser(model, config), 2723 stream_response=False if self._emit_connector_builder_messages else True, 2724 ) 2725 2726 def create_gzip_decoder( 2727 self, model: GzipDecoderModel, config: Config, **kwargs: Any 2728 ) -> Decoder: 2729 _compressed_response_types = { 2730 "gzip", 2731 "x-gzip", 2732 "gzip, deflate", 2733 "x-gzip, deflate", 2734 "application/zip", 2735 "application/gzip", 2736 "application/x-gzip", 2737 "application/x-zip-compressed", 2738 } 2739 2740 gzip_parser: GzipParser = ModelToComponentFactory._get_parser(model, config) # type: ignore # based on the model, we know this will be a GzipParser 2741 2742 if self._emit_connector_builder_messages: 2743 # This is very surprising but if the response is not streamed, 2744 # CompositeRawDecoder calls response.content and the requests library actually uncompress the data as opposed to response.raw, 2745 # which uses urllib3 directly and does not uncompress the data. 2746 return CompositeRawDecoder(gzip_parser.inner_parser, False) 2747 2748 return CompositeRawDecoder.by_headers( 2749 [({"Content-Encoding", "Content-Type"}, _compressed_response_types, gzip_parser)], 2750 stream_response=True, 2751 fallback_parser=gzip_parser.inner_parser, 2752 ) 2753 2754 @staticmethod 2755 def create_iterable_decoder( 2756 model: IterableDecoderModel, config: Config, **kwargs: Any 2757 ) -> IterableDecoder: 2758 return IterableDecoder(parameters={}) 2759 2760 @staticmethod 2761 def create_xml_decoder(model: XmlDecoderModel, config: Config, **kwargs: Any) -> XmlDecoder: 2762 return XmlDecoder(parameters={}) 2763 2764 def create_zipfile_decoder( 2765 self, model: ZipfileDecoderModel, config: Config, **kwargs: Any 2766 ) -> ZipfileDecoder: 2767 return ZipfileDecoder(parser=ModelToComponentFactory._get_parser(model.decoder, config)) 2768 2769 @staticmethod 2770 def _get_parser(model: BaseModel, config: Config) -> Parser: 2771 if isinstance(model, JsonDecoderModel): 2772 # Note that the logic is a bit different from the JsonDecoder as there is some legacy that is maintained to return {} on error cases 2773 return JsonParser() 2774 elif isinstance(model, JsonItemsDecoderModel): 2775 return JsonItemsParser( 2776 items_path=model.items_path, 2777 encoding=model.encoding, 2778 ) 2779 elif isinstance(model, JsonlDecoderModel): 2780 return JsonLineParser() 2781 elif isinstance(model, CsvDecoderModel): 2782 return CsvParser( 2783 encoding=model.encoding, 2784 delimiter=model.delimiter, 2785 set_values_to_none=model.set_values_to_none, 2786 ) 2787 elif isinstance(model, GzipDecoderModel): 2788 return GzipParser( 2789 inner_parser=ModelToComponentFactory._get_parser(model.decoder, config) 2790 ) 2791 elif isinstance( 2792 model, (CustomDecoderModel, IterableDecoderModel, XmlDecoderModel, ZipfileDecoderModel) 2793 ): 2794 raise ValueError(f"Decoder type {model} does not have parser associated to it") 2795 2796 raise ValueError(f"Unknown decoder type {model}") 2797 2798 @staticmethod 2799 def create_json_file_schema_loader( 2800 model: JsonFileSchemaLoaderModel, config: Config, **kwargs: Any 2801 ) -> JsonFileSchemaLoader: 2802 return JsonFileSchemaLoader( 2803 file_path=model.file_path or "", config=config, parameters=model.parameters or {} 2804 ) 2805 2806 def create_jwt_authenticator( 2807 self, model: JwtAuthenticatorModel, config: Config, **kwargs: Any 2808 ) -> JwtAuthenticator: 2809 jwt_headers = model.jwt_headers or JwtHeadersModel(kid=None, typ="JWT", cty=None) 2810 jwt_payload = model.jwt_payload or JwtPayloadModel(iss=None, sub=None, aud=None) 2811 request_option = ( 2812 self._create_component_from_model(model.request_option, config) 2813 if model.request_option 2814 else None 2815 ) 2816 return JwtAuthenticator( 2817 config=config, 2818 parameters=model.parameters or {}, 2819 algorithm=JwtAlgorithm(model.algorithm.value), 2820 secret_key=model.secret_key, 2821 base64_encode_secret_key=model.base64_encode_secret_key, 2822 token_duration=model.token_duration, 2823 header_prefix=model.header_prefix, 2824 kid=jwt_headers.kid, 2825 typ=jwt_headers.typ, 2826 cty=jwt_headers.cty, 2827 iss=jwt_payload.iss, 2828 sub=jwt_payload.sub, 2829 aud=jwt_payload.aud, 2830 additional_jwt_headers=model.additional_jwt_headers, 2831 additional_jwt_payload=model.additional_jwt_payload, 2832 passphrase=model.passphrase, 2833 request_option=request_option, 2834 ) 2835 2836 def create_list_partition_router( 2837 self, model: ListPartitionRouterModel, config: Config, **kwargs: Any 2838 ) -> ListPartitionRouter: 2839 request_option = ( 2840 self._create_component_from_model(model.request_option, config) 2841 if model.request_option 2842 else None 2843 ) 2844 return ListPartitionRouter( 2845 cursor_field=model.cursor_field, 2846 request_option=request_option, 2847 values=model.values, 2848 config=config, 2849 parameters=model.parameters or {}, 2850 ) 2851 2852 @staticmethod 2853 def create_min_max_datetime( 2854 model: MinMaxDatetimeModel, config: Config, **kwargs: Any 2855 ) -> MinMaxDatetime: 2856 return MinMaxDatetime( 2857 datetime=model.datetime, 2858 datetime_format=model.datetime_format or "", 2859 max_datetime=model.max_datetime or "", 2860 min_datetime=model.min_datetime or "", 2861 parameters=model.parameters or {}, 2862 ) 2863 2864 @staticmethod 2865 def create_no_auth(model: NoAuthModel, config: Config, **kwargs: Any) -> NoAuth: 2866 return NoAuth(parameters=model.parameters or {}) 2867 2868 @staticmethod 2869 def create_no_pagination( 2870 model: NoPaginationModel, config: Config, **kwargs: Any 2871 ) -> NoPagination: 2872 return NoPagination(parameters={}) 2873 2874 def create_oauth_authenticator( 2875 self, model: OAuthAuthenticatorModel, config: Config, **kwargs: Any 2876 ) -> DeclarativeOauth2Authenticator: 2877 profile_assertion = ( 2878 self._create_component_from_model(model.profile_assertion, config=config) 2879 if model.profile_assertion 2880 else None 2881 ) 2882 2883 refresh_token_error_status_codes, refresh_token_error_key, refresh_token_error_values = ( 2884 self._get_refresh_token_error_information(model) 2885 ) 2886 if model.refresh_token_updater: 2887 # ignore type error because fixing it would have a lot of dependencies, revisit later 2888 return DeclarativeSingleUseRefreshTokenOauth2Authenticator( # type: ignore 2889 config, 2890 InterpolatedString.create( 2891 model.token_refresh_endpoint, # type: ignore 2892 parameters=model.parameters or {}, 2893 ).eval(config), 2894 access_token_name=InterpolatedString.create( 2895 model.access_token_name or "access_token", parameters=model.parameters or {} 2896 ).eval(config), 2897 refresh_token_name=model.refresh_token_updater.refresh_token_name, 2898 expires_in_name=InterpolatedString.create( 2899 model.expires_in_name or "expires_in", parameters=model.parameters or {} 2900 ).eval(config), 2901 client_id_name=InterpolatedString.create( 2902 model.client_id_name or "client_id", parameters=model.parameters or {} 2903 ).eval(config), 2904 client_id=InterpolatedString.create( 2905 model.client_id, parameters=model.parameters or {} 2906 ).eval(config) 2907 if model.client_id 2908 else model.client_id, 2909 client_secret_name=InterpolatedString.create( 2910 model.client_secret_name or "client_secret", parameters=model.parameters or {} 2911 ).eval(config), 2912 client_secret=InterpolatedString.create( 2913 model.client_secret, parameters=model.parameters or {} 2914 ).eval(config) 2915 if model.client_secret 2916 else model.client_secret, 2917 access_token_config_path=model.refresh_token_updater.access_token_config_path, 2918 refresh_token_config_path=model.refresh_token_updater.refresh_token_config_path, 2919 token_expiry_date_config_path=model.refresh_token_updater.token_expiry_date_config_path, 2920 grant_type_name=InterpolatedString.create( 2921 model.grant_type_name or "grant_type", parameters=model.parameters or {} 2922 ).eval(config), 2923 grant_type=InterpolatedString.create( 2924 model.grant_type or "refresh_token", parameters=model.parameters or {} 2925 ).eval(config), 2926 refresh_request_body=InterpolatedMapping( 2927 model.refresh_request_body or {}, parameters=model.parameters or {} 2928 ).eval(config), 2929 refresh_request_headers=InterpolatedMapping( 2930 model.refresh_request_headers or {}, parameters=model.parameters or {} 2931 ).eval(config), 2932 send_refresh_request_as_query_params=bool( 2933 model.send_refresh_request_as_query_params 2934 ), 2935 scopes=model.scopes, 2936 token_expiry_date_format=model.token_expiry_date_format, 2937 token_expiry_is_time_of_expiration=bool(model.token_expiry_date_format), 2938 message_repository=self._message_repository, 2939 refresh_token_error_status_codes=refresh_token_error_status_codes, 2940 refresh_token_error_key=refresh_token_error_key, 2941 refresh_token_error_values=refresh_token_error_values, 2942 ) 2943 # ignore type error because fixing it would have a lot of dependencies, revisit later 2944 return DeclarativeOauth2Authenticator( # type: ignore 2945 access_token_name=model.access_token_name or "access_token", 2946 access_token_value=model.access_token_value, 2947 client_id_name=model.client_id_name or "client_id", 2948 client_id=model.client_id, 2949 client_secret_name=model.client_secret_name or "client_secret", 2950 client_secret=model.client_secret, 2951 expires_in_name=model.expires_in_name or "expires_in", 2952 grant_type_name=model.grant_type_name or "grant_type", 2953 grant_type=model.grant_type or "refresh_token", 2954 refresh_request_body=model.refresh_request_body, 2955 refresh_request_headers=model.refresh_request_headers, 2956 send_refresh_request_as_query_params=bool(model.send_refresh_request_as_query_params), 2957 refresh_token_name=model.refresh_token_name or "refresh_token", 2958 refresh_token=model.refresh_token, 2959 scopes=model.scopes, 2960 token_expiry_date=model.token_expiry_date, 2961 token_expiry_date_format=model.token_expiry_date_format, 2962 token_expiry_is_time_of_expiration=bool(model.token_expiry_date_format), 2963 token_refresh_endpoint=model.token_refresh_endpoint, 2964 config=config, 2965 parameters=model.parameters or {}, 2966 message_repository=self._message_repository, 2967 profile_assertion=profile_assertion, 2968 use_profile_assertion=model.use_profile_assertion, 2969 refresh_token_error_status_codes=refresh_token_error_status_codes, 2970 refresh_token_error_key=refresh_token_error_key, 2971 refresh_token_error_values=refresh_token_error_values, 2972 ) 2973 2974 @staticmethod 2975 def _get_refresh_token_error_information( 2976 model: OAuthAuthenticatorModel, 2977 ) -> Tuple[Tuple[int, ...], str, Tuple[str, ...]]: 2978 """ 2979 In a previous version of the CDK, the auth error as config_error was only done if a refresh token updater was 2980 defined. As a transition, we added those fields on the OAuthAuthenticatorModel. This method ensures that the 2981 information is defined only once and return the right fields. 2982 """ 2983 refresh_token_updater = model.refresh_token_updater 2984 is_defined_on_refresh_token_updated = refresh_token_updater and ( 2985 refresh_token_updater.refresh_token_error_status_codes 2986 or refresh_token_updater.refresh_token_error_key 2987 or refresh_token_updater.refresh_token_error_values 2988 ) 2989 is_defined_on_oauth_authenticator = ( 2990 model.refresh_token_error_status_codes 2991 or model.refresh_token_error_key 2992 or model.refresh_token_error_values 2993 ) 2994 if is_defined_on_refresh_token_updated and is_defined_on_oauth_authenticator: 2995 raise ValueError( 2996 "refresh_token_error should either be defined on the OAuthAuthenticatorModel or the RefreshTokenUpdaterModel, not both" 2997 ) 2998 2999 if is_defined_on_refresh_token_updated: 3000 not_optional_refresh_token_updater: RefreshTokenUpdaterModel = refresh_token_updater # type: ignore # we know from the condition that this is not None 3001 return ( 3002 tuple(not_optional_refresh_token_updater.refresh_token_error_status_codes) 3003 if not_optional_refresh_token_updater.refresh_token_error_status_codes 3004 else (), 3005 not_optional_refresh_token_updater.refresh_token_error_key or "", 3006 tuple(not_optional_refresh_token_updater.refresh_token_error_values) 3007 if not_optional_refresh_token_updater.refresh_token_error_values 3008 else (), 3009 ) 3010 elif is_defined_on_oauth_authenticator: 3011 return ( 3012 tuple(model.refresh_token_error_status_codes) 3013 if model.refresh_token_error_status_codes 3014 else (), 3015 model.refresh_token_error_key or "", 3016 tuple(model.refresh_token_error_values) if model.refresh_token_error_values else (), 3017 ) 3018 3019 # returning default values we think cover most cases 3020 return (400,), "error", ("invalid_grant", "invalid_permissions") 3021 3022 def create_offset_increment( 3023 self, 3024 model: OffsetIncrementModel, 3025 config: Config, 3026 decoder: Decoder, 3027 extractor_model: Optional[Union[CustomRecordExtractorModel, DpathExtractorModel]] = None, 3028 **kwargs: Any, 3029 ) -> OffsetIncrement: 3030 if isinstance(decoder, PaginationDecoderDecorator): 3031 inner_decoder = decoder.decoder 3032 else: 3033 inner_decoder = decoder 3034 decoder = PaginationDecoderDecorator(decoder=decoder) 3035 3036 if self._is_supported_decoder_for_pagination(inner_decoder): 3037 decoder_to_use = decoder 3038 else: 3039 raise ValueError( 3040 self._UNSUPPORTED_DECODER_ERROR.format(decoder_type=type(inner_decoder)) 3041 ) 3042 3043 # Ideally we would instantiate the runtime extractor from highest most level (in this case the SimpleRetriever) 3044 # so that it can be shared by OffSetIncrement and RecordSelector. However, due to how we instantiate the 3045 # decoder with various decorators here, but not in create_record_selector, it is simpler to retain existing 3046 # behavior by having two separate extractors with identical behavior since they use the same extractor model. 3047 # When we have more time to investigate we can look into reusing the same component. 3048 extractor = ( 3049 self._create_component_from_model( 3050 model=extractor_model, config=config, decoder=decoder_to_use 3051 ) 3052 if extractor_model 3053 else None 3054 ) 3055 3056 # Pydantic v1 Union type coercion can convert int to string depending on Union order. 3057 # If page_size is a string that represents an integer (not an interpolation), convert it back. 3058 page_size = model.page_size 3059 if isinstance(page_size, str) and page_size.isdigit(): 3060 page_size = int(page_size) 3061 3062 return OffsetIncrement( 3063 page_size=page_size, 3064 config=config, 3065 decoder=decoder_to_use, 3066 extractor=extractor, 3067 inject_on_first_request=model.inject_on_first_request or False, 3068 parameters=model.parameters or {}, 3069 ) 3070 3071 def create_page_increment( 3072 self, 3073 model: PageIncrementModel, 3074 config: Config, 3075 decoder: Optional[Decoder] = None, 3076 extractor_model: Optional[Union[CustomRecordExtractorModel, DpathExtractorModel]] = None, 3077 **kwargs: Any, 3078 ) -> PageIncrement: 3079 # Like OffsetIncrement, we instantiate a separate extractor with identical behavior to the 3080 # RecordSelector's so the strategy can count the raw records in the response. This ensures 3081 # pagination is driven by the API's page size, not the post-filter record count. 3082 extractor = ( 3083 self._create_component_from_model(model=extractor_model, config=config, decoder=decoder) 3084 if extractor_model 3085 else None 3086 ) 3087 3088 # Pydantic v1 Union type coercion can convert int to string depending on Union order. 3089 # If page_size is a string that represents an integer (not an interpolation), convert it back. 3090 page_size = model.page_size 3091 if isinstance(page_size, str) and page_size.isdigit(): 3092 page_size = int(page_size) 3093 3094 return PageIncrement( 3095 page_size=page_size, 3096 config=config, 3097 start_from_page=model.start_from_page or 0, 3098 inject_on_first_request=model.inject_on_first_request or False, 3099 extractor=extractor, 3100 parameters=model.parameters or {}, 3101 ) 3102 3103 def create_parent_stream_config( 3104 self, model: ParentStreamConfigModel, config: Config, *, stream_name: str, **kwargs: Any 3105 ) -> ParentStreamConfig: 3106 declarative_stream = self._create_component_from_model( 3107 model.stream, 3108 config=config, 3109 is_parent=True, 3110 **kwargs, 3111 ) 3112 request_option = ( 3113 self._create_component_from_model(model.request_option, config=config) 3114 if model.request_option 3115 else None 3116 ) 3117 3118 if model.lazy_read_pointer and any("*" in pointer for pointer in model.lazy_read_pointer): 3119 raise ValueError( 3120 "The '*' wildcard in 'lazy_read_pointer' is not supported — only direct paths are allowed." 3121 ) 3122 3123 model_lazy_read_pointer: List[Union[InterpolatedString, str]] = ( 3124 [x for x in model.lazy_read_pointer] if model.lazy_read_pointer else [] 3125 ) 3126 3127 return ParentStreamConfig( 3128 parent_key=model.parent_key, 3129 request_option=request_option, 3130 stream=declarative_stream, 3131 partition_field=model.partition_field, 3132 config=config, 3133 incremental_dependency=model.incremental_dependency or False, 3134 parameters=model.parameters or {}, 3135 extra_fields=model.extra_fields, 3136 lazy_read_pointer=model_lazy_read_pointer, 3137 ) 3138 3139 def create_properties_from_endpoint( 3140 self, model: PropertiesFromEndpointModel, config: Config, **kwargs: Any 3141 ) -> PropertiesFromEndpoint: 3142 retriever = self._create_component_from_model( 3143 model=model.retriever, 3144 config=config, 3145 name="dynamic_properties", 3146 primary_key=None, 3147 stream_slicer=None, 3148 transformations=[], 3149 use_cache=True, # Enable caching on the HttpRequester/HttpClient because the properties endpoint will be called for every slice being processed, and it is highly unlikely for the response to different 3150 ) 3151 return PropertiesFromEndpoint( 3152 property_field_path=model.property_field_path, 3153 retriever=retriever, 3154 config=config, 3155 parameters=model.parameters or {}, 3156 ) 3157 3158 def create_property_chunking( 3159 self, model: PropertyChunkingModel, config: Config, **kwargs: Any 3160 ) -> PropertyChunking: 3161 record_merge_strategy = ( 3162 self._create_component_from_model( 3163 model=model.record_merge_strategy, config=config, **kwargs 3164 ) 3165 if model.record_merge_strategy 3166 else None 3167 ) 3168 3169 property_limit_type: PropertyLimitType 3170 match model.property_limit_type: 3171 case PropertyLimitTypeModel.property_count: 3172 property_limit_type = PropertyLimitType.property_count 3173 case PropertyLimitTypeModel.characters: 3174 property_limit_type = PropertyLimitType.characters 3175 case _: 3176 raise ValueError(f"Invalid PropertyLimitType {property_limit_type}") 3177 3178 return PropertyChunking( 3179 property_limit_type=property_limit_type, 3180 property_limit=model.property_limit, 3181 record_merge_strategy=record_merge_strategy, 3182 config=config, 3183 parameters=model.parameters or {}, 3184 ) 3185 3186 def create_query_properties( 3187 self, model: QueryPropertiesModel, config: Config, *, stream_name: str, **kwargs: Any 3188 ) -> QueryProperties: 3189 if isinstance(model.property_list, list): 3190 property_list = model.property_list 3191 else: 3192 property_list = self._create_component_from_model( 3193 model=model.property_list, config=config, **kwargs 3194 ) 3195 3196 property_chunking = ( 3197 self._create_component_from_model( 3198 model=model.property_chunking, config=config, **kwargs 3199 ) 3200 if model.property_chunking 3201 else None 3202 ) 3203 3204 property_selector = ( 3205 self._create_component_from_model( 3206 model=model.property_selector, config=config, stream_name=stream_name, **kwargs 3207 ) 3208 if model.property_selector 3209 else None 3210 ) 3211 3212 return QueryProperties( 3213 property_list=property_list, 3214 always_include_properties=model.always_include_properties, 3215 property_chunking=property_chunking, 3216 property_selector=property_selector, 3217 config=config, 3218 parameters=model.parameters or {}, 3219 ) 3220 3221 def create_json_schema_property_selector( 3222 self, 3223 model: JsonSchemaPropertySelectorModel, 3224 config: Config, 3225 *, 3226 stream_name: str, 3227 **kwargs: Any, 3228 ) -> JsonSchemaPropertySelector: 3229 configured_stream = self._stream_name_to_configured_stream.get(stream_name) 3230 3231 transformations = [] 3232 if model.transformations: 3233 for transformation_model in model.transformations: 3234 transformations.append( 3235 self._create_component_from_model(model=transformation_model, config=config) 3236 ) 3237 3238 return JsonSchemaPropertySelector( 3239 configured_stream=configured_stream, 3240 properties_transformations=transformations, 3241 config=config, 3242 parameters=model.parameters or {}, 3243 ) 3244 3245 @staticmethod 3246 def create_record_filter( 3247 model: RecordFilterModel, config: Config, **kwargs: Any 3248 ) -> RecordFilter: 3249 return RecordFilter( 3250 condition=model.condition or "", config=config, parameters=model.parameters or {} 3251 ) 3252 3253 @staticmethod 3254 def create_request_path(model: RequestPathModel, config: Config, **kwargs: Any) -> RequestPath: 3255 return RequestPath(parameters={}) 3256 3257 @staticmethod 3258 def create_request_option( 3259 model: RequestOptionModel, config: Config, **kwargs: Any 3260 ) -> RequestOption: 3261 inject_into = RequestOptionType(model.inject_into.value) 3262 field_path: Optional[List[Union[InterpolatedString, str]]] = ( 3263 [ 3264 InterpolatedString.create(segment, parameters=kwargs.get("parameters", {})) 3265 for segment in model.field_path 3266 ] 3267 if model.field_path 3268 else None 3269 ) 3270 field_name = ( 3271 InterpolatedString.create(model.field_name, parameters=kwargs.get("parameters", {})) 3272 if model.field_name 3273 else None 3274 ) 3275 return RequestOption( 3276 field_name=field_name, 3277 field_path=field_path, 3278 inject_into=inject_into, 3279 parameters=kwargs.get("parameters", {}), 3280 ) 3281 3282 def create_record_selector( 3283 self, 3284 model: RecordSelectorModel, 3285 config: Config, 3286 *, 3287 name: str, 3288 transformations: List[RecordTransformation] | None = None, 3289 decoder: Decoder | None = None, 3290 client_side_incremental_sync_cursor: Optional[Cursor] = None, 3291 file_uploader: Optional[DefaultFileUploader] = None, 3292 **kwargs: Any, 3293 ) -> RecordSelector: 3294 extractor = self._create_component_from_model( 3295 model=model.extractor, decoder=decoder, config=config 3296 ) 3297 record_filter = ( 3298 self._create_component_from_model(model.record_filter, config=config) 3299 if model.record_filter 3300 else None 3301 ) 3302 3303 transform_before_filtering = ( 3304 False if model.transform_before_filtering is None else model.transform_before_filtering 3305 ) 3306 if client_side_incremental_sync_cursor: 3307 record_filter = ClientSideIncrementalRecordFilterDecorator( 3308 config=config, 3309 parameters=model.parameters, 3310 condition=model.record_filter.condition 3311 if (model.record_filter and hasattr(model.record_filter, "condition")) 3312 else None, 3313 cursor=client_side_incremental_sync_cursor, 3314 ) 3315 transform_before_filtering = ( 3316 True 3317 if model.transform_before_filtering is None 3318 else model.transform_before_filtering 3319 ) 3320 3321 if model.schema_normalization is None: 3322 # default to no schema normalization if not set 3323 model.schema_normalization = SchemaNormalizationModel.None_ 3324 3325 schema_normalization = ( 3326 TypeTransformer(SCHEMA_TRANSFORMER_TYPE_MAPPING[model.schema_normalization]) 3327 if isinstance(model.schema_normalization, SchemaNormalizationModel) 3328 else self._create_component_from_model(model.schema_normalization, config=config) # type: ignore[arg-type] # custom normalization model expected here 3329 ) 3330 3331 return RecordSelector( 3332 extractor=extractor, 3333 name=name, 3334 config=config, 3335 record_filter=record_filter, 3336 transformations=transformations or [], 3337 file_uploader=file_uploader, 3338 schema_normalization=schema_normalization, 3339 parameters=model.parameters or {}, 3340 transform_before_filtering=transform_before_filtering, 3341 ) 3342 3343 @staticmethod 3344 def create_remove_fields( 3345 model: RemoveFieldsModel, config: Config, **kwargs: Any 3346 ) -> RemoveFields: 3347 return RemoveFields( 3348 field_pointers=model.field_pointers, condition=model.condition or "", parameters={} 3349 ) 3350 3351 def create_selective_authenticator( 3352 self, model: SelectiveAuthenticatorModel, config: Config, **kwargs: Any 3353 ) -> DeclarativeAuthenticator: 3354 authenticators = { 3355 name: self._create_component_from_model(model=auth, config=config) 3356 for name, auth in model.authenticators.items() 3357 } 3358 # SelectiveAuthenticator will return instance of DeclarativeAuthenticator or raise ValueError error 3359 return SelectiveAuthenticator( # type: ignore[abstract] 3360 config=config, 3361 authenticators=authenticators, 3362 authenticator_selection_path=model.authenticator_selection_path, 3363 **kwargs, 3364 ) 3365 3366 @staticmethod 3367 def create_legacy_session_token_authenticator( 3368 model: LegacySessionTokenAuthenticatorModel, config: Config, *, url_base: str, **kwargs: Any 3369 ) -> LegacySessionTokenAuthenticator: 3370 return LegacySessionTokenAuthenticator( 3371 api_url=url_base, 3372 header=model.header, 3373 login_url=model.login_url, 3374 password=model.password or "", 3375 session_token=model.session_token or "", 3376 session_token_response_key=model.session_token_response_key or "", 3377 username=model.username or "", 3378 validate_session_url=model.validate_session_url, 3379 config=config, 3380 parameters=model.parameters or {}, 3381 ) 3382 3383 def create_simple_retriever( 3384 self, 3385 model: SimpleRetrieverModel, 3386 config: Config, 3387 *, 3388 name: str, 3389 primary_key: Optional[Union[str, List[str], List[List[str]]]], 3390 request_options_provider: Optional[RequestOptionsProvider] = None, 3391 cursor: Optional[Cursor] = None, 3392 has_stop_condition_cursor: bool = False, 3393 is_client_side_incremental_sync: bool = False, 3394 transformations: List[RecordTransformation], 3395 file_uploader: Optional[DefaultFileUploader] = None, 3396 incremental_sync: Optional[ 3397 Union[IncrementingCountCursorModel, DatetimeBasedCursorModel] 3398 ] = None, 3399 use_cache: Optional[bool] = None, 3400 log_formatter: Optional[Callable[[Response], Any]] = None, 3401 partition_router: Optional[PartitionRouter] = None, 3402 **kwargs: Any, 3403 ) -> SimpleRetriever: 3404 def _get_url(req: Requester) -> str: 3405 """ 3406 Closure to get the URL from the requester. This is used to get the URL in the case of a lazy retriever. 3407 This is needed because the URL is not set until the requester is created. 3408 """ 3409 3410 _url: str = ( 3411 model.requester.url 3412 if hasattr(model.requester, "url") and model.requester.url is not None 3413 else req.get_url(stream_state=None, stream_slice=None, next_page_token=None) 3414 ) 3415 _url_base: str = ( 3416 model.requester.url_base 3417 if hasattr(model.requester, "url_base") and model.requester.url_base is not None 3418 else req.get_url_base(stream_state=None, stream_slice=None, next_page_token=None) 3419 ) 3420 3421 return _url or _url_base 3422 3423 if cursor is None: 3424 cursor = FinalStateCursor(name, None, self._message_repository) 3425 3426 decoder = ( 3427 self._create_component_from_model(model=model.decoder, config=config) 3428 if model.decoder 3429 else JsonDecoder(parameters={}) 3430 ) 3431 record_selector = self._create_component_from_model( 3432 model=model.record_selector, 3433 name=name, 3434 config=config, 3435 decoder=decoder, 3436 transformations=transformations, 3437 client_side_incremental_sync_cursor=cursor if is_client_side_incremental_sync else None, 3438 file_uploader=file_uploader, 3439 ) 3440 3441 query_properties: Optional[QueryProperties] = None 3442 query_properties_key: Optional[str] = None 3443 self._ensure_query_properties_to_model(model.requester) 3444 if self._has_query_properties_in_request_parameters(model.requester): 3445 # It is better to be explicit about an error if PropertiesFromEndpoint is defined in multiple 3446 # places instead of default to request_parameters which isn't clearly documented 3447 if ( 3448 hasattr(model.requester, "fetch_properties_from_endpoint") 3449 and model.requester.fetch_properties_from_endpoint 3450 ): 3451 raise ValueError( 3452 f"PropertiesFromEndpoint should only be specified once per stream, but found in {model.requester.type}.fetch_properties_from_endpoint and {model.requester.type}.request_parameters" 3453 ) 3454 3455 query_properties_definitions = [] 3456 for key, request_parameter in model.requester.request_parameters.items(): # type: ignore # request_parameters is already validated to be a Mapping using _has_query_properties_in_request_parameters() 3457 if isinstance(request_parameter, QueryPropertiesModel): 3458 query_properties_key = key 3459 query_properties_definitions.append(request_parameter) 3460 3461 if len(query_properties_definitions) > 1: 3462 raise ValueError( 3463 f"request_parameters only supports defining one QueryProperties field, but found {len(query_properties_definitions)} usages" 3464 ) 3465 3466 if len(query_properties_definitions) == 1: 3467 query_properties = self._create_component_from_model( 3468 model=query_properties_definitions[0], stream_name=name, config=config 3469 ) 3470 3471 # Removes QueryProperties components from the interpolated mappings because it has been designed 3472 # to be used by the SimpleRetriever and will be resolved from the provider from the slice directly 3473 # instead of through jinja interpolation 3474 if hasattr(model.requester, "request_parameters") and isinstance( 3475 model.requester.request_parameters, Mapping 3476 ): 3477 model.requester.request_parameters = self._remove_query_properties( 3478 model.requester.request_parameters 3479 ) 3480 elif ( 3481 hasattr(model.requester, "fetch_properties_from_endpoint") 3482 and model.requester.fetch_properties_from_endpoint 3483 ): 3484 # todo: Deprecate this condition once dependent connectors migrate to query_properties 3485 query_properties_definition = QueryPropertiesModel( 3486 type="QueryProperties", 3487 property_list=model.requester.fetch_properties_from_endpoint, 3488 always_include_properties=None, 3489 property_chunking=None, 3490 ) # type: ignore # $parameters has a default value 3491 3492 query_properties = self.create_query_properties( 3493 model=query_properties_definition, 3494 stream_name=name, 3495 config=config, 3496 ) 3497 elif hasattr(model.requester, "query_properties") and model.requester.query_properties: 3498 query_properties = self.create_query_properties( 3499 model=model.requester.query_properties, 3500 stream_name=name, 3501 config=config, 3502 ) 3503 3504 requester = self._create_component_from_model( 3505 model=model.requester, 3506 decoder=decoder, 3507 name=name, 3508 query_properties_key=query_properties_key, 3509 use_cache=use_cache, 3510 config=config, 3511 ) 3512 3513 if not request_options_provider: 3514 request_options_provider = DefaultRequestOptionsProvider(parameters={}) 3515 if isinstance(request_options_provider, DefaultRequestOptionsProvider) and isinstance( 3516 partition_router, PartitionRouter 3517 ): 3518 request_options_provider = partition_router 3519 3520 paginator = ( 3521 self._create_component_from_model( 3522 model=model.paginator, 3523 config=config, 3524 url_base=_get_url(requester), 3525 extractor_model=model.record_selector.extractor, 3526 decoder=decoder, 3527 cursor_used_for_stop_condition=cursor if has_stop_condition_cursor else None, 3528 ) 3529 if model.paginator 3530 else NoPagination(parameters={}) 3531 ) 3532 3533 ignore_stream_slicer_parameters_on_paginated_requests = ( 3534 model.ignore_stream_slicer_parameters_on_paginated_requests or False 3535 ) 3536 3537 if ( 3538 model.partition_router 3539 and isinstance(model.partition_router, SubstreamPartitionRouterModel) 3540 and not bool(self._connector_state_manager.get_stream_state(name, None)) 3541 and any( 3542 parent_stream_config.lazy_read_pointer 3543 for parent_stream_config in model.partition_router.parent_stream_configs 3544 ) 3545 ): 3546 if incremental_sync: 3547 if incremental_sync.type != "DatetimeBasedCursor": 3548 raise ValueError( 3549 f"LazySimpleRetriever only supports DatetimeBasedCursor. Found: {incremental_sync.type}." 3550 ) 3551 3552 elif incremental_sync.step or incremental_sync.cursor_granularity: 3553 raise ValueError( 3554 f"Found more that one slice per parent. LazySimpleRetriever only supports single slice read for stream - {name}." 3555 ) 3556 3557 if model.decoder and model.decoder.type != "JsonDecoder": 3558 raise ValueError( 3559 f"LazySimpleRetriever only supports JsonDecoder. Found: {model.decoder.type}." 3560 ) 3561 3562 return LazySimpleRetriever( 3563 name=name, 3564 paginator=paginator, 3565 primary_key=primary_key, 3566 requester=requester, 3567 record_selector=record_selector, 3568 stream_slicer=_NO_STREAM_SLICING, 3569 request_option_provider=request_options_provider, 3570 config=config, 3571 ignore_stream_slicer_parameters_on_paginated_requests=ignore_stream_slicer_parameters_on_paginated_requests, 3572 parameters=model.parameters or {}, 3573 ) 3574 3575 if ( 3576 model.record_selector.record_filter 3577 and model.pagination_reset 3578 and model.pagination_reset.limits 3579 ): 3580 raise ValueError("PaginationResetLimits are not supported while having record filter.") 3581 3582 return SimpleRetriever( 3583 name=name, 3584 paginator=paginator, 3585 primary_key=primary_key, 3586 requester=requester, 3587 record_selector=record_selector, 3588 stream_slicer=_NO_STREAM_SLICING, 3589 request_option_provider=request_options_provider, 3590 config=config, 3591 ignore_stream_slicer_parameters_on_paginated_requests=ignore_stream_slicer_parameters_on_paginated_requests, 3592 additional_query_properties=query_properties, 3593 log_formatter=self._get_log_formatter(log_formatter, name), 3594 pagination_tracker_factory=self._create_pagination_tracker_factory( 3595 model.pagination_reset, cursor 3596 ), 3597 parameters=model.parameters or {}, 3598 ) 3599 3600 def _create_pagination_tracker_factory( 3601 self, model: Optional[PaginationResetModel], cursor: Cursor 3602 ) -> Callable[[], PaginationTracker]: 3603 if model is None: 3604 return lambda: PaginationTracker() 3605 3606 # Until we figure out a way to use any cursor for PaginationTracker, we will have to have this cursor selector logic 3607 cursor_factory: Callable[[], Optional[ConcurrentCursor]] = lambda: None 3608 if model.action == PaginationResetActionModel.RESET: 3609 # in that case, we will let cursor_factory to return None even if the stream has a cursor 3610 pass 3611 elif model.action == PaginationResetActionModel.SPLIT_USING_CURSOR: 3612 if isinstance(cursor, ConcurrentCursor): 3613 cursor_factory = lambda: cursor.copy_without_state() # type: ignore # the if condition validates that it is a ConcurrentCursor 3614 elif isinstance(cursor, ConcurrentPerPartitionCursor): 3615 cursor_factory = lambda: cursor._cursor_factory.create( # type: ignore # if this becomes a problem, we would need to extract the cursor_factory instantiation logic and make it accessible here 3616 {}, datetime.timedelta(0) 3617 ) 3618 elif not isinstance(cursor, FinalStateCursor): 3619 LOGGER.warning( 3620 "Unknown cursor for PaginationTracker. Pagination resets might not work properly" 3621 ) 3622 else: 3623 raise ValueError(f"Unknown PaginationReset action: {model.action}") 3624 3625 limit = model.limits.number_of_records if model and model.limits else None 3626 return lambda: PaginationTracker(cursor_factory(), limit) 3627 3628 def _get_log_formatter( 3629 self, log_formatter: Callable[[Response], Any] | None, name: str 3630 ) -> Callable[[Response], Any] | None: 3631 if self._should_limit_slices_fetched(): 3632 return ( 3633 ( 3634 lambda response: format_http_message( 3635 response, 3636 f"Stream '{name}' request", 3637 f"Request performed in order to extract records for stream '{name}'", 3638 name, 3639 ) 3640 ) 3641 if not log_formatter 3642 else log_formatter 3643 ) 3644 return None 3645 3646 def _should_limit_slices_fetched(self) -> bool: 3647 """ 3648 Returns True if the number of slices fetched should be limited, False otherwise. 3649 This is used to limit the number of slices fetched during tests. 3650 """ 3651 return bool(self._limit_slices_fetched or self._emit_connector_builder_messages) 3652 3653 @staticmethod 3654 def _has_query_properties_in_request_parameters( 3655 requester: Union[HttpRequesterModel, CustomRequesterModel], 3656 ) -> bool: 3657 if not hasattr(requester, "request_parameters"): 3658 return False 3659 request_parameters = requester.request_parameters 3660 if request_parameters and isinstance(request_parameters, Mapping): 3661 for request_parameter in request_parameters.values(): 3662 if isinstance(request_parameter, QueryPropertiesModel): 3663 return True 3664 return False 3665 3666 @staticmethod 3667 def _remove_query_properties( 3668 request_parameters: Mapping[str, Union[str, QueryPropertiesModel]], 3669 ) -> Mapping[str, str]: 3670 return { 3671 parameter_field: request_parameter 3672 for parameter_field, request_parameter in request_parameters.items() 3673 if not isinstance(request_parameter, QueryPropertiesModel) 3674 } 3675 3676 def create_state_delegating_stream( 3677 self, 3678 model: StateDelegatingStreamModel, 3679 config: Config, 3680 **kwargs: Any, 3681 ) -> DefaultStream: 3682 if ( 3683 model.full_refresh_stream.name != model.name 3684 or model.name != model.incremental_stream.name 3685 ): 3686 raise ValueError( 3687 f"state_delegating_stream, full_refresh_stream name and incremental_stream must have equal names. Instead has {model.name}, {model.full_refresh_stream.name} and {model.incremental_stream.name}." 3688 ) 3689 3690 # Resolve api_retention_period with config context (supports Jinja2 interpolation) 3691 resolved_retention_period: Optional[str] = None 3692 if model.api_retention_period: 3693 interpolated_retention = InterpolatedString.create( 3694 model.api_retention_period, parameters=model.parameters or {} 3695 ) 3696 resolved_value = interpolated_retention.eval(config=config) 3697 if resolved_value: 3698 resolved_retention_period = str(resolved_value) 3699 3700 if resolved_retention_period: 3701 for stream_model in (model.full_refresh_stream, model.incremental_stream): 3702 if isinstance(stream_model.incremental_sync, IncrementingCountCursorModel): 3703 raise ValueError( 3704 f"Stream '{model.name}' uses IncrementingCountCursor which is not supported " 3705 f"with api_retention_period. IncrementingCountCursor does not use datetime-based " 3706 f"cursors, so cursor age validation cannot be performed." 3707 ) 3708 3709 stream_state = self._connector_state_manager.get_stream_state(model.name, None) 3710 3711 if not stream_state: 3712 return self._create_component_from_model( # type: ignore[no-any-return] 3713 model.full_refresh_stream, config=config, **kwargs 3714 ) 3715 3716 incremental_stream: DefaultStream = self._create_component_from_model( 3717 model.incremental_stream, config=config, **kwargs 3718 ) # type: ignore[assignment] 3719 3720 # Only run cursor age validation for streams that are in the configured 3721 # catalog (or when no catalog was provided, e.g. during discover / connector 3722 # builder). Streams not selected by the user but instantiated as parent-stream 3723 # dependencies must not go through this path because it emits state messages 3724 # that the destination does not know about, causing "Stream not found" crashes. 3725 stream_is_in_catalog = ( 3726 not self._stream_name_to_configured_stream # no catalog → validate by default 3727 or model.name in self._stream_name_to_configured_stream 3728 ) 3729 if resolved_retention_period and stream_is_in_catalog: 3730 full_refresh_stream: DefaultStream = self._create_component_from_model( 3731 model.full_refresh_stream, config=config, **kwargs 3732 ) # type: ignore[assignment] 3733 if self._is_cursor_older_than_retention_period( 3734 stream_state, 3735 full_refresh_stream.cursor, 3736 incremental_stream.cursor, 3737 resolved_retention_period, 3738 model.name, 3739 ): 3740 # Clear state BEFORE constructing the full_refresh_stream so that 3741 # its cursor starts from start_date instead of the stale cursor. 3742 self._connector_state_manager.update_state_for_stream(model.name, None, {}) 3743 state_message = self._connector_state_manager.create_state_message(model.name, None) 3744 self._message_repository.emit_message(state_message) 3745 return self._create_component_from_model( # type: ignore[no-any-return] 3746 model.full_refresh_stream, config=config, **kwargs 3747 ) 3748 3749 return incremental_stream 3750 3751 @staticmethod 3752 def _is_cursor_older_than_retention_period( 3753 stream_state: Mapping[str, Any], 3754 full_refresh_cursor: Cursor, 3755 incremental_cursor: Cursor, 3756 api_retention_period: str, 3757 stream_name: str, 3758 ) -> bool: 3759 """Check if the cursor value in the state is older than the API's retention period. 3760 3761 Checks cursors in sequence: full refresh cursor first, then incremental cursor. 3762 FinalStateCursor returns now() for completed full refresh state (NO_CURSOR_STATE_KEY), 3763 which is always within retention, so we use incremental. For other states, it returns 3764 None and we fall back to checking the incremental cursor. 3765 3766 Returns True if the cursor is older than the retention period (should use full refresh). 3767 Returns False if the cursor is within the retention period (safe to use incremental). 3768 """ 3769 retention_duration = parse_duration(api_retention_period) 3770 retention_cutoff = datetime.datetime.now(datetime.timezone.utc) - retention_duration 3771 3772 # Check full refresh cursor first 3773 cursor_datetime = full_refresh_cursor.get_cursor_datetime_from_state(stream_state) 3774 3775 # If full refresh cursor returns None, check incremental cursor 3776 if cursor_datetime is None: 3777 cursor_datetime = incremental_cursor.get_cursor_datetime_from_state(stream_state) 3778 3779 if cursor_datetime is None: 3780 # Neither cursor could parse the state - fall back to full refresh to be safe 3781 return True 3782 3783 if cursor_datetime < retention_cutoff: 3784 logging.warning( 3785 f"Stream '{stream_name}' has a cursor value older than " 3786 f"the API's retention period of {api_retention_period} " 3787 f"(cutoff: {retention_cutoff.isoformat()}). " 3788 f"Falling back to full refresh to avoid data loss." 3789 ) 3790 return True 3791 3792 return False 3793 3794 def _get_state_delegating_stream_model( 3795 self, 3796 model: StateDelegatingStreamModel, 3797 parent_state: Optional[Mapping[str, Any]] = None, 3798 ) -> DeclarativeStreamModel: 3799 """Return the appropriate underlying stream model based on state.""" 3800 return ( 3801 model.incremental_stream 3802 if self._connector_state_manager.get_stream_state(model.name, None) or parent_state 3803 else model.full_refresh_stream 3804 ) 3805 3806 _OPTIONAL_ASYNC_STATUS_FIELDS = {"skipped"} 3807 3808 def _create_async_job_status_mapping( 3809 self, model: AsyncJobStatusMapModel, config: Config, **kwargs: Any 3810 ) -> Mapping[str, AsyncJobStatus]: 3811 api_status_to_cdk_status = {} 3812 for cdk_status, api_statuses in model.dict().items(): 3813 if cdk_status == "type": 3814 # This is an element of the dict because of the typing of the CDK but it is not a CDK status 3815 continue 3816 3817 if api_statuses is None: 3818 if cdk_status in self._OPTIONAL_ASYNC_STATUS_FIELDS: 3819 continue 3820 raise ValueError( 3821 f"Required CDK status '{cdk_status}' has no API statuses mapped. " 3822 f"Please provide at least an empty list for required status fields." 3823 ) 3824 3825 for status in api_statuses: 3826 if status in api_status_to_cdk_status: 3827 raise ValueError( 3828 f"API status {status} is already set for CDK status {cdk_status}. Please ensure API statuses are only provided once" 3829 ) 3830 api_status_to_cdk_status[status] = self._get_async_job_status(cdk_status) 3831 return api_status_to_cdk_status 3832 3833 def _get_async_job_status(self, status: str) -> AsyncJobStatus: 3834 match status: 3835 case "running": 3836 return AsyncJobStatus.RUNNING 3837 case "completed": 3838 return AsyncJobStatus.COMPLETED 3839 case "failed": 3840 return AsyncJobStatus.FAILED 3841 case "timeout": 3842 return AsyncJobStatus.TIMED_OUT 3843 case "skipped": 3844 return AsyncJobStatus.SKIPPED 3845 case _: 3846 raise ValueError(f"Unsupported CDK status {status}") 3847 3848 def create_async_retriever( 3849 self, 3850 model: AsyncRetrieverModel, 3851 config: Config, 3852 *, 3853 name: str, 3854 primary_key: Optional[ 3855 Union[str, List[str], List[List[str]]] 3856 ], # this seems to be needed to match create_simple_retriever 3857 stream_slicer: Optional[StreamSlicer], 3858 client_side_incremental_sync: Optional[Dict[str, Any]] = None, 3859 transformations: List[RecordTransformation], 3860 **kwargs: Any, 3861 ) -> AsyncRetriever: 3862 if model.download_target_requester and not model.download_target_extractor: 3863 raise ValueError( 3864 f"`download_target_extractor` required if using a `download_target_requester`" 3865 ) 3866 3867 def _get_download_retriever( 3868 requester: Requester, extractor: RecordExtractor, _decoder: Decoder 3869 ) -> SimpleRetriever: 3870 # We create a record selector for the download retriever 3871 # with no schema normalization and no transformations, neither record filter 3872 # as all this occurs in the record_selector of the AsyncRetriever 3873 record_selector = RecordSelector( 3874 extractor=extractor, 3875 name=name, 3876 record_filter=None, 3877 transformations=[], 3878 schema_normalization=TypeTransformer(TransformConfig.NoTransform), 3879 config=config, 3880 parameters={}, 3881 ) 3882 paginator = ( 3883 self._create_component_from_model( 3884 model=model.download_paginator, 3885 decoder=_decoder, 3886 config=config, 3887 url_base="", 3888 ) 3889 if model.download_paginator 3890 else NoPagination(parameters={}) 3891 ) 3892 3893 return SimpleRetriever( 3894 requester=requester, 3895 record_selector=record_selector, 3896 primary_key=None, 3897 name=name, 3898 paginator=paginator, 3899 config=config, 3900 parameters={}, 3901 log_formatter=self._get_log_formatter(None, name), 3902 ) 3903 3904 def _get_job_timeout() -> datetime.timedelta: 3905 user_defined_timeout: Optional[int] = ( 3906 int( 3907 InterpolatedString.create( 3908 str(model.polling_job_timeout), 3909 parameters={}, 3910 ).eval(config) 3911 ) 3912 if model.polling_job_timeout 3913 else None 3914 ) 3915 3916 # check for user defined timeout during the test read or 15 minutes 3917 test_read_timeout = datetime.timedelta(minutes=user_defined_timeout or 15) 3918 # default value for non-connector builder is 60 minutes. 3919 default_sync_timeout = datetime.timedelta(minutes=user_defined_timeout or 60) 3920 3921 return ( 3922 test_read_timeout if self._emit_connector_builder_messages else default_sync_timeout 3923 ) 3924 3925 decoder = ( 3926 self._create_component_from_model(model=model.decoder, config=config) 3927 if model.decoder 3928 else JsonDecoder(parameters={}) 3929 ) 3930 record_selector = self._create_component_from_model( 3931 model=model.record_selector, 3932 config=config, 3933 decoder=decoder, 3934 name=name, 3935 transformations=transformations, 3936 client_side_incremental_sync=client_side_incremental_sync, 3937 ) 3938 3939 stream_slicer = stream_slicer or SinglePartitionRouter(parameters={}) 3940 if self._should_limit_slices_fetched(): 3941 stream_slicer = cast( 3942 StreamSlicer, 3943 StreamSlicerTestReadDecorator( 3944 wrapped_slicer=stream_slicer, 3945 maximum_number_of_slices=self._limit_slices_fetched or 5, 3946 ), 3947 ) 3948 3949 creation_requester = self._create_component_from_model( 3950 model=model.creation_requester, 3951 decoder=decoder, 3952 config=config, 3953 name=f"job creation - {name}", 3954 ) 3955 polling_requester = self._create_component_from_model( 3956 model=model.polling_requester, 3957 decoder=decoder, 3958 config=config, 3959 name=f"job polling - {name}", 3960 ) 3961 job_download_components_name = f"job download - {name}" 3962 download_decoder = ( 3963 self._create_component_from_model(model=model.download_decoder, config=config) 3964 if model.download_decoder 3965 else JsonDecoder(parameters={}) 3966 ) 3967 download_extractor = ( 3968 self._create_component_from_model( 3969 model=model.download_extractor, 3970 config=config, 3971 decoder=download_decoder, 3972 parameters=model.parameters, 3973 ) 3974 if model.download_extractor 3975 else DpathExtractor( 3976 [], 3977 config=config, 3978 decoder=download_decoder, 3979 parameters=model.parameters or {}, 3980 ) 3981 ) 3982 download_requester = self._create_component_from_model( 3983 model=model.download_requester, 3984 decoder=download_decoder, 3985 config=config, 3986 name=job_download_components_name, 3987 ) 3988 download_retriever = _get_download_retriever( 3989 download_requester, download_extractor, download_decoder 3990 ) 3991 abort_requester = ( 3992 self._create_component_from_model( 3993 model=model.abort_requester, 3994 decoder=decoder, 3995 config=config, 3996 name=f"job abort - {name}", 3997 ) 3998 if model.abort_requester 3999 else None 4000 ) 4001 delete_requester = ( 4002 self._create_component_from_model( 4003 model=model.delete_requester, 4004 decoder=decoder, 4005 config=config, 4006 name=f"job delete - {name}", 4007 ) 4008 if model.delete_requester 4009 else None 4010 ) 4011 download_target_requester = ( 4012 self._create_component_from_model( 4013 model=model.download_target_requester, 4014 decoder=decoder, 4015 config=config, 4016 name=f"job extract_url - {name}", 4017 ) 4018 if model.download_target_requester 4019 else None 4020 ) 4021 status_extractor = self._create_component_from_model( 4022 model=model.status_extractor, decoder=decoder, config=config, name=name 4023 ) 4024 download_target_extractor = ( 4025 self._create_component_from_model( 4026 model=model.download_target_extractor, 4027 decoder=decoder, 4028 config=config, 4029 name=name, 4030 ) 4031 if model.download_target_extractor 4032 else None 4033 ) 4034 4035 job_repository: AsyncJobRepository = AsyncHttpJobRepository( 4036 creation_requester=creation_requester, 4037 polling_requester=polling_requester, 4038 download_retriever=download_retriever, 4039 download_target_requester=download_target_requester, 4040 abort_requester=abort_requester, 4041 delete_requester=delete_requester, 4042 status_extractor=status_extractor, 4043 status_mapping=self._create_async_job_status_mapping(model.status_mapping, config), 4044 download_target_extractor=download_target_extractor, 4045 job_timeout=_get_job_timeout(), 4046 ) 4047 4048 failed_retry_wait_time_in_seconds: Optional[int] = ( 4049 int( 4050 InterpolatedString.create( 4051 str(model.failed_retry_wait_time_in_seconds), 4052 parameters={}, 4053 ).eval(config) 4054 ) 4055 if model.failed_retry_wait_time_in_seconds 4056 else None 4057 ) 4058 4059 async_job_partition_router = AsyncJobPartitionRouter( 4060 job_orchestrator_factory=lambda stream_slices: AsyncJobOrchestrator( 4061 job_repository, 4062 stream_slices, 4063 self._job_tracker, 4064 self._message_repository, 4065 # FIXME work would need to be done here in order to detect if a stream as a parent stream that is bulk 4066 has_bulk_parent=False, 4067 # set the `job_max_retry` to 1 for the `Connector Builder`` use-case. 4068 # `None` == default retry is set to 3 attempts, under the hood. 4069 job_max_retry=1 if self._emit_connector_builder_messages else None, 4070 failed_retry_wait_time_in_seconds=failed_retry_wait_time_in_seconds, 4071 ), 4072 stream_slicer=stream_slicer, 4073 config=config, 4074 parameters=model.parameters or {}, 4075 ) 4076 4077 return AsyncRetriever( 4078 record_selector=record_selector, 4079 stream_slicer=async_job_partition_router, 4080 config=config, 4081 parameters=model.parameters or {}, 4082 ) 4083 4084 def create_spec(self, model: SpecModel, config: Config, **kwargs: Any) -> Spec: 4085 config_migrations = [ 4086 self._create_component_from_model(migration, config) 4087 for migration in ( 4088 model.config_normalization_rules.config_migrations 4089 if ( 4090 model.config_normalization_rules 4091 and model.config_normalization_rules.config_migrations 4092 ) 4093 else [] 4094 ) 4095 ] 4096 config_transformations = [ 4097 self._create_component_from_model(transformation, config) 4098 for transformation in ( 4099 model.config_normalization_rules.transformations 4100 if ( 4101 model.config_normalization_rules 4102 and model.config_normalization_rules.transformations 4103 ) 4104 else [] 4105 ) 4106 ] 4107 config_validations = [ 4108 self._create_component_from_model(validation, config) 4109 for validation in ( 4110 model.config_normalization_rules.validations 4111 if ( 4112 model.config_normalization_rules 4113 and model.config_normalization_rules.validations 4114 ) 4115 else [] 4116 ) 4117 ] 4118 4119 return Spec( 4120 connection_specification=model.connection_specification, 4121 documentation_url=model.documentation_url, 4122 advanced_auth=model.advanced_auth, 4123 parameters={}, 4124 config_migrations=config_migrations, 4125 config_transformations=config_transformations, 4126 config_validations=config_validations, 4127 ) 4128 4129 def create_substream_partition_router( 4130 self, 4131 model: SubstreamPartitionRouterModel, 4132 config: Config, 4133 *, 4134 stream_name: str, 4135 **kwargs: Any, 4136 ) -> SubstreamPartitionRouter: 4137 parent_stream_configs = [] 4138 if model.parent_stream_configs: 4139 parent_stream_configs.extend( 4140 [ 4141 self.create_parent_stream_config_with_substream_wrapper( 4142 model=parent_stream_config, config=config, stream_name=stream_name, **kwargs 4143 ) 4144 for parent_stream_config in model.parent_stream_configs 4145 ] 4146 ) 4147 4148 return SubstreamPartitionRouter( 4149 parent_stream_configs=parent_stream_configs, 4150 parameters=model.parameters or {}, 4151 config=config, 4152 ) 4153 4154 def create_parent_stream_config_with_substream_wrapper( 4155 self, model: ParentStreamConfigModel, config: Config, *, stream_name: str, **kwargs: Any 4156 ) -> Any: 4157 child_state = self._connector_state_manager.get_stream_state(stream_name, None) 4158 4159 parent_state: Optional[Mapping[str, Any]] = ( 4160 child_state if model.incremental_dependency and child_state else None 4161 ) 4162 connector_state_manager = self._instantiate_parent_stream_state_manager( 4163 child_state, config, model, parent_state 4164 ) 4165 4166 substream_factory = ModelToComponentFactory( 4167 custom_components_trusted=self._custom_components_trusted, 4168 connector_state_manager=connector_state_manager, 4169 limit_pages_fetched_per_slice=self._limit_pages_fetched_per_slice, 4170 limit_slices_fetched=self._limit_slices_fetched, 4171 emit_connector_builder_messages=self._emit_connector_builder_messages, 4172 disable_retries=self._disable_retries, 4173 disable_cache=self._disable_cache, 4174 message_repository=StateFilteringMessageRepository( 4175 LogAppenderMessageRepositoryDecorator( 4176 { 4177 "airbyte_cdk": {"stream": {"is_substream": True}}, 4178 "http": {"is_auxiliary": True}, 4179 }, 4180 self._message_repository, 4181 self._evaluate_log_level(self._emit_connector_builder_messages), 4182 ), 4183 ), 4184 api_budget=self._api_budget, 4185 # Share the authenticator registry so parent and child streams draw from the 4186 # same token quota counters 4187 rate_limited_authenticators=self._rate_limited_authenticators, 4188 ) 4189 4190 return substream_factory.create_parent_stream_config( 4191 model=model, config=config, stream_name=stream_name, **kwargs 4192 ) 4193 4194 def _instantiate_parent_stream_state_manager( 4195 self, 4196 child_state: MutableMapping[str, Any], 4197 config: Config, 4198 model: ParentStreamConfigModel, 4199 parent_state: Optional[Mapping[str, Any]] = None, 4200 ) -> ConnectorStateManager: 4201 """ 4202 With DefaultStream, the state needs to be provided during __init__ of the cursor as opposed to the 4203 `set_initial_state` flow that existed for the declarative cursors. This state is taken from 4204 self._connector_state_manager.get_stream_state (`self` being a newly created ModelToComponentFactory to account 4205 for the MessageRepository being different). So we need to pass a ConnectorStateManager to the 4206 ModelToComponentFactory that has the parent states. This method populates this if there is a child state and if 4207 incremental_dependency is set. 4208 """ 4209 if model.incremental_dependency and child_state: 4210 parent_stream_name = model.stream.name or "" 4211 extracted_parent_state = ConcurrentPerPartitionCursor.get_parent_state( 4212 child_state, parent_stream_name 4213 ) 4214 4215 if not extracted_parent_state: 4216 extracted_parent_state = ConcurrentPerPartitionCursor.get_global_state( 4217 child_state, parent_stream_name 4218 ) 4219 4220 if not extracted_parent_state and not isinstance(extracted_parent_state, dict): 4221 cursor_values = child_state.values() 4222 if cursor_values and len(cursor_values) == 1: 4223 incremental_sync_model: Union[ 4224 DatetimeBasedCursorModel, 4225 IncrementingCountCursorModel, 4226 ] = ( 4227 model.stream.incremental_sync # type: ignore # if we are there, it is because there is incremental_dependency and therefore there is an incremental_sync on the parent stream 4228 if isinstance(model.stream, DeclarativeStreamModel) 4229 else self._get_state_delegating_stream_model( 4230 model.stream, parent_state=parent_state 4231 ).incremental_sync 4232 ) 4233 cursor_field = InterpolatedString.create( 4234 incremental_sync_model.cursor_field, 4235 parameters=incremental_sync_model.parameters or {}, 4236 ).eval(config) 4237 extracted_parent_state = AirbyteStateMessage( 4238 type=AirbyteStateType.STREAM, 4239 stream=AirbyteStreamState( 4240 stream_descriptor=StreamDescriptor( 4241 name=parent_stream_name, namespace=None 4242 ), 4243 stream_state=AirbyteStateBlob( 4244 {cursor_field: list(cursor_values)[0]} 4245 ), 4246 ), 4247 ) 4248 return ConnectorStateManager([extracted_parent_state] if extracted_parent_state else []) 4249 4250 return ConnectorStateManager([]) 4251 4252 @staticmethod 4253 def create_wait_time_from_header( 4254 model: WaitTimeFromHeaderModel, config: Config, **kwargs: Any 4255 ) -> WaitTimeFromHeaderBackoffStrategy: 4256 return WaitTimeFromHeaderBackoffStrategy( 4257 header=model.header, 4258 parameters=model.parameters or {}, 4259 config=config, 4260 regex=model.regex, 4261 max_waiting_time_in_seconds=model.max_waiting_time_in_seconds 4262 if model.max_waiting_time_in_seconds is not None 4263 else None, 4264 ) 4265 4266 @staticmethod 4267 def create_wait_until_time_from_header( 4268 model: WaitUntilTimeFromHeaderModel, config: Config, **kwargs: Any 4269 ) -> WaitUntilTimeFromHeaderBackoffStrategy: 4270 return WaitUntilTimeFromHeaderBackoffStrategy( 4271 header=model.header, 4272 parameters=model.parameters or {}, 4273 config=config, 4274 min_wait=model.min_wait, 4275 regex=model.regex, 4276 ) 4277 4278 def get_message_repository(self) -> MessageRepository: 4279 return self._message_repository 4280 4281 def _evaluate_log_level(self, emit_connector_builder_messages: bool) -> Level: 4282 return Level.DEBUG if emit_connector_builder_messages else Level.INFO 4283 4284 @staticmethod 4285 def create_components_mapping_definition( 4286 model: ComponentMappingDefinitionModel, config: Config, **kwargs: Any 4287 ) -> ComponentMappingDefinition: 4288 interpolated_value = InterpolatedString.create( 4289 model.value, parameters=model.parameters or {} 4290 ) 4291 field_path = [ 4292 InterpolatedString.create(path, parameters=model.parameters or {}) 4293 for path in model.field_path 4294 ] 4295 return ComponentMappingDefinition( 4296 field_path=field_path, # type: ignore[arg-type] # field_path can be str and InterpolatedString 4297 value=interpolated_value, 4298 value_type=ModelToComponentFactory._json_schema_type_name_to_type(model.value_type), 4299 create_or_update=model.create_or_update, 4300 condition=model.condition, 4301 parameters=model.parameters or {}, 4302 ) 4303 4304 def create_http_components_resolver( 4305 self, model: HttpComponentsResolverModel, config: Config, stream_name: Optional[str] = None 4306 ) -> Any: 4307 retriever = self._create_component_from_model( 4308 model=model.retriever, 4309 config=config, 4310 name=f"{stream_name if stream_name else '__http_components_resolver'}", 4311 primary_key=None, 4312 stream_slicer=self._build_stream_slicer_from_partition_router(model.retriever, config), 4313 transformations=[], 4314 ) 4315 4316 components_mapping = [] 4317 for component_mapping_definition_model in model.components_mapping: 4318 if component_mapping_definition_model.condition: 4319 raise ValueError("`condition` is only supported for `ConfigComponentsResolver`") 4320 components_mapping.append( 4321 self._create_component_from_model( 4322 model=component_mapping_definition_model, 4323 value_type=ModelToComponentFactory._json_schema_type_name_to_type( 4324 component_mapping_definition_model.value_type 4325 ), 4326 config=config, 4327 ) 4328 ) 4329 4330 return HttpComponentsResolver( 4331 retriever=retriever, 4332 stream_slicer=self._build_stream_slicer_from_partition_router(model.retriever, config), 4333 config=config, 4334 components_mapping=components_mapping, 4335 parameters=model.parameters or {}, 4336 ) 4337 4338 @staticmethod 4339 def create_stream_config( 4340 model: StreamConfigModel, config: Config, **kwargs: Any 4341 ) -> StreamConfig: 4342 model_configs_pointer: List[Union[InterpolatedString, str]] = ( 4343 [x for x in model.configs_pointer] if model.configs_pointer else [] 4344 ) 4345 4346 return StreamConfig( 4347 configs_pointer=model_configs_pointer, 4348 default_values=model.default_values, 4349 parameters=model.parameters or {}, 4350 ) 4351 4352 def create_config_components_resolver( 4353 self, 4354 model: ConfigComponentsResolverModel, 4355 config: Config, 4356 ) -> Any: 4357 model_stream_configs = ( 4358 model.stream_config if isinstance(model.stream_config, list) else [model.stream_config] 4359 ) 4360 4361 stream_configs = [ 4362 self._create_component_from_model( 4363 stream_config, config=config, parameters=model.parameters or {} 4364 ) 4365 for stream_config in model_stream_configs 4366 ] 4367 4368 components_mapping = [ 4369 self._create_component_from_model( 4370 model=components_mapping_definition_model, 4371 value_type=ModelToComponentFactory._json_schema_type_name_to_type( 4372 components_mapping_definition_model.value_type 4373 ), 4374 config=config, 4375 parameters=model.parameters, 4376 ) 4377 for components_mapping_definition_model in model.components_mapping 4378 ] 4379 4380 return ConfigComponentsResolver( 4381 stream_configs=stream_configs, 4382 config=config, 4383 components_mapping=components_mapping, 4384 parameters=model.parameters or {}, 4385 ) 4386 4387 def create_parametrized_components_resolver( 4388 self, 4389 model: ParametrizedComponentsResolverModel, 4390 config: Config, 4391 ) -> ParametrizedComponentsResolver: 4392 stream_parameters = StreamParametersDefinition( 4393 list_of_parameters_for_stream=model.stream_parameters.list_of_parameters_for_stream 4394 ) 4395 4396 components_mapping = [] 4397 for components_mapping_definition_model in model.components_mapping: 4398 if components_mapping_definition_model.condition: 4399 raise ValueError("`condition` is only supported for `ConfigComponentsResolver`") 4400 components_mapping.append( 4401 self._create_component_from_model( 4402 model=components_mapping_definition_model, 4403 value_type=ModelToComponentFactory._json_schema_type_name_to_type( 4404 components_mapping_definition_model.value_type 4405 ), 4406 config=config, 4407 ) 4408 ) 4409 return ParametrizedComponentsResolver( 4410 stream_parameters=stream_parameters, 4411 config=config, 4412 components_mapping=components_mapping, 4413 parameters=model.parameters or {}, 4414 ) 4415 4416 _UNSUPPORTED_DECODER_ERROR = ( 4417 "Specified decoder of {decoder_type} is not supported for pagination." 4418 "Please set as `JsonDecoder`, `XmlDecoder`, or a `CompositeRawDecoder` with an inner_parser of `JsonParser` or `GzipParser` instead." 4419 "If using `GzipParser`, please ensure that the lowest level inner_parser is a `JsonParser`." 4420 ) 4421 4422 def _is_supported_decoder_for_pagination(self, decoder: Decoder) -> bool: 4423 if isinstance(decoder, (JsonDecoder, XmlDecoder)): 4424 return True 4425 elif isinstance(decoder, CompositeRawDecoder): 4426 return self._is_supported_parser_for_pagination(decoder.parser) 4427 else: 4428 return False 4429 4430 def _is_supported_parser_for_pagination(self, parser: Parser) -> bool: 4431 if isinstance(parser, JsonParser): 4432 return True 4433 elif isinstance(parser, GzipParser): 4434 return isinstance(parser.inner_parser, JsonParser) 4435 else: 4436 return False 4437 4438 def create_http_api_budget( 4439 self, model: HTTPAPIBudgetModel, config: Config, **kwargs: Any 4440 ) -> HttpAPIBudget: 4441 policies = [ 4442 self._create_component_from_model(model=policy, config=config) 4443 for policy in model.policies 4444 ] 4445 4446 return HttpAPIBudget( 4447 policies=policies, 4448 ratelimit_reset_header=model.ratelimit_reset_header or "ratelimit-reset", 4449 ratelimit_remaining_header=model.ratelimit_remaining_header or "ratelimit-remaining", 4450 status_codes_for_ratelimit_hit=model.status_codes_for_ratelimit_hit or [429], 4451 ) 4452 4453 def create_fixed_window_call_rate_policy( 4454 self, model: FixedWindowCallRatePolicyModel, config: Config, **kwargs: Any 4455 ) -> FixedWindowCallRatePolicy: 4456 matchers = [ 4457 self._create_component_from_model(model=matcher, config=config) 4458 for matcher in model.matchers 4459 ] 4460 4461 # Set the initial reset timestamp to 10 days from now. 4462 # This value will be updated by the first request. 4463 return FixedWindowCallRatePolicy( 4464 next_reset_ts=datetime.datetime.now() + datetime.timedelta(days=10), 4465 period=parse_duration(model.period), 4466 call_limit=model.call_limit, 4467 matchers=matchers, 4468 ) 4469 4470 def create_file_uploader( 4471 self, model: FileUploaderModel, config: Config, **kwargs: Any 4472 ) -> FileUploader: 4473 name = "File Uploader" 4474 requester = self._create_component_from_model( 4475 model=model.requester, 4476 config=config, 4477 name=name, 4478 **kwargs, 4479 ) 4480 download_target_extractor = self._create_component_from_model( 4481 model=model.download_target_extractor, 4482 config=config, 4483 name=name, 4484 **kwargs, 4485 ) 4486 emit_connector_builder_messages = self._emit_connector_builder_messages 4487 file_uploader = DefaultFileUploader( 4488 requester=requester, 4489 download_target_extractor=download_target_extractor, 4490 config=config, 4491 file_writer=NoopFileWriter() 4492 if emit_connector_builder_messages 4493 else LocalFileSystemFileWriter(), 4494 parameters=model.parameters or {}, 4495 filename_extractor=model.filename_extractor if model.filename_extractor else None, 4496 ) 4497 4498 return ( 4499 ConnectorBuilderFileUploader(file_uploader) 4500 if emit_connector_builder_messages 4501 else file_uploader 4502 ) 4503 4504 def create_moving_window_call_rate_policy( 4505 self, model: MovingWindowCallRatePolicyModel, config: Config, **kwargs: Any 4506 ) -> MovingWindowCallRatePolicy: 4507 rates = [ 4508 self._create_component_from_model(model=rate, config=config) for rate in model.rates 4509 ] 4510 matchers = [ 4511 self._create_component_from_model(model=matcher, config=config) 4512 for matcher in model.matchers 4513 ] 4514 return MovingWindowCallRatePolicy( 4515 rates=rates, 4516 matchers=matchers, 4517 ) 4518 4519 def create_unlimited_call_rate_policy( 4520 self, model: UnlimitedCallRatePolicyModel, config: Config, **kwargs: Any 4521 ) -> UnlimitedCallRatePolicy: 4522 matchers = [ 4523 self._create_component_from_model(model=matcher, config=config) 4524 for matcher in model.matchers 4525 ] 4526 4527 return UnlimitedCallRatePolicy( 4528 matchers=matchers, 4529 ) 4530 4531 def create_rate(self, model: RateModel, config: Config, **kwargs: Any) -> Rate: 4532 interpolated_limit = InterpolatedString.create(str(model.limit), parameters={}) 4533 return Rate( 4534 limit=int(interpolated_limit.eval(config=config)), 4535 interval=parse_duration(model.interval), 4536 ) 4537 4538 def create_http_request_matcher( 4539 self, model: HttpRequestRegexMatcherModel, config: Config, **kwargs: Any 4540 ) -> HttpRequestRegexMatcher: 4541 weight = model.weight 4542 if weight is not None: 4543 if isinstance(weight, str): 4544 weight = int(InterpolatedString.create(weight, parameters={}).eval(config)) 4545 else: 4546 weight = int(weight) 4547 if weight < 1: 4548 raise ValueError(f"weight must be >= 1, got {weight}") 4549 return HttpRequestRegexMatcher( 4550 method=model.method, 4551 url_base=model.url_base, 4552 url_path_pattern=model.url_path_pattern, 4553 params=model.params, 4554 headers=model.headers, 4555 weight=weight, 4556 ) 4557 4558 def create_rate_limited_multiple_token_authenticator( 4559 self, 4560 model: RateLimitedMultipleTokenAuthenticatorModel, 4561 config: Config, 4562 **kwargs: Any, 4563 ) -> RateLimitedMultipleTokenAuthenticator: 4564 if isinstance(model.tokens, str): 4565 tokens_value = InterpolatedString.create(model.tokens, parameters={}).eval(config) 4566 delimiter = model.token_delimiter or "," 4567 tokens = [ 4568 token.strip() for token in str(tokens_value).split(delimiter) if token.strip() 4569 ] 4570 else: 4571 tokens = [ 4572 token_value 4573 for token in model.tokens 4574 if ( 4575 token_value := str( 4576 InterpolatedString.create(token, parameters={}).eval(config) 4577 ).strip() 4578 ) 4579 ] 4580 4581 quota_specs = [ 4582 { 4583 "name": quota_model.name, 4584 "remaining_path": quota_model.remaining_path, 4585 "reset_path": quota_model.reset_path, 4586 "limit_path": quota_model.limit_path, 4587 "matchers": [ 4588 { 4589 "method": matcher_model.method, 4590 "url_base": matcher_model.url_base, 4591 "url_path_pattern": matcher_model.url_path_pattern, 4592 "params": matcher_model.params, 4593 "headers": matcher_model.headers, 4594 "weight": matcher_model.weight, 4595 } 4596 for matcher_model in quota_model.matchers or [] 4597 ], 4598 } 4599 for quota_model in model.quotas 4600 ] 4601 4602 quota_status_url = str( 4603 InterpolatedString.create(model.quota_status_source.url, parameters={}).eval(config) 4604 ) 4605 quota_status_http_method = ( 4606 model.quota_status_source.http_method.value 4607 if model.quota_status_source.http_method 4608 else "GET" 4609 ) 4610 quota_status_headers = { 4611 key: str(InterpolatedString.create(value, parameters={}).eval(config)) 4612 for key, value in (model.quota_status_source.request_headers or {}).items() 4613 } 4614 auth_method = model.auth_method or "Bearer" 4615 header = model.header or "Authorization" 4616 max_wait_time_str = str( 4617 InterpolatedString.create(model.max_wait_time or "PT2H", parameters={}).eval(config) 4618 ) 4619 max_wait_time = parse_duration(max_wait_time_str) 4620 if not isinstance(max_wait_time, datetime.timedelta): 4621 raise ValueError( 4622 f"max_wait_time must be a fixed-length ISO 8601 duration (e.g. 'PT2H'); " 4623 f"calendar-unit durations like '{max_wait_time_str}' are not supported" 4624 ) 4625 budget_reserve_fraction = ( 4626 model.budget_reserve_fraction if model.budget_reserve_fraction is not None else 0.1 4627 ) 4628 budget_min_reserve = ( 4629 model.budget_min_reserve if model.budget_min_reserve is not None else 50 4630 ) 4631 4632 # Reuse the same instance for identical definitions so that all streams share the 4633 # same token quota counters (similar to how api_budget is shared). The key is built 4634 # from the resolved constructor arguments rather than the raw model so that 4635 # stream-specific `$parameters` propagated onto the model (and its nested components) 4636 # cannot break instance sharing. 4637 cache_key = json.dumps( 4638 { 4639 "tokens": tokens, 4640 "quotas": quota_specs, 4641 "quota_status_url": quota_status_url, 4642 "quota_status_http_method": quota_status_http_method, 4643 "quota_status_headers": quota_status_headers, 4644 "auth_method": auth_method, 4645 "header": header, 4646 "max_wait_time": max_wait_time.total_seconds(), 4647 "budget_reserve_fraction": budget_reserve_fraction, 4648 "budget_min_reserve": budget_min_reserve, 4649 }, 4650 sort_keys=True, 4651 ) 4652 if cache_key in self._rate_limited_authenticators: 4653 return self._rate_limited_authenticators[cache_key] 4654 4655 quotas = [ 4656 TokenQuota( 4657 name=quota_model.name, 4658 remaining_path=quota_model.remaining_path, 4659 reset_path=quota_model.reset_path, 4660 limit_path=quota_model.limit_path, 4661 matchers=[ 4662 self.create_http_request_matcher(matcher_model, config) 4663 for matcher_model in quota_model.matchers or [] 4664 ], 4665 ) 4666 for quota_model in model.quotas 4667 ] 4668 4669 authenticator = RateLimitedMultipleTokenAuthenticator( 4670 tokens=tokens, 4671 quotas=quotas, 4672 quota_status_url=quota_status_url, 4673 quota_status_http_method=quota_status_http_method, 4674 quota_status_headers=quota_status_headers, 4675 auth_method=auth_method, 4676 header=header, 4677 max_wait_time=max_wait_time, 4678 budget_reserve_fraction=budget_reserve_fraction, 4679 budget_min_reserve=budget_min_reserve, 4680 ) 4681 self._rate_limited_authenticators[cache_key] = authenticator 4682 return authenticator 4683 4684 def set_api_budget(self, component_definition: ComponentDefinition, config: Config) -> None: 4685 self._api_budget = self.create_component( 4686 model_type=HTTPAPIBudgetModel, component_definition=component_definition, config=config 4687 ) 4688 4689 def create_grouping_partition_router( 4690 self, 4691 model: GroupingPartitionRouterModel, 4692 config: Config, 4693 *, 4694 stream_name: str, 4695 **kwargs: Any, 4696 ) -> GroupingPartitionRouter: 4697 underlying_router = self._create_component_from_model( 4698 model=model.underlying_partition_router, 4699 config=config, 4700 stream_name=stream_name, 4701 **kwargs, 4702 ) 4703 if model.group_size < 1: 4704 raise ValueError(f"Group size must be greater than 0, got {model.group_size}") 4705 4706 # Request options in underlying partition routers are not supported for GroupingPartitionRouter 4707 # because they are specific to individual partitions and cannot be aggregated or handled 4708 # when grouping, potentially leading to incorrect API calls. Any request customization 4709 # should be managed at the stream level through the requester's configuration. 4710 if isinstance(underlying_router, SubstreamPartitionRouter): 4711 if any( 4712 parent_config.request_option 4713 for parent_config in underlying_router.parent_stream_configs 4714 ): 4715 raise ValueError("Request options are not supported for GroupingPartitionRouter.") 4716 4717 if isinstance(underlying_router, ListPartitionRouter): 4718 if underlying_router.request_option: 4719 raise ValueError("Request options are not supported for GroupingPartitionRouter.") 4720 4721 return GroupingPartitionRouter( 4722 group_size=model.group_size, 4723 underlying_partition_router=underlying_router, 4724 deduplicate=model.deduplicate if model.deduplicate is not None else True, 4725 config=config, 4726 ) 4727 4728 def _ensure_query_properties_to_model( 4729 self, requester: Union[HttpRequesterModel, CustomRequesterModel] 4730 ) -> None: 4731 """ 4732 For some reason, it seems like CustomRequesterModel request_parameters stays as dictionaries which means that 4733 the other conditions relying on it being QueryPropertiesModel instead of a dict fail. Here, we migrate them to 4734 proper model. 4735 """ 4736 if not hasattr(requester, "request_parameters"): 4737 return 4738 4739 request_parameters = requester.request_parameters 4740 if request_parameters and isinstance(request_parameters, Dict): 4741 for request_parameter_key in request_parameters.keys(): 4742 request_parameter = request_parameters[request_parameter_key] 4743 if ( 4744 isinstance(request_parameter, Dict) 4745 and request_parameter.get("type") == "QueryProperties" 4746 ): 4747 request_parameters[request_parameter_key] = QueryPropertiesModel.parse_obj( 4748 request_parameter 4749 ) 4750 4751 def _get_catalog_defined_cursor_field( 4752 self, stream_name: str, allow_catalog_defined_cursor_field: bool 4753 ) -> Optional[CursorField]: 4754 if not allow_catalog_defined_cursor_field: 4755 return None 4756 4757 configured_stream = self._stream_name_to_configured_stream.get(stream_name) 4758 4759 # Depending on the operation is being performed, there may not be a configured stream yet. In this 4760 # case we return None which will then use the default cursor field defined on the cursor model. 4761 # We also treat cursor_field: [""] (list with empty string) as no cursor field, since this can 4762 # occur when the platform serializes "no cursor configured" streams incorrectly. 4763 if ( 4764 not configured_stream 4765 or not configured_stream.cursor_field 4766 or not configured_stream.cursor_field[0] 4767 ): 4768 return None 4769 elif len(configured_stream.cursor_field) > 1: 4770 raise ValueError( 4771 f"The `{stream_name}` stream does not support nested cursor_field. Please specify only a single cursor_field for the stream in the configured catalog." 4772 ) 4773 else: 4774 return CursorField( 4775 cursor_field_key=configured_stream.cursor_field[0], 4776 supports_catalog_defined_cursor_field=allow_catalog_defined_cursor_field, 4777 )
ModelToComponentFactory( limit_pages_fetched_per_slice: Optional[int] = None, limit_slices_fetched: Optional[int] = None, emit_connector_builder_messages: bool = False, disable_retries: bool = False, disable_cache: bool = False, message_repository: Optional[airbyte_cdk.MessageRepository] = None, connector_state_manager: Optional[airbyte_cdk.ConnectorStateManager] = None, max_concurrent_async_job_count: Optional[int] = None, configured_catalog: Optional[airbyte_protocol_dataclasses.models.airbyte_protocol.ConfiguredAirbyteCatalog] = None, api_budget: Optional[airbyte_cdk.sources.streams.call_rate.APIBudget] = None, rate_limited_authenticators: Optional[Dict[str, airbyte_cdk.sources.declarative.auth.RateLimitedMultipleTokenAuthenticator]] = None, custom_components_trusted: bool = True)
705 def __init__( 706 self, 707 limit_pages_fetched_per_slice: Optional[int] = None, 708 limit_slices_fetched: Optional[int] = None, 709 emit_connector_builder_messages: bool = False, 710 disable_retries: bool = False, 711 disable_cache: bool = False, 712 message_repository: Optional[MessageRepository] = None, 713 connector_state_manager: Optional[ConnectorStateManager] = None, 714 max_concurrent_async_job_count: Optional[int] = None, 715 configured_catalog: Optional[ConfiguredAirbyteCatalog] = None, 716 api_budget: Optional[APIBudget] = None, 717 rate_limited_authenticators: Optional[ 718 Dict[str, RateLimitedMultipleTokenAuthenticator] 719 ] = None, 720 custom_components_trusted: bool = True, 721 ): 722 self._init_mappings() 723 self._custom_components_trusted = custom_components_trusted 724 self._limit_pages_fetched_per_slice = limit_pages_fetched_per_slice 725 self._limit_slices_fetched = limit_slices_fetched 726 self._emit_connector_builder_messages = emit_connector_builder_messages 727 self._disable_retries = disable_retries 728 self._disable_cache = disable_cache 729 self._message_repository = message_repository or InMemoryMessageRepository( 730 self._evaluate_log_level(emit_connector_builder_messages) 731 ) 732 self._stream_name_to_configured_stream = self._create_stream_name_to_configured_stream( 733 configured_catalog 734 ) 735 self._connector_state_manager = connector_state_manager or ConnectorStateManager() 736 self._api_budget: Optional[Union[APIBudget]] = api_budget 737 # Shared instances so all streams see the same token quota counters (like api_budget) 738 self._rate_limited_authenticators: Dict[str, RateLimitedMultipleTokenAuthenticator] = ( 739 rate_limited_authenticators if rate_limited_authenticators is not None else {} 740 ) 741 self._job_tracker: JobTracker = JobTracker(max_concurrent_async_job_count or 1) 742 # placeholder for deprecation warnings 743 self._collected_deprecation_logs: List[ConnectorBuilderLogMessage] = []
def
create_component( self, model_type: Type[pydantic.v1.main.BaseModel], component_definition: Mapping[str, Any], config: Mapping[str, Any], **kwargs: Any) -> Any:
869 def create_component( 870 self, 871 model_type: Type[BaseModel], 872 component_definition: ComponentDefinition, 873 config: Config, 874 **kwargs: Any, 875 ) -> Any: 876 """ 877 Takes a given Pydantic model type and Mapping representing a component definition and creates a declarative component and 878 subcomponents which will be used at runtime. This is done by first parsing the mapping into a Pydantic model and then creating 879 creating declarative components from that model. 880 881 :param model_type: The type of declarative component that is being initialized 882 :param component_definition: The mapping that represents a declarative component 883 :param config: The connector config that is provided by the customer 884 :return: The declarative component to be used at runtime 885 """ 886 887 component_type = component_definition.get("type") 888 if component_definition.get("type") != model_type.__name__: 889 raise ValueError( 890 f"Expected manifest component of type {model_type.__name__}, but received {component_type} instead" 891 ) 892 893 declarative_component_model = model_type.parse_obj(component_definition) 894 895 if not isinstance(declarative_component_model, model_type): 896 raise ValueError( 897 f"Expected {model_type.__name__} component, but received {declarative_component_model.__class__.__name__}" 898 ) 899 900 return self._create_component_from_model( 901 model=declarative_component_model, config=config, **kwargs 902 )
Takes a given Pydantic model type and Mapping representing a component definition and creates a declarative component and subcomponents which will be used at runtime. This is done by first parsing the mapping into a Pydantic model and then creating creating declarative components from that model.
Parameters
- model_type: The type of declarative component that is being initialized
- component_definition: The mapping that represents a declarative component
- config: The connector config that is provided by the customer
Returns
The declarative component to be used at runtime
919 def get_model_deprecations(self) -> List[ConnectorBuilderLogMessage]: 920 """ 921 Returns the deprecation warnings that were collected during the creation of components. 922 """ 923 return self._collected_deprecation_logs
Returns the deprecation warnings that were collected during the creation of components.
def
create_config_migration( self, model: airbyte_cdk.sources.declarative.models.declarative_component_schema.ConfigMigration, config: Mapping[str, Any]) -> airbyte_cdk.sources.declarative.spec.ConfigMigration:
940 def create_config_migration( 941 self, model: ConfigMigrationModel, config: Config 942 ) -> ConfigMigration: 943 transformations: List[ConfigTransformation] = [ 944 self._create_component_from_model(transformation, config) 945 for transformation in model.transformations 946 ] 947 948 return ConfigMigration( 949 description=model.description, 950 transformations=transformations, 951 )
def
create_config_add_fields( self, model: airbyte_cdk.sources.declarative.models.declarative_component_schema.ConfigAddFields, config: Mapping[str, Any], **kwargs: Any) -> airbyte_cdk.sources.declarative.transformations.config_transformations.ConfigAddFields:
953 def create_config_add_fields( 954 self, model: ConfigAddFieldsModel, config: Config, **kwargs: Any 955 ) -> ConfigAddFields: 956 fields = [self._create_component_from_model(field, config) for field in model.fields] 957 return ConfigAddFields( 958 fields=fields, 959 condition=model.condition or "", 960 )
@staticmethod
def
create_config_remove_fields( model: airbyte_cdk.sources.declarative.models.declarative_component_schema.ConfigRemoveFields, config: Mapping[str, Any], **kwargs: Any) -> airbyte_cdk.sources.declarative.transformations.config_transformations.ConfigRemoveFields:
@staticmethod
def
create_config_remap_field( model: airbyte_cdk.sources.declarative.models.declarative_component_schema.ConfigRemapField, config: Mapping[str, Any], **kwargs: Any) -> airbyte_cdk.sources.declarative.transformations.config_transformations.ConfigRemapField:
def
create_dpath_validator( self, model: airbyte_cdk.sources.declarative.models.declarative_component_schema.DpathValidator, config: Mapping[str, Any]) -> airbyte_cdk.sources.declarative.validators.DpathValidator:
def
create_predicate_validator( self, model: airbyte_cdk.sources.declarative.models.declarative_component_schema.PredicateValidator, config: Mapping[str, Any]) -> airbyte_cdk.sources.declarative.validators.PredicateValidator:
@staticmethod
def
create_validate_adheres_to_schema( model: airbyte_cdk.sources.declarative.models.declarative_component_schema.ValidateAdheresToSchema, config: Mapping[str, Any], **kwargs: Any) -> airbyte_cdk.sources.declarative.validators.ValidateAdheresToSchema:
1000 @staticmethod 1001 def create_validate_adheres_to_schema( 1002 model: ValidateAdheresToSchemaModel, config: Config, **kwargs: Any 1003 ) -> ValidateAdheresToSchema: 1004 base_schema = cast(Mapping[str, Any], model.base_schema) 1005 return ValidateAdheresToSchema( 1006 schema=base_schema, 1007 )
@staticmethod
def
create_added_field_definition( model: airbyte_cdk.sources.declarative.models.declarative_component_schema.AddedFieldDefinition, config: Mapping[str, Any], **kwargs: Any) -> airbyte_cdk.AddedFieldDefinition:
1009 @staticmethod 1010 def create_added_field_definition( 1011 model: AddedFieldDefinitionModel, config: Config, **kwargs: Any 1012 ) -> AddedFieldDefinition: 1013 interpolated_value = InterpolatedString.create( 1014 model.value, parameters=model.parameters or {} 1015 ) 1016 return AddedFieldDefinition( 1017 path=model.path, 1018 value=interpolated_value, 1019 value_type=ModelToComponentFactory._json_schema_type_name_to_type(model.value_type), 1020 parameters=model.parameters or {}, 1021 )
def
create_add_fields( self, model: airbyte_cdk.sources.declarative.models.declarative_component_schema.AddFields, config: Mapping[str, Any], **kwargs: Any) -> airbyte_cdk.AddFields:
1023 def create_add_fields(self, model: AddFieldsModel, config: Config, **kwargs: Any) -> AddFields: 1024 added_field_definitions = [ 1025 self._create_component_from_model( 1026 model=added_field_definition_model, 1027 value_type=ModelToComponentFactory._json_schema_type_name_to_type( 1028 added_field_definition_model.value_type 1029 ), 1030 config=config, 1031 ) 1032 for added_field_definition_model in model.fields 1033 ] 1034 return AddFields( 1035 fields=added_field_definitions, 1036 condition=model.condition or "", 1037 parameters=model.parameters or {}, 1038 )
def
create_keys_to_lower_transformation( self, model: airbyte_cdk.sources.declarative.models.declarative_component_schema.KeysToLower, config: Mapping[str, Any], **kwargs: Any) -> airbyte_cdk.sources.declarative.transformations.keys_to_lower_transformation.KeysToLowerTransformation:
def
create_keys_to_snake_transformation( self, model: airbyte_cdk.sources.declarative.models.declarative_component_schema.KeysToSnakeCase, config: Mapping[str, Any], **kwargs: Any) -> airbyte_cdk.sources.declarative.transformations.keys_to_snake_transformation.KeysToSnakeCaseTransformation:
def
create_keys_replace_transformation( self, model: airbyte_cdk.sources.declarative.models.declarative_component_schema.KeysReplace, config: Mapping[str, Any], **kwargs: Any) -> airbyte_cdk.sources.declarative.transformations.keys_replace_transformation.KeysReplaceTransformation:
def
create_flatten_fields( self, model: airbyte_cdk.sources.declarative.models.declarative_component_schema.FlattenFields, config: Mapping[str, Any], **kwargs: Any) -> airbyte_cdk.sources.declarative.transformations.flatten_fields.FlattenFields:
def
create_dpath_flatten_fields( self, model: airbyte_cdk.sources.declarative.models.declarative_component_schema.DpathFlattenFields, config: Mapping[str, Any], **kwargs: Any) -> airbyte_cdk.sources.declarative.transformations.dpath_flatten_fields.DpathFlattenFields:
1064 def create_dpath_flatten_fields( 1065 self, model: DpathFlattenFieldsModel, config: Config, **kwargs: Any 1066 ) -> DpathFlattenFields: 1067 model_field_path: List[Union[InterpolatedString, str]] = [x for x in model.field_path] 1068 key_transformation = ( 1069 KeyTransformation( 1070 config=config, 1071 prefix=model.key_transformation.prefix, 1072 suffix=model.key_transformation.suffix, 1073 parameters=model.parameters or {}, 1074 ) 1075 if model.key_transformation is not None 1076 else None 1077 ) 1078 return DpathFlattenFields( 1079 config=config, 1080 field_path=model_field_path, 1081 delete_origin_value=model.delete_origin_value 1082 if model.delete_origin_value is not None 1083 else False, 1084 replace_record=model.replace_record if model.replace_record is not None else False, 1085 key_transformation=key_transformation, 1086 parameters=model.parameters or {}, 1087 )
def
create_api_key_authenticator( self, model: airbyte_cdk.sources.declarative.models.declarative_component_schema.ApiKeyAuthenticator, config: Mapping[str, Any], token_provider: Optional[airbyte_cdk.sources.declarative.auth.token_provider.TokenProvider] = None, **kwargs: Any) -> airbyte_cdk.ApiKeyAuthenticator:
1101 def create_api_key_authenticator( 1102 self, 1103 model: ApiKeyAuthenticatorModel, 1104 config: Config, 1105 token_provider: Optional[TokenProvider] = None, 1106 **kwargs: Any, 1107 ) -> ApiKeyAuthenticator: 1108 if model.inject_into is None and model.header is None: 1109 raise ValueError( 1110 "Expected either inject_into or header to be set for ApiKeyAuthenticator" 1111 ) 1112 1113 if model.inject_into is not None and model.header is not None: 1114 raise ValueError( 1115 "inject_into and header cannot be set both for ApiKeyAuthenticator - remove the deprecated header option" 1116 ) 1117 1118 if token_provider is not None and model.api_token != "": 1119 raise ValueError( 1120 "If token_provider is set, api_token is ignored and has to be set to empty string." 1121 ) 1122 1123 request_option = ( 1124 self._create_component_from_model( 1125 model.inject_into, config, parameters=model.parameters or {} 1126 ) 1127 if model.inject_into 1128 else RequestOption( 1129 inject_into=RequestOptionType.header, 1130 field_name=model.header or "", 1131 parameters=model.parameters or {}, 1132 ) 1133 ) 1134 1135 return ApiKeyAuthenticator( 1136 token_provider=( 1137 token_provider 1138 if token_provider is not None 1139 else InterpolatedStringTokenProvider( 1140 api_token=model.api_token or "", 1141 config=config, 1142 parameters=model.parameters or {}, 1143 ) 1144 ), 1145 request_option=request_option, 1146 config=config, 1147 parameters=model.parameters or {}, 1148 )
def
create_legacy_to_per_partition_state_migration( self, model: airbyte_cdk.sources.declarative.models.declarative_component_schema.LegacyToPerPartitionStateMigration, config: Mapping[str, Any], declarative_stream: airbyte_cdk.sources.declarative.models.declarative_component_schema.DeclarativeStream) -> airbyte_cdk.LegacyToPerPartitionStateMigration:
1150 def create_legacy_to_per_partition_state_migration( 1151 self, 1152 model: LegacyToPerPartitionStateMigrationModel, 1153 config: Mapping[str, Any], 1154 declarative_stream: DeclarativeStreamModel, 1155 ) -> LegacyToPerPartitionStateMigration: 1156 retriever = declarative_stream.retriever 1157 if not isinstance(retriever, (SimpleRetrieverModel, AsyncRetrieverModel)): 1158 raise ValueError( 1159 f"LegacyToPerPartitionStateMigrations can only be applied on a DeclarativeStream with a SimpleRetriever or AsyncRetriever. Got {type(retriever)}" 1160 ) 1161 partition_router = retriever.partition_router 1162 if not isinstance( 1163 partition_router, (SubstreamPartitionRouterModel, CustomPartitionRouterModel) 1164 ): 1165 raise ValueError( 1166 f"LegacyToPerPartitionStateMigrations can only be applied on a SimpleRetriever with a Substream partition router. Got {type(partition_router)}" 1167 ) 1168 if not hasattr(partition_router, "parent_stream_configs"): 1169 raise ValueError( 1170 "LegacyToPerPartitionStateMigrations can only be applied with a parent stream configuration." 1171 ) 1172 1173 if not hasattr(declarative_stream, "incremental_sync"): 1174 raise ValueError( 1175 "LegacyToPerPartitionStateMigrations can only be applied with an incremental_sync configuration." 1176 ) 1177 1178 return LegacyToPerPartitionStateMigration( 1179 partition_router, # type: ignore # was already checked above 1180 declarative_stream.incremental_sync, # type: ignore # was already checked. Migration can be applied only to incremental streams. 1181 config, 1182 declarative_stream.parameters, # type: ignore # different type is expected here Mapping[str, Any], got Dict[str, Any] 1183 )
def
create_session_token_authenticator( self, model: airbyte_cdk.sources.declarative.models.declarative_component_schema.SessionTokenAuthenticator, config: Mapping[str, Any], name: str, **kwargs: Any) -> Union[airbyte_cdk.ApiKeyAuthenticator, airbyte_cdk.BearerAuthenticator]:
1185 def create_session_token_authenticator( 1186 self, model: SessionTokenAuthenticatorModel, config: Config, name: str, **kwargs: Any 1187 ) -> Union[ApiKeyAuthenticator, BearerAuthenticator]: 1188 decoder = ( 1189 self._create_component_from_model(model=model.decoder, config=config) 1190 if model.decoder 1191 else JsonDecoder(parameters={}) 1192 ) 1193 login_requester = self._create_component_from_model( 1194 model=model.login_requester, 1195 config=config, 1196 name=f"{name}_login_requester", 1197 decoder=decoder, 1198 ) 1199 token_provider = SessionTokenProvider( 1200 login_requester=login_requester, 1201 session_token_path=model.session_token_path, 1202 expiration_duration=parse_duration(model.expiration_duration) 1203 if model.expiration_duration 1204 else None, 1205 parameters=model.parameters or {}, 1206 message_repository=self._message_repository, 1207 decoder=decoder, 1208 ) 1209 if model.request_authentication.type == "Bearer": 1210 return ModelToComponentFactory.create_bearer_authenticator( 1211 BearerAuthenticatorModel(type="BearerAuthenticator", api_token=""), # type: ignore # $parameters has a default value 1212 config, 1213 token_provider=token_provider, 1214 ) 1215 else: 1216 # Get the api_token template if specified, default to just the session token 1217 api_token_template = ( 1218 getattr(model.request_authentication, "api_token", None) or "{{ session_token }}" 1219 ) 1220 final_token_provider: TokenProvider = InterpolatedSessionTokenProvider( 1221 config=config, 1222 api_token=api_token_template, 1223 session_token_provider=token_provider, 1224 parameters=model.parameters or {}, 1225 ) 1226 return self.create_api_key_authenticator( 1227 ApiKeyAuthenticatorModel( 1228 type="ApiKeyAuthenticator", 1229 api_token="", 1230 inject_into=model.request_authentication.inject_into, 1231 ), # type: ignore # $parameters and headers default to None 1232 config=config, 1233 token_provider=final_token_provider, 1234 )
@staticmethod
def
create_basic_http_authenticator( model: airbyte_cdk.sources.declarative.models.declarative_component_schema.BasicHttpAuthenticator, config: Mapping[str, Any], **kwargs: Any) -> airbyte_cdk.BasicHttpAuthenticator:
1236 @staticmethod 1237 def create_basic_http_authenticator( 1238 model: BasicHttpAuthenticatorModel, config: Config, **kwargs: Any 1239 ) -> BasicHttpAuthenticator: 1240 return BasicHttpAuthenticator( 1241 password=model.password or "", 1242 username=model.username, 1243 config=config, 1244 parameters=model.parameters or {}, 1245 )
@staticmethod
def
create_bearer_authenticator( model: airbyte_cdk.sources.declarative.models.declarative_component_schema.BearerAuthenticator, config: Mapping[str, Any], token_provider: Optional[airbyte_cdk.sources.declarative.auth.token_provider.TokenProvider] = None, **kwargs: Any) -> airbyte_cdk.BearerAuthenticator:
1247 @staticmethod 1248 def create_bearer_authenticator( 1249 model: BearerAuthenticatorModel, 1250 config: Config, 1251 token_provider: Optional[TokenProvider] = None, 1252 **kwargs: Any, 1253 ) -> BearerAuthenticator: 1254 if token_provider is not None and model.api_token != "": 1255 raise ValueError( 1256 "If token_provider is set, api_token is ignored and has to be set to empty string." 1257 ) 1258 return BearerAuthenticator( 1259 token_provider=( 1260 token_provider 1261 if token_provider is not None 1262 else InterpolatedStringTokenProvider( 1263 api_token=model.api_token or "", 1264 config=config, 1265 parameters=model.parameters or {}, 1266 ) 1267 ), 1268 config=config, 1269 parameters=model.parameters or {}, 1270 )
@staticmethod
def
create_dynamic_stream_check_config( model: airbyte_cdk.sources.declarative.models.declarative_component_schema.DynamicStreamCheckConfig, config: Mapping[str, Any], **kwargs: Any) -> airbyte_cdk.sources.declarative.checks.DynamicStreamCheckConfig:
1272 @staticmethod 1273 def create_dynamic_stream_check_config( 1274 model: DynamicStreamCheckConfigModel, config: Config, **kwargs: Any 1275 ) -> DynamicStreamCheckConfig: 1276 return DynamicStreamCheckConfig( 1277 dynamic_stream_name=model.dynamic_stream_name, 1278 stream_count=model.stream_count, 1279 )
def
create_check_stream( self, model: airbyte_cdk.sources.declarative.models.declarative_component_schema.CheckStream, config: Mapping[str, Any], **kwargs: Any) -> airbyte_cdk.sources.declarative.checks.CheckStream:
1281 def create_check_stream( 1282 self, model: CheckStreamModel, config: Config, **kwargs: Any 1283 ) -> CheckStream: 1284 if model.dynamic_streams_check_configs is None and model.stream_names is None: 1285 raise ValueError( 1286 "Expected either stream_names or dynamic_streams_check_configs to be set for CheckStream" 1287 ) 1288 1289 dynamic_streams_check_configs = ( 1290 [ 1291 self._create_component_from_model(model=dynamic_stream_check_config, config=config) 1292 for dynamic_stream_check_config in model.dynamic_streams_check_configs 1293 ] 1294 if model.dynamic_streams_check_configs 1295 else [] 1296 ) 1297 1298 return CheckStream( 1299 stream_names=model.stream_names or [], 1300 dynamic_streams_check_configs=dynamic_streams_check_configs, 1301 parameters={}, 1302 )
@staticmethod
def
create_check_dynamic_stream( model: airbyte_cdk.sources.declarative.models.declarative_component_schema.CheckDynamicStream, config: Mapping[str, Any], **kwargs: Any) -> airbyte_cdk.sources.declarative.checks.CheckDynamicStream:
1304 @staticmethod 1305 def create_check_dynamic_stream( 1306 model: CheckDynamicStreamModel, config: Config, **kwargs: Any 1307 ) -> CheckDynamicStream: 1308 assert model.use_check_availability is not None # for mypy 1309 1310 use_check_availability = model.use_check_availability 1311 1312 return CheckDynamicStream( 1313 stream_count=model.stream_count, 1314 use_check_availability=use_check_availability, 1315 parameters={}, 1316 )
def
create_composite_error_handler( self, model: airbyte_cdk.sources.declarative.models.declarative_component_schema.CompositeErrorHandler, config: Mapping[str, Any], **kwargs: Any) -> airbyte_cdk.sources.declarative.requesters.error_handlers.CompositeErrorHandler:
1318 def create_composite_error_handler( 1319 self, model: CompositeErrorHandlerModel, config: Config, **kwargs: Any 1320 ) -> CompositeErrorHandler: 1321 error_handlers = [ 1322 self._create_component_from_model(model=error_handler_model, config=config) 1323 for error_handler_model in model.error_handlers 1324 ] 1325 return CompositeErrorHandler( 1326 error_handlers=error_handlers, parameters=model.parameters or {} 1327 )
@staticmethod
def
create_concurrency_level( model: airbyte_cdk.sources.declarative.models.declarative_component_schema.ConcurrencyLevel, config: Mapping[str, Any], **kwargs: Any) -> airbyte_cdk.sources.declarative.concurrency_level.ConcurrencyLevel:
1329 @staticmethod 1330 def create_concurrency_level( 1331 model: ConcurrencyLevelModel, config: Config, **kwargs: Any 1332 ) -> ConcurrencyLevel: 1333 return ConcurrencyLevel( 1334 default_concurrency=model.default_concurrency, 1335 max_concurrency=model.max_concurrency, 1336 config=config, 1337 parameters={}, 1338 )
@staticmethod
def
apply_stream_state_migrations( stream_state_migrations: Optional[List[Any]], stream_state: MutableMapping[str, Any]) -> MutableMapping[str, Any]:
1340 @staticmethod 1341 def apply_stream_state_migrations( 1342 stream_state_migrations: List[Any] | None, stream_state: MutableMapping[str, Any] 1343 ) -> MutableMapping[str, Any]: 1344 if stream_state_migrations: 1345 for state_migration in stream_state_migrations: 1346 if state_migration.should_migrate(stream_state): 1347 # The state variable is expected to be mutable but the migrate method returns an immutable mapping. 1348 stream_state = dict(state_migration.migrate(stream_state)) 1349 return stream_state
def
create_concurrent_cursor_from_datetime_based_cursor( self, model_type: Type[pydantic.v1.main.BaseModel], component_definition: Mapping[str, Any], stream_name: str, stream_namespace: Optional[str], stream_state: MutableMapping[str, Any], config: Mapping[str, Any], message_repository: Optional[airbyte_cdk.MessageRepository] = None, runtime_lookback_window: Optional[datetime.timedelta] = None, **kwargs: Any) -> airbyte_cdk.ConcurrentCursor:
1351 def create_concurrent_cursor_from_datetime_based_cursor( 1352 self, 1353 model_type: Type[BaseModel], 1354 component_definition: ComponentDefinition, 1355 stream_name: str, 1356 stream_namespace: Optional[str], 1357 stream_state: MutableMapping[str, Any], 1358 config: Config, 1359 message_repository: Optional[MessageRepository] = None, 1360 runtime_lookback_window: Optional[datetime.timedelta] = None, 1361 **kwargs: Any, 1362 ) -> ConcurrentCursor: 1363 component_type = component_definition.get("type") 1364 if component_definition.get("type") != model_type.__name__: 1365 raise ValueError( 1366 f"Expected manifest component of type {model_type.__name__}, but received {component_type} instead" 1367 ) 1368 1369 # FIXME the interfaces of the concurrent cursor are kind of annoying as they take a `ComponentDefinition` instead of the actual model. This was done because the ConcurrentDeclarativeSource didn't have access to the models [here for example](https://github.com/airbytehq/airbyte-python-cdk/blob/f525803b3fec9329e4cc8478996a92bf884bfde9/airbyte_cdk/sources/declarative/concurrent_declarative_source.py#L354C54-L354C91). So now we have two cases: 1370 # * The ComponentDefinition comes from model.__dict__ in which case we have `parameters` 1371 # * The ComponentDefinition comes from the manifest as a dict in which case we have `$parameters` 1372 # We should change those interfaces to use the model once we clean up the code in CDS at which point the parameter propagation should happen as part of the ModelToComponentFactory. 1373 if "$parameters" not in component_definition and "parameters" in component_definition: 1374 component_definition["$parameters"] = component_definition.get("parameters") # type: ignore # This is a dict 1375 datetime_based_cursor_model = model_type.parse_obj(component_definition) 1376 1377 if not isinstance(datetime_based_cursor_model, DatetimeBasedCursorModel): 1378 raise ValueError( 1379 f"Expected {model_type.__name__} component, but received {datetime_based_cursor_model.__class__.__name__}" 1380 ) 1381 1382 model_parameters = datetime_based_cursor_model.parameters or {} 1383 1384 cursor_field = self._get_catalog_defined_cursor_field( 1385 stream_name=stream_name, 1386 allow_catalog_defined_cursor_field=datetime_based_cursor_model.allow_catalog_defined_cursor_field 1387 or False, 1388 ) 1389 1390 if not cursor_field: 1391 interpolated_cursor_field = InterpolatedString.create( 1392 datetime_based_cursor_model.cursor_field, 1393 parameters=model_parameters, 1394 ) 1395 cursor_field = CursorField( 1396 cursor_field_key=interpolated_cursor_field.eval(config=config), 1397 supports_catalog_defined_cursor_field=datetime_based_cursor_model.allow_catalog_defined_cursor_field 1398 or False, 1399 ) 1400 1401 interpolated_partition_field_start = InterpolatedString.create( 1402 datetime_based_cursor_model.partition_field_start or "start_time", 1403 parameters=model_parameters, 1404 ) 1405 interpolated_partition_field_end = InterpolatedString.create( 1406 datetime_based_cursor_model.partition_field_end or "end_time", 1407 parameters=model_parameters, 1408 ) 1409 1410 slice_boundary_fields = ( 1411 interpolated_partition_field_start.eval(config=config), 1412 interpolated_partition_field_end.eval(config=config), 1413 ) 1414 1415 datetime_format = datetime_based_cursor_model.datetime_format 1416 1417 cursor_granularity = ( 1418 parse_duration(datetime_based_cursor_model.cursor_granularity) 1419 if datetime_based_cursor_model.cursor_granularity 1420 else None 1421 ) 1422 1423 lookback_window = None 1424 interpolated_lookback_window = ( 1425 InterpolatedString.create( 1426 datetime_based_cursor_model.lookback_window, 1427 parameters=model_parameters, 1428 ) 1429 if datetime_based_cursor_model.lookback_window 1430 else None 1431 ) 1432 if interpolated_lookback_window: 1433 evaluated_lookback_window = interpolated_lookback_window.eval(config=config) 1434 if evaluated_lookback_window: 1435 lookback_window = parse_duration(evaluated_lookback_window) 1436 1437 connector_state_converter: DateTimeStreamStateConverter 1438 connector_state_converter = CustomFormatConcurrentStreamStateConverter( 1439 datetime_format=datetime_format, 1440 input_datetime_formats=datetime_based_cursor_model.cursor_datetime_formats, 1441 is_sequential_state=True, # ConcurrentPerPartitionCursor only works with sequential state 1442 cursor_granularity=cursor_granularity, 1443 ) 1444 1445 # Adjusts the stream state by applying the runtime lookback window. 1446 # This is used to ensure correct state handling in case of failed partitions. 1447 stream_state_value = stream_state.get(cursor_field.cursor_field_key) 1448 if runtime_lookback_window and stream_state_value: 1449 new_stream_state = ( 1450 connector_state_converter.parse_timestamp(stream_state_value) 1451 - runtime_lookback_window 1452 ) 1453 stream_state[cursor_field.cursor_field_key] = connector_state_converter.output_format( 1454 new_stream_state 1455 ) 1456 1457 start_date_runtime_value: Union[InterpolatedString, str, MinMaxDatetime] 1458 if isinstance(datetime_based_cursor_model.start_datetime, MinMaxDatetimeModel): 1459 start_date_runtime_value = self.create_min_max_datetime( 1460 model=datetime_based_cursor_model.start_datetime, config=config 1461 ) 1462 else: 1463 start_date_runtime_value = datetime_based_cursor_model.start_datetime 1464 1465 end_date_runtime_value: Optional[Union[InterpolatedString, str, MinMaxDatetime]] 1466 if isinstance(datetime_based_cursor_model.end_datetime, MinMaxDatetimeModel): 1467 end_date_runtime_value = self.create_min_max_datetime( 1468 model=datetime_based_cursor_model.end_datetime, config=config 1469 ) 1470 else: 1471 end_date_runtime_value = datetime_based_cursor_model.end_datetime 1472 1473 interpolated_start_date = MinMaxDatetime.create( 1474 interpolated_string_or_min_max_datetime=start_date_runtime_value, 1475 parameters=datetime_based_cursor_model.parameters, 1476 ) 1477 interpolated_end_date = ( 1478 None 1479 if not end_date_runtime_value 1480 else MinMaxDatetime.create( 1481 end_date_runtime_value, datetime_based_cursor_model.parameters 1482 ) 1483 ) 1484 1485 # If datetime format is not specified then start/end datetime should inherit it from the stream slicer 1486 if not interpolated_start_date.datetime_format: 1487 interpolated_start_date.datetime_format = datetime_format 1488 if interpolated_end_date and not interpolated_end_date.datetime_format: 1489 interpolated_end_date.datetime_format = datetime_format 1490 1491 start_date = interpolated_start_date.get_datetime(config=config) 1492 end_date_provider = ( 1493 partial(interpolated_end_date.get_datetime, config) 1494 if interpolated_end_date 1495 else connector_state_converter.get_end_provider() 1496 ) 1497 1498 if ( 1499 datetime_based_cursor_model.step and not datetime_based_cursor_model.cursor_granularity 1500 ) or ( 1501 not datetime_based_cursor_model.step and datetime_based_cursor_model.cursor_granularity 1502 ): 1503 raise ValueError( 1504 f"If step is defined, cursor_granularity should be as well and vice-versa. " 1505 f"Right now, step is `{datetime_based_cursor_model.step}` and cursor_granularity is `{datetime_based_cursor_model.cursor_granularity}`" 1506 ) 1507 1508 # When step is not defined, default to a step size from the starting date to the present moment 1509 step_length = datetime.timedelta.max 1510 interpolated_step = ( 1511 InterpolatedString.create( 1512 datetime_based_cursor_model.step, 1513 parameters=model_parameters, 1514 ) 1515 if datetime_based_cursor_model.step 1516 else None 1517 ) 1518 if interpolated_step: 1519 evaluated_step = interpolated_step.eval(config) 1520 if evaluated_step: 1521 step_length = parse_duration(evaluated_step) 1522 1523 clamping_strategy: ClampingStrategy = NoClamping() 1524 if datetime_based_cursor_model.clamping: 1525 # While it is undesirable to interpolate within the model factory (as opposed to at runtime), 1526 # it is still better than shifting interpolation low-code concept into the ConcurrentCursor runtime 1527 # object which we want to keep agnostic of being low-code 1528 target = InterpolatedString( 1529 string=datetime_based_cursor_model.clamping.target, 1530 parameters=model_parameters, 1531 ) 1532 evaluated_target = target.eval(config=config) 1533 match evaluated_target: 1534 case "DAY": 1535 clamping_strategy = DayClampingStrategy() 1536 end_date_provider = ClampingEndProvider( 1537 DayClampingStrategy(is_ceiling=False), 1538 end_date_provider, # type: ignore # Having issues w/ inspection for GapType and CursorValueType as shown in existing tests. Confirmed functionality is working in practice 1539 granularity=cursor_granularity or datetime.timedelta(seconds=1), 1540 ) 1541 case "WEEK": 1542 if ( 1543 not datetime_based_cursor_model.clamping.target_details 1544 or "weekday" not in datetime_based_cursor_model.clamping.target_details 1545 ): 1546 raise ValueError( 1547 "Given WEEK clamping, weekday needs to be provided as target_details" 1548 ) 1549 weekday = self._assemble_weekday( 1550 datetime_based_cursor_model.clamping.target_details["weekday"] 1551 ) 1552 clamping_strategy = WeekClampingStrategy(weekday) 1553 end_date_provider = ClampingEndProvider( 1554 WeekClampingStrategy(weekday, is_ceiling=False), 1555 end_date_provider, # type: ignore # Having issues w/ inspection for GapType and CursorValueType as shown in existing tests. Confirmed functionality is working in practice 1556 granularity=cursor_granularity or datetime.timedelta(days=1), 1557 ) 1558 case "MONTH": 1559 clamping_strategy = MonthClampingStrategy() 1560 end_date_provider = ClampingEndProvider( 1561 MonthClampingStrategy(is_ceiling=False), 1562 end_date_provider, # type: ignore # Having issues w/ inspection for GapType and CursorValueType as shown in existing tests. Confirmed functionality is working in practice 1563 granularity=cursor_granularity or datetime.timedelta(days=1), 1564 ) 1565 case _: 1566 raise ValueError( 1567 f"Invalid clamping target {evaluated_target}, expected DAY, WEEK, MONTH" 1568 ) 1569 1570 return ConcurrentCursor( 1571 stream_name=stream_name, 1572 stream_namespace=stream_namespace, 1573 stream_state=stream_state, 1574 message_repository=message_repository or self._message_repository, 1575 connector_state_manager=self._connector_state_manager, 1576 connector_state_converter=connector_state_converter, 1577 cursor_field=cursor_field, 1578 slice_boundary_fields=slice_boundary_fields, 1579 start=start_date, # type: ignore # Having issues w/ inspection for GapType and CursorValueType as shown in existing tests. Confirmed functionality is working in practice 1580 end_provider=end_date_provider, # type: ignore # Having issues w/ inspection for GapType and CursorValueType as shown in existing tests. Confirmed functionality is working in practice 1581 lookback_window=lookback_window, 1582 slice_range=step_length, 1583 cursor_granularity=cursor_granularity, 1584 clamping_strategy=clamping_strategy, 1585 )
def
create_concurrent_cursor_from_incrementing_count_cursor( self, model_type: Type[pydantic.v1.main.BaseModel], component_definition: Mapping[str, Any], stream_name: str, stream_namespace: Optional[str], stream_state: MutableMapping[str, Any], config: Mapping[str, Any], message_repository: Optional[airbyte_cdk.MessageRepository] = None, **kwargs: Any) -> airbyte_cdk.ConcurrentCursor:
1587 def create_concurrent_cursor_from_incrementing_count_cursor( 1588 self, 1589 model_type: Type[BaseModel], 1590 component_definition: ComponentDefinition, 1591 stream_name: str, 1592 stream_namespace: Optional[str], 1593 stream_state: MutableMapping[str, Any], 1594 config: Config, 1595 message_repository: Optional[MessageRepository] = None, 1596 **kwargs: Any, 1597 ) -> ConcurrentCursor: 1598 component_type = component_definition.get("type") 1599 if component_definition.get("type") != model_type.__name__: 1600 raise ValueError( 1601 f"Expected manifest component of type {model_type.__name__}, but received {component_type} instead" 1602 ) 1603 1604 incrementing_count_cursor_model = model_type.parse_obj(component_definition) 1605 1606 if not isinstance(incrementing_count_cursor_model, IncrementingCountCursorModel): 1607 raise ValueError( 1608 f"Expected {model_type.__name__} component, but received {incrementing_count_cursor_model.__class__.__name__}" 1609 ) 1610 1611 start_value: Union[int, str, None] = incrementing_count_cursor_model.start_value 1612 # Pydantic Union type coercion can convert int 0 to string '0' depending on Union order. 1613 # We need to handle both int and str representations of numeric values. 1614 # Evaluate the InterpolatedString and convert to int for the ConcurrentCursor. 1615 if start_value is not None: 1616 interpolated_start_value = InterpolatedString.create( 1617 str(start_value), # Ensure we pass a string to InterpolatedString.create 1618 parameters=incrementing_count_cursor_model.parameters or {}, 1619 ) 1620 evaluated_start_value: int = int(interpolated_start_value.eval(config=config)) 1621 else: 1622 evaluated_start_value = 0 1623 1624 cursor_field = self._get_catalog_defined_cursor_field( 1625 stream_name=stream_name, 1626 allow_catalog_defined_cursor_field=incrementing_count_cursor_model.allow_catalog_defined_cursor_field 1627 or False, 1628 ) 1629 1630 if not cursor_field: 1631 interpolated_cursor_field = InterpolatedString.create( 1632 incrementing_count_cursor_model.cursor_field, 1633 parameters=incrementing_count_cursor_model.parameters or {}, 1634 ) 1635 cursor_field = CursorField( 1636 cursor_field_key=interpolated_cursor_field.eval(config=config), 1637 supports_catalog_defined_cursor_field=incrementing_count_cursor_model.allow_catalog_defined_cursor_field 1638 or False, 1639 ) 1640 1641 connector_state_converter = IncrementingCountStreamStateConverter( 1642 is_sequential_state=True, # ConcurrentPerPartitionCursor only works with sequential state 1643 ) 1644 1645 return ConcurrentCursor( 1646 stream_name=stream_name, 1647 stream_namespace=stream_namespace, 1648 stream_state=stream_state, 1649 message_repository=message_repository or self._message_repository, 1650 connector_state_manager=self._connector_state_manager, 1651 connector_state_converter=connector_state_converter, 1652 cursor_field=cursor_field, 1653 slice_boundary_fields=None, 1654 start=evaluated_start_value, # type: ignore # Having issues w/ inspection for GapType and CursorValueType as shown in existing tests. Confirmed functionality is working in practice 1655 end_provider=connector_state_converter.get_end_provider(), # type: ignore # Having issues w/ inspection for GapType and CursorValueType as shown in existing tests. Confirmed functionality is working in practice 1656 )
def
create_concurrent_cursor_from_perpartition_cursor( self, state_manager: airbyte_cdk.ConnectorStateManager, model_type: Type[pydantic.v1.main.BaseModel], component_definition: Mapping[str, Any], stream_name: str, stream_namespace: Optional[str], config: Mapping[str, Any], stream_state: MutableMapping[str, Any], partition_router: airbyte_cdk.sources.declarative.partition_routers.PartitionRouter, attempt_to_create_cursor_if_not_provided: bool = False, **kwargs: Any) -> airbyte_cdk.sources.declarative.incremental.ConcurrentPerPartitionCursor:
1677 def create_concurrent_cursor_from_perpartition_cursor( 1678 self, 1679 state_manager: ConnectorStateManager, 1680 model_type: Type[BaseModel], 1681 component_definition: ComponentDefinition, 1682 stream_name: str, 1683 stream_namespace: Optional[str], 1684 config: Config, 1685 stream_state: MutableMapping[str, Any], 1686 partition_router: PartitionRouter, 1687 attempt_to_create_cursor_if_not_provided: bool = False, 1688 **kwargs: Any, 1689 ) -> ConcurrentPerPartitionCursor: 1690 component_type = component_definition.get("type") 1691 if component_definition.get("type") != model_type.__name__: 1692 raise ValueError( 1693 f"Expected manifest component of type {model_type.__name__}, but received {component_type} instead" 1694 ) 1695 1696 # FIXME the interfaces of the concurrent cursor are kind of annoying as they take a `ComponentDefinition` instead of the actual model. This was done because the ConcurrentDeclarativeSource didn't have access to the models [here for example](https://github.com/airbytehq/airbyte-python-cdk/blob/f525803b3fec9329e4cc8478996a92bf884bfde9/airbyte_cdk/sources/declarative/concurrent_declarative_source.py#L354C54-L354C91). So now we have two cases: 1697 # * The ComponentDefinition comes from model.__dict__ in which case we have `parameters` 1698 # * The ComponentDefinition comes from the manifest as a dict in which case we have `$parameters` 1699 # We should change those interfaces to use the model once we clean up the code in CDS at which point the parameter propagation should happen as part of the ModelToComponentFactory. 1700 if "$parameters" not in component_definition and "parameters" in component_definition: 1701 component_definition["$parameters"] = component_definition.get("parameters") # type: ignore # This is a dict 1702 datetime_based_cursor_model = model_type.parse_obj(component_definition) 1703 1704 if not isinstance(datetime_based_cursor_model, DatetimeBasedCursorModel): 1705 raise ValueError( 1706 f"Expected {model_type.__name__} component, but received {datetime_based_cursor_model.__class__.__name__}" 1707 ) 1708 1709 cursor_field = self._get_catalog_defined_cursor_field( 1710 stream_name=stream_name, 1711 allow_catalog_defined_cursor_field=datetime_based_cursor_model.allow_catalog_defined_cursor_field 1712 or False, 1713 ) 1714 1715 if not cursor_field: 1716 interpolated_cursor_field = InterpolatedString.create( 1717 datetime_based_cursor_model.cursor_field, 1718 # FIXME the interfaces of the concurrent cursor are kind of annoying as they take a `ComponentDefinition` instead of the actual model. This was done because the ConcurrentDeclarativeSource didn't have access to the models [here for example](https://github.com/airbytehq/airbyte-python-cdk/blob/f525803b3fec9329e4cc8478996a92bf884bfde9/airbyte_cdk/sources/declarative/concurrent_declarative_source.py#L354C54-L354C91). So now we have two cases: 1719 # * The ComponentDefinition comes from model.__dict__ in which case we have `parameters` 1720 # * The ComponentDefinition comes from the manifest as a dict in which case we have `$parameters` 1721 # We should change those interfaces to use the model once we clean up the code in CDS at which point the parameter propagation should happen as part of the ModelToComponentFactory. 1722 parameters=datetime_based_cursor_model.parameters or {}, 1723 ) 1724 cursor_field = CursorField( 1725 cursor_field_key=interpolated_cursor_field.eval(config=config), 1726 supports_catalog_defined_cursor_field=datetime_based_cursor_model.allow_catalog_defined_cursor_field 1727 or False, 1728 ) 1729 1730 datetime_format = datetime_based_cursor_model.datetime_format 1731 1732 cursor_granularity = ( 1733 parse_duration(datetime_based_cursor_model.cursor_granularity) 1734 if datetime_based_cursor_model.cursor_granularity 1735 else None 1736 ) 1737 1738 connector_state_converter: DateTimeStreamStateConverter 1739 connector_state_converter = CustomFormatConcurrentStreamStateConverter( 1740 datetime_format=datetime_format, 1741 input_datetime_formats=datetime_based_cursor_model.cursor_datetime_formats, 1742 is_sequential_state=True, # ConcurrentPerPartitionCursor only works with sequential state 1743 cursor_granularity=cursor_granularity, 1744 ) 1745 1746 # Create the cursor factory 1747 cursor_factory = ConcurrentCursorFactory( 1748 partial( 1749 self.create_concurrent_cursor_from_datetime_based_cursor, 1750 state_manager=state_manager, 1751 model_type=model_type, 1752 component_definition=component_definition, 1753 stream_name=stream_name, 1754 stream_namespace=stream_namespace, 1755 config=config, 1756 message_repository=NoopMessageRepository(), 1757 ) 1758 ) 1759 1760 # Per-partition state doesn't make sense for GroupingPartitionRouter, so force the global state 1761 use_global_cursor = isinstance( 1762 partition_router, GroupingPartitionRouter 1763 ) or component_definition.get("global_substream_cursor", False) 1764 1765 # Return the concurrent cursor and state converter 1766 return ConcurrentPerPartitionCursor( 1767 cursor_factory=cursor_factory, 1768 partition_router=partition_router, 1769 stream_name=stream_name, 1770 stream_namespace=stream_namespace, 1771 stream_state=stream_state, 1772 message_repository=self._message_repository, # type: ignore 1773 connector_state_manager=state_manager, 1774 connector_state_converter=connector_state_converter, 1775 cursor_field=cursor_field, 1776 use_global_cursor=use_global_cursor, 1777 attempt_to_create_cursor_if_not_provided=attempt_to_create_cursor_if_not_provided, 1778 )
@staticmethod
def
create_constant_backoff_strategy( model: airbyte_cdk.sources.declarative.models.declarative_component_schema.ConstantBackoffStrategy, config: Mapping[str, Any], **kwargs: Any) -> airbyte_cdk.sources.declarative.requesters.error_handlers.backoff_strategies.ConstantBackoffStrategy:
1780 @staticmethod 1781 def create_constant_backoff_strategy( 1782 model: ConstantBackoffStrategyModel, config: Config, **kwargs: Any 1783 ) -> ConstantBackoffStrategy: 1784 ModelToComponentFactory._validate_jitter_range(model.jitter_range_in_seconds) 1785 return ConstantBackoffStrategy( 1786 backoff_time_in_seconds=model.backoff_time_in_seconds, 1787 jitter_range_in_seconds=model.jitter_range_in_seconds, 1788 config=config, 1789 parameters=model.parameters or {}, 1790 )
def
create_cursor_pagination( self, model: airbyte_cdk.sources.declarative.models.declarative_component_schema.CursorPagination, config: Mapping[str, Any], decoder: airbyte_cdk.Decoder, **kwargs: Any) -> airbyte_cdk.CursorPaginationStrategy:
1797 def create_cursor_pagination( 1798 self, model: CursorPaginationModel, config: Config, decoder: Decoder, **kwargs: Any 1799 ) -> CursorPaginationStrategy: 1800 if isinstance(decoder, PaginationDecoderDecorator): 1801 inner_decoder = decoder.decoder 1802 else: 1803 inner_decoder = decoder 1804 decoder = PaginationDecoderDecorator(decoder=decoder) 1805 1806 if self._is_supported_decoder_for_pagination(inner_decoder): 1807 decoder_to_use = decoder 1808 else: 1809 raise ValueError( 1810 self._UNSUPPORTED_DECODER_ERROR.format(decoder_type=type(inner_decoder)) 1811 ) 1812 1813 # Pydantic v1 Union type coercion can convert int to string depending on Union order. 1814 # If page_size is a string that represents an integer (not an interpolation), convert it back. 1815 page_size = model.page_size 1816 if isinstance(page_size, str) and page_size.isdigit(): 1817 page_size = int(page_size) 1818 1819 return CursorPaginationStrategy( 1820 cursor_value=model.cursor_value, 1821 decoder=decoder_to_use, 1822 page_size=page_size, 1823 stop_condition=model.stop_condition, 1824 config=config, 1825 parameters=model.parameters or {}, 1826 )
def
create_custom_component(self, model: Any, config: Mapping[str, Any], **kwargs: Any) -> Any:
1828 def create_custom_component(self, model: Any, config: Config, **kwargs: Any) -> Any: 1829 """ 1830 Generically creates a custom component based on the model type and a class_name reference to the custom Python class being 1831 instantiated. Only the model's additional properties that match the custom class definition are passed to the constructor 1832 :param model: The Pydantic model of the custom component being created 1833 :param config: The custom defined connector config 1834 :return: The declarative component built from the Pydantic model to be used at runtime 1835 """ 1836 # Instantiating a custom component means importing and executing arbitrary code referenced 1837 # by `class_name`. Manifests supplied by a caller, whether through the config or directly to 1838 # the manifest server, are untrusted input and could point `class_name` at any importable 1839 # callable, so they honor the same `AIRBYTE_ENABLE_UNSAFE_CODE` gate as injected 1840 # `components.py` code. Manifests bundled in a published connector image are trusted and may 1841 # always use their bundled custom components. 1842 manifest_is_untrusted = not self._custom_components_trusted or bool( 1843 config.get(INJECTED_MANIFEST) 1844 ) 1845 if manifest_is_untrusted and not custom_code_execution_permitted(): 1846 raise AirbyteCustomCodeNotPermittedError 1847 1848 custom_component_class = self._get_class_from_fully_qualified_class_name(model.class_name) 1849 component_fields = get_type_hints(custom_component_class) 1850 model_args = model.dict() 1851 model_args["config"] = config 1852 1853 # There are cases where a parent component will pass arguments to a child component via kwargs. When there are field collisions 1854 # we defer to these arguments over the component's definition 1855 for key, arg in kwargs.items(): 1856 model_args[key] = arg 1857 1858 # Pydantic is unable to parse a custom component's fields that are subcomponents into models because their fields and types are not 1859 # defined in the schema. The fields and types are defined within the Python class implementation. Pydantic can only parse down to 1860 # the custom component and this code performs a second parse to convert the sub-fields first into models, then declarative components 1861 for model_field, model_value in model_args.items(): 1862 # If a custom component field doesn't have a type set, we try to use the type hints to infer the type 1863 if ( 1864 isinstance(model_value, dict) 1865 and "type" not in model_value 1866 and model_field in component_fields 1867 ): 1868 derived_type = self._derive_component_type_from_type_hints( 1869 component_fields.get(model_field) 1870 ) 1871 if derived_type: 1872 model_value["type"] = derived_type 1873 1874 if self._is_component(model_value): 1875 model_args[model_field] = self._create_nested_component( 1876 model, 1877 model_field, 1878 model_value, 1879 config, 1880 **kwargs, 1881 ) 1882 elif isinstance(model_value, list): 1883 vals = [] 1884 for v in model_value: 1885 if isinstance(v, dict) and "type" not in v and model_field in component_fields: 1886 derived_type = self._derive_component_type_from_type_hints( 1887 component_fields.get(model_field) 1888 ) 1889 if derived_type: 1890 v["type"] = derived_type 1891 if self._is_component(v): 1892 vals.append( 1893 self._create_nested_component( 1894 model, 1895 model_field, 1896 v, 1897 config, 1898 **kwargs, 1899 ) 1900 ) 1901 else: 1902 vals.append(v) 1903 model_args[model_field] = vals 1904 1905 kwargs = { 1906 class_field: model_args[class_field] 1907 for class_field in component_fields.keys() 1908 if class_field in model_args 1909 } 1910 1911 if "api_budget" in component_fields and kwargs.get("api_budget") is None: 1912 kwargs["api_budget"] = self._api_budget 1913 1914 return custom_component_class(**kwargs)
Generically creates a custom component based on the model type and a class_name reference to the custom Python class being instantiated. Only the model's additional properties that match the custom class definition are passed to the constructor
Parameters
- model: The Pydantic model of the custom component being created
- config: The custom defined connector config
Returns
The declarative component built from the Pydantic model to be used at runtime
def
create_default_stream( self, model: airbyte_cdk.sources.declarative.models.declarative_component_schema.DeclarativeStream, config: Mapping[str, Any], is_parent: bool = False, **kwargs: Any) -> airbyte_cdk.sources.streams.concurrent.abstract_stream.AbstractStream:
2049 def create_default_stream( 2050 self, model: DeclarativeStreamModel, config: Config, is_parent: bool = False, **kwargs: Any 2051 ) -> AbstractStream: 2052 primary_key = model.primary_key.__root__ if model.primary_key else None 2053 self._migrate_state(model, config) 2054 2055 partition_router = self._build_stream_slicer_from_partition_router( 2056 model.retriever, 2057 config, 2058 stream_name=model.name, 2059 **kwargs, 2060 ) 2061 concurrent_cursor = self._build_concurrent_cursor(model, partition_router, config) 2062 if model.incremental_sync and isinstance(model.incremental_sync, DatetimeBasedCursorModel): 2063 cursor_model: DatetimeBasedCursorModel = model.incremental_sync 2064 2065 end_time_option = ( 2066 self._create_component_from_model( 2067 cursor_model.end_time_option, config, parameters=cursor_model.parameters or {} 2068 ) 2069 if cursor_model.end_time_option 2070 else None 2071 ) 2072 start_time_option = ( 2073 self._create_component_from_model( 2074 cursor_model.start_time_option, config, parameters=cursor_model.parameters or {} 2075 ) 2076 if cursor_model.start_time_option 2077 else None 2078 ) 2079 2080 datetime_request_options_provider = DatetimeBasedRequestOptionsProvider( 2081 start_time_option=start_time_option, 2082 end_time_option=end_time_option, 2083 partition_field_start=cursor_model.partition_field_start, 2084 partition_field_end=cursor_model.partition_field_end, 2085 config=config, 2086 parameters=model.parameters or {}, 2087 ) 2088 request_options_provider = ( 2089 datetime_request_options_provider 2090 if not isinstance(concurrent_cursor, ConcurrentPerPartitionCursor) 2091 else PerPartitionRequestOptionsProvider( 2092 partition_router, datetime_request_options_provider 2093 ) 2094 ) 2095 elif model.incremental_sync and isinstance( 2096 model.incremental_sync, IncrementingCountCursorModel 2097 ): 2098 if isinstance(concurrent_cursor, ConcurrentPerPartitionCursor): 2099 raise ValueError( 2100 "PerPartition does not support per partition states because switching to global state is time based" 2101 ) 2102 2103 cursor_model: IncrementingCountCursorModel = model.incremental_sync # type: ignore 2104 2105 start_time_option = ( 2106 self._create_component_from_model( 2107 cursor_model.start_value_option, # type: ignore # mypy still thinks cursor_model of type DatetimeBasedCursor 2108 config, 2109 parameters=cursor_model.parameters or {}, 2110 ) 2111 if cursor_model.start_value_option # type: ignore # mypy still thinks cursor_model of type DatetimeBasedCursor 2112 else None 2113 ) 2114 2115 # The concurrent engine defaults the start/end fields on the slice to "start" and "end", but 2116 # the default DatetimeBasedRequestOptionsProvider() sets them to start_time/end_time 2117 partition_field_start = "start" 2118 2119 request_options_provider = DatetimeBasedRequestOptionsProvider( 2120 start_time_option=start_time_option, 2121 partition_field_start=partition_field_start, 2122 config=config, 2123 parameters=model.parameters or {}, 2124 ) 2125 else: 2126 request_options_provider = None 2127 2128 transformations = [] 2129 if model.transformations: 2130 for transformation_model in model.transformations: 2131 transformations.append( 2132 self._create_component_from_model(model=transformation_model, config=config) 2133 ) 2134 file_uploader = None 2135 if model.file_uploader: 2136 file_uploader = self._create_component_from_model( 2137 model=model.file_uploader, config=config 2138 ) 2139 2140 stream_slicer: ConcurrentStreamSlicer = ( 2141 partition_router 2142 if isinstance(concurrent_cursor, FinalStateCursor) 2143 else concurrent_cursor 2144 ) 2145 2146 retriever = self._create_component_from_model( 2147 model=model.retriever, 2148 config=config, 2149 name=model.name, 2150 primary_key=primary_key, 2151 request_options_provider=request_options_provider, 2152 stream_slicer=stream_slicer, 2153 partition_router=partition_router, 2154 has_stop_condition_cursor=self._is_stop_condition_on_cursor(model), 2155 is_client_side_incremental_sync=self._is_client_side_filtering_enabled(model), 2156 cursor=concurrent_cursor, 2157 transformations=transformations, 2158 file_uploader=file_uploader, 2159 incremental_sync=model.incremental_sync, 2160 ) 2161 if isinstance(retriever, AsyncRetriever): 2162 stream_slicer = retriever.stream_slicer 2163 2164 schema_loader: SchemaLoader 2165 if model.schema_loader and isinstance(model.schema_loader, list): 2166 nested_schema_loaders = [ 2167 self._create_component_from_model(model=nested_schema_loader, config=config) 2168 for nested_schema_loader in model.schema_loader 2169 ] 2170 schema_loader = CompositeSchemaLoader( 2171 schema_loaders=nested_schema_loaders, parameters={} 2172 ) 2173 elif model.schema_loader: 2174 schema_loader = self._create_component_from_model( 2175 model=model.schema_loader, # type: ignore # If defined, schema_loader is guaranteed not to be a list and will be one of the existing base models 2176 config=config, 2177 ) 2178 else: 2179 options = model.parameters or {} 2180 if "name" not in options: 2181 options["name"] = model.name 2182 schema_loader = DefaultSchemaLoader(config=config, parameters=options) 2183 schema_loader = CachingSchemaLoaderDecorator(schema_loader) 2184 2185 stream_name = model.name or "" 2186 return DefaultStream( 2187 partition_generator=StreamSlicerPartitionGenerator( 2188 DeclarativePartitionFactory( 2189 stream_name, 2190 schema_loader, 2191 retriever, 2192 self._message_repository, 2193 ), 2194 stream_slicer, 2195 slice_limit=self._limit_slices_fetched, 2196 ), 2197 name=stream_name, 2198 json_schema=schema_loader.get_json_schema, 2199 primary_key=get_primary_key_from_stream(primary_key), 2200 cursor_field=( 2201 concurrent_cursor.cursor_field 2202 if hasattr(concurrent_cursor, "cursor_field") 2203 else None 2204 ), 2205 logger=logging.getLogger(f"airbyte.{stream_name}"), 2206 cursor=concurrent_cursor, 2207 supports_file_transfer=hasattr(model, "file_uploader") and bool(model.file_uploader), 2208 )
def
create_default_error_handler( self, model: airbyte_cdk.sources.declarative.models.declarative_component_schema.DefaultErrorHandler, config: Mapping[str, Any], **kwargs: Any) -> airbyte_cdk.sources.declarative.requesters.error_handlers.DefaultErrorHandler:
2350 def create_default_error_handler( 2351 self, model: DefaultErrorHandlerModel, config: Config, **kwargs: Any 2352 ) -> DefaultErrorHandler: 2353 backoff_strategies = [] 2354 if model.backoff_strategies: 2355 for backoff_strategy_model in model.backoff_strategies: 2356 backoff_strategies.append( 2357 self._create_component_from_model(model=backoff_strategy_model, config=config) 2358 ) 2359 2360 response_filters = [] 2361 if model.response_filters: 2362 for response_filter_model in model.response_filters: 2363 response_filters.append( 2364 self._create_component_from_model(model=response_filter_model, config=config) 2365 ) 2366 response_filters.append( 2367 HttpResponseFilter(config=config, parameters=model.parameters or {}) 2368 ) 2369 2370 return DefaultErrorHandler( 2371 backoff_strategies=backoff_strategies, 2372 max_retries=model.max_retries, 2373 response_filters=response_filters, 2374 config=config, 2375 parameters=model.parameters or {}, 2376 )
def
create_default_paginator( self, model: airbyte_cdk.sources.declarative.models.declarative_component_schema.DefaultPaginator, config: Mapping[str, Any], *, url_base: str, extractor_model: Union[airbyte_cdk.sources.declarative.models.declarative_component_schema.DpathExtractor, airbyte_cdk.sources.declarative.models.declarative_component_schema.CustomRecordExtractor, NoneType] = None, decoder: Optional[airbyte_cdk.Decoder] = None, cursor_used_for_stop_condition: Optional[airbyte_cdk.Cursor] = None) -> Union[airbyte_cdk.DefaultPaginator, airbyte_cdk.sources.declarative.requesters.paginators.PaginatorTestReadDecorator]:
2378 def create_default_paginator( 2379 self, 2380 model: DefaultPaginatorModel, 2381 config: Config, 2382 *, 2383 url_base: str, 2384 extractor_model: Optional[Union[CustomRecordExtractorModel, DpathExtractorModel]] = None, 2385 decoder: Optional[Decoder] = None, 2386 cursor_used_for_stop_condition: Optional[Cursor] = None, 2387 ) -> Union[DefaultPaginator, PaginatorTestReadDecorator]: 2388 if decoder: 2389 if self._is_supported_decoder_for_pagination(decoder): 2390 decoder_to_use = PaginationDecoderDecorator(decoder=decoder) 2391 else: 2392 raise ValueError(self._UNSUPPORTED_DECODER_ERROR.format(decoder_type=type(decoder))) 2393 else: 2394 decoder_to_use = PaginationDecoderDecorator(decoder=JsonDecoder(parameters={})) 2395 page_size_option = ( 2396 self._create_component_from_model(model=model.page_size_option, config=config) 2397 if model.page_size_option 2398 else None 2399 ) 2400 page_token_option = ( 2401 self._create_component_from_model(model=model.page_token_option, config=config) 2402 if model.page_token_option 2403 else None 2404 ) 2405 pagination_strategy = self._create_component_from_model( 2406 model=model.pagination_strategy, 2407 config=config, 2408 decoder=decoder_to_use, 2409 extractor_model=extractor_model, 2410 ) 2411 if cursor_used_for_stop_condition: 2412 pagination_strategy = StopConditionPaginationStrategyDecorator( 2413 pagination_strategy, CursorStopCondition(cursor_used_for_stop_condition) 2414 ) 2415 paginator = DefaultPaginator( 2416 decoder=decoder_to_use, 2417 page_size_option=page_size_option, 2418 page_token_option=page_token_option, 2419 pagination_strategy=pagination_strategy, 2420 url_base=url_base, 2421 config=config, 2422 parameters=model.parameters or {}, 2423 ) 2424 if self._limit_pages_fetched_per_slice: 2425 return PaginatorTestReadDecorator(paginator, self._limit_pages_fetched_per_slice) 2426 return paginator
def
create_dpath_extractor( self, model: airbyte_cdk.sources.declarative.models.declarative_component_schema.DpathExtractor, config: Mapping[str, Any], decoder: Optional[airbyte_cdk.Decoder] = None, **kwargs: Any) -> airbyte_cdk.DpathExtractor:
2428 def create_dpath_extractor( 2429 self, 2430 model: DpathExtractorModel, 2431 config: Config, 2432 decoder: Optional[Decoder] = None, 2433 **kwargs: Any, 2434 ) -> DpathExtractor: 2435 if decoder: 2436 decoder_to_use = decoder 2437 else: 2438 decoder_to_use = JsonDecoder(parameters={}) 2439 model_field_path: List[Union[InterpolatedString, str]] = [x for x in model.field_path] 2440 2441 record_expander = None 2442 if model.record_expander: 2443 record_expander = self._create_component_from_model( 2444 model=model.record_expander, 2445 config=config, 2446 ) 2447 2448 return DpathExtractor( 2449 decoder=decoder_to_use, 2450 field_path=model_field_path, 2451 config=config, 2452 parameters=model.parameters or {}, 2453 record_expander=record_expander, 2454 )
def
create_record_expander( self, model: airbyte_cdk.sources.declarative.models.declarative_component_schema.RecordExpander, config: Mapping[str, Any], **kwargs: Any) -> airbyte_cdk.sources.declarative.expanders.RecordExpander:
2456 def create_record_expander( 2457 self, 2458 model: RecordExpanderModel, 2459 config: Config, 2460 **kwargs: Any, 2461 ) -> RecordExpander: 2462 return RecordExpander( 2463 expand_records_from_field=model.expand_records_from_field, 2464 config=config, 2465 parameters=model.parameters or {}, 2466 remain_original_record=model.remain_original_record or False, 2467 on_no_records=OnNoRecords(model.on_no_records.value) 2468 if model.on_no_records 2469 else OnNoRecords.skip, 2470 )
@staticmethod
def
create_response_to_file_extractor( model: airbyte_cdk.sources.declarative.models.declarative_component_schema.ResponseToFileExtractor, **kwargs: Any) -> airbyte_cdk.sources.declarative.extractors.ResponseToFileExtractor:
2472 @staticmethod 2473 def create_response_to_file_extractor( 2474 model: ResponseToFileExtractorModel, 2475 **kwargs: Any, 2476 ) -> ResponseToFileExtractor: 2477 return ResponseToFileExtractor( 2478 parameters=model.parameters or {}, 2479 preserve_na_values=model.preserve_na_values or False, 2480 )
@staticmethod
def
create_exponential_backoff_strategy( model: airbyte_cdk.sources.declarative.models.declarative_component_schema.ExponentialBackoffStrategy, config: Mapping[str, Any]) -> airbyte_cdk.sources.declarative.requesters.error_handlers.backoff_strategies.ExponentialBackoffStrategy:
2482 @staticmethod 2483 def create_exponential_backoff_strategy( 2484 model: ExponentialBackoffStrategyModel, config: Config 2485 ) -> ExponentialBackoffStrategy: 2486 ModelToComponentFactory._validate_jitter_range(model.jitter_range_in_seconds) 2487 return ExponentialBackoffStrategy( 2488 factor=model.factor or 5, 2489 jitter_range_in_seconds=model.jitter_range_in_seconds, 2490 parameters=model.parameters or {}, 2491 config=config, 2492 )
@staticmethod
def
create_group_by_key( model: airbyte_cdk.sources.declarative.models.declarative_component_schema.GroupByKeyMergeStrategy, config: Mapping[str, Any]) -> airbyte_cdk.sources.declarative.requesters.query_properties.strategies.GroupByKey:
def
create_http_requester( self, model: airbyte_cdk.sources.declarative.models.declarative_component_schema.HttpRequester, config: Mapping[str, Any], decoder: airbyte_cdk.Decoder = JsonDecoder(), query_properties_key: Optional[str] = None, use_cache: Optional[bool] = None, *, name: str) -> airbyte_cdk.HttpRequester:
2498 def create_http_requester( 2499 self, 2500 model: HttpRequesterModel, 2501 config: Config, 2502 decoder: Decoder = JsonDecoder(parameters={}), 2503 query_properties_key: Optional[str] = None, 2504 use_cache: Optional[bool] = None, 2505 *, 2506 name: str, 2507 ) -> HttpRequester: 2508 authenticator = ( 2509 self._create_component_from_model( 2510 model=model.authenticator, 2511 config=config, 2512 url_base=model.url or model.url_base, 2513 name=name, 2514 decoder=decoder, 2515 ) 2516 if model.authenticator 2517 else None 2518 ) 2519 error_handler = ( 2520 self._create_component_from_model(model=model.error_handler, config=config) 2521 if model.error_handler 2522 else DefaultErrorHandler( 2523 backoff_strategies=[], 2524 response_filters=[], 2525 config=config, 2526 parameters=model.parameters or {}, 2527 ) 2528 ) 2529 2530 api_budget = self._api_budget 2531 2532 request_options_provider = InterpolatedRequestOptionsProvider( 2533 request_body=model.request_body, 2534 request_body_data=model.request_body_data, 2535 request_body_json=model.request_body_json, 2536 request_headers=model.request_headers, 2537 request_parameters=model.request_parameters, # type: ignore # QueryProperties have been removed in `create_simple_retriever` 2538 query_properties_key=query_properties_key, 2539 config=config, 2540 parameters=model.parameters or {}, 2541 ) 2542 2543 assert model.use_cache is not None # for mypy 2544 assert model.http_method is not None # for mypy 2545 2546 should_use_cache = (model.use_cache or bool(use_cache)) and not self._disable_cache 2547 2548 return HttpRequester( 2549 name=name, 2550 url=model.url, 2551 url_base=model.url_base, 2552 path=model.path, 2553 authenticator=authenticator, 2554 error_handler=error_handler, 2555 api_budget=api_budget, 2556 http_method=HttpMethod[model.http_method.value], 2557 request_options_provider=request_options_provider, 2558 config=config, 2559 disable_retries=self._disable_retries, 2560 parameters=model.parameters or {}, 2561 message_repository=self._message_repository, 2562 use_cache=should_use_cache, 2563 decoder=decoder, 2564 stream_response=decoder.is_stream_response() if decoder else False, 2565 )
@staticmethod
def
create_http_response_filter( model: airbyte_cdk.sources.declarative.models.declarative_component_schema.HttpResponseFilter, config: Mapping[str, Any], **kwargs: Any) -> airbyte_cdk.sources.declarative.requesters.error_handlers.HttpResponseFilter:
2567 @staticmethod 2568 def create_http_response_filter( 2569 model: HttpResponseFilterModel, config: Config, **kwargs: Any 2570 ) -> HttpResponseFilter: 2571 if model.action: 2572 action = ResponseAction(model.action.value) 2573 else: 2574 action = None 2575 2576 failure_type = FailureType(model.failure_type.value) if model.failure_type else None 2577 2578 http_codes = ( 2579 set(model.http_codes) if model.http_codes else set() 2580 ) # JSON schema notation has no set data type. The schema enforces an array of unique elements 2581 2582 return HttpResponseFilter( 2583 action=action, 2584 failure_type=failure_type, 2585 error_message=model.error_message or "", 2586 error_message_contains=model.error_message_contains or "", 2587 http_codes=http_codes, 2588 predicate=model.predicate or "", 2589 config=config, 2590 parameters=model.parameters or {}, 2591 )
@staticmethod
def
create_inline_schema_loader( model: airbyte_cdk.sources.declarative.models.declarative_component_schema.InlineSchemaLoader, config: Mapping[str, Any], **kwargs: Any) -> airbyte_cdk.sources.declarative.schema.InlineSchemaLoader:
def
create_complex_field_type( self, model: airbyte_cdk.sources.declarative.models.declarative_component_schema.ComplexFieldType, config: Mapping[str, Any], **kwargs: Any) -> airbyte_cdk.sources.declarative.schema.ComplexFieldType:
2599 def create_complex_field_type( 2600 self, model: ComplexFieldTypeModel, config: Config, **kwargs: Any 2601 ) -> ComplexFieldType: 2602 items = ( 2603 self._create_component_from_model(model=model.items, config=config) 2604 if isinstance(model.items, ComplexFieldTypeModel) 2605 else model.items 2606 ) 2607 2608 return ComplexFieldType(field_type=model.field_type, items=items)
def
create_types_map( self, model: airbyte_cdk.sources.declarative.models.declarative_component_schema.TypesMap, config: Mapping[str, Any], **kwargs: Any) -> airbyte_cdk.sources.declarative.schema.TypesMap:
2610 def create_types_map(self, model: TypesMapModel, config: Config, **kwargs: Any) -> TypesMap: 2611 target_type = ( 2612 self._create_component_from_model(model=model.target_type, config=config) 2613 if isinstance(model.target_type, ComplexFieldTypeModel) 2614 else model.target_type 2615 ) 2616 2617 return TypesMap( 2618 target_type=target_type, 2619 current_type=model.current_type, 2620 condition=model.condition if model.condition is not None else "True", 2621 )
def
create_schema_type_identifier( self, model: airbyte_cdk.sources.declarative.models.declarative_component_schema.SchemaTypeIdentifier, config: Mapping[str, Any], **kwargs: Any) -> airbyte_cdk.sources.declarative.schema.SchemaTypeIdentifier:
2623 def create_schema_type_identifier( 2624 self, model: SchemaTypeIdentifierModel, config: Config, **kwargs: Any 2625 ) -> SchemaTypeIdentifier: 2626 types_mapping = [] 2627 if model.types_mapping: 2628 types_mapping.extend( 2629 [ 2630 self._create_component_from_model(types_map, config=config) 2631 for types_map in model.types_mapping 2632 ] 2633 ) 2634 model_schema_pointer: List[Union[InterpolatedString, str]] = ( 2635 [x for x in model.schema_pointer] if model.schema_pointer else [] 2636 ) 2637 model_key_pointer: List[Union[InterpolatedString, str]] = [x for x in model.key_pointer] 2638 model_type_pointer: Optional[List[Union[InterpolatedString, str]]] = ( 2639 [x for x in model.type_pointer] if model.type_pointer else None 2640 ) 2641 2642 return SchemaTypeIdentifier( 2643 schema_pointer=model_schema_pointer, 2644 key_pointer=model_key_pointer, 2645 type_pointer=model_type_pointer, 2646 types_mapping=types_mapping, 2647 parameters=model.parameters or {}, 2648 )
def
create_dynamic_schema_loader( self, model: airbyte_cdk.sources.declarative.models.declarative_component_schema.DynamicSchemaLoader, config: Mapping[str, Any], **kwargs: Any) -> airbyte_cdk.sources.declarative.schema.DynamicSchemaLoader:
2650 def create_dynamic_schema_loader( 2651 self, model: DynamicSchemaLoaderModel, config: Config, **kwargs: Any 2652 ) -> DynamicSchemaLoader: 2653 schema_transformations = [] 2654 if model.schema_transformations: 2655 for transformation_model in model.schema_transformations: 2656 schema_transformations.append( 2657 self._create_component_from_model(model=transformation_model, config=config) 2658 ) 2659 name = "dynamic_properties" 2660 retriever = self._create_component_from_model( 2661 model=model.retriever, 2662 config=config, 2663 name=name, 2664 primary_key=None, 2665 partition_router=self._build_stream_slicer_from_partition_router( 2666 model.retriever, config 2667 ), 2668 transformations=[], 2669 use_cache=True, 2670 log_formatter=( 2671 lambda response: format_http_message( 2672 response, 2673 f"Schema loader '{name}' request", 2674 f"Request performed in order to extract schema.", 2675 name, 2676 is_auxiliary=True, 2677 ) 2678 ), 2679 ) 2680 schema_type_identifier = self._create_component_from_model( 2681 model.schema_type_identifier, config=config, parameters=model.parameters or {} 2682 ) 2683 schema_filter = ( 2684 self._create_component_from_model( 2685 model.schema_filter, config=config, parameters=model.parameters or {} 2686 ) 2687 if model.schema_filter is not None 2688 else None 2689 ) 2690 2691 return DynamicSchemaLoader( 2692 retriever=retriever, 2693 config=config, 2694 schema_transformations=schema_transformations, 2695 schema_filter=schema_filter, 2696 schema_type_identifier=schema_type_identifier, 2697 parameters=model.parameters or {}, 2698 )
@staticmethod
def
create_json_decoder( model: airbyte_cdk.sources.declarative.models.declarative_component_schema.JsonDecoder, config: Mapping[str, Any], **kwargs: Any) -> airbyte_cdk.Decoder:
def
create_csv_decoder( self, model: airbyte_cdk.sources.declarative.models.declarative_component_schema.CsvDecoder, config: Mapping[str, Any], **kwargs: Any) -> airbyte_cdk.Decoder:
def
create_jsonl_decoder( self, model: airbyte_cdk.sources.declarative.models.declarative_component_schema.JsonlDecoder, config: Mapping[str, Any], **kwargs: Any) -> airbyte_cdk.Decoder:
def
create_json_items_decoder( self, model: airbyte_cdk.sources.declarative.models.declarative_component_schema.JsonItemsDecoder, config: Mapping[str, Any], **kwargs: Any) -> airbyte_cdk.Decoder:
2718 def create_json_items_decoder( 2719 self, model: JsonItemsDecoderModel, config: Config, **kwargs: Any 2720 ) -> Decoder: 2721 return CompositeRawDecoder( 2722 parser=ModelToComponentFactory._get_parser(model, config), 2723 stream_response=False if self._emit_connector_builder_messages else True, 2724 )
def
create_gzip_decoder( self, model: airbyte_cdk.sources.declarative.models.declarative_component_schema.GzipDecoder, config: Mapping[str, Any], **kwargs: Any) -> airbyte_cdk.Decoder:
2726 def create_gzip_decoder( 2727 self, model: GzipDecoderModel, config: Config, **kwargs: Any 2728 ) -> Decoder: 2729 _compressed_response_types = { 2730 "gzip", 2731 "x-gzip", 2732 "gzip, deflate", 2733 "x-gzip, deflate", 2734 "application/zip", 2735 "application/gzip", 2736 "application/x-gzip", 2737 "application/x-zip-compressed", 2738 } 2739 2740 gzip_parser: GzipParser = ModelToComponentFactory._get_parser(model, config) # type: ignore # based on the model, we know this will be a GzipParser 2741 2742 if self._emit_connector_builder_messages: 2743 # This is very surprising but if the response is not streamed, 2744 # CompositeRawDecoder calls response.content and the requests library actually uncompress the data as opposed to response.raw, 2745 # which uses urllib3 directly and does not uncompress the data. 2746 return CompositeRawDecoder(gzip_parser.inner_parser, False) 2747 2748 return CompositeRawDecoder.by_headers( 2749 [({"Content-Encoding", "Content-Type"}, _compressed_response_types, gzip_parser)], 2750 stream_response=True, 2751 fallback_parser=gzip_parser.inner_parser, 2752 )
@staticmethod
def
create_iterable_decoder( model: airbyte_cdk.sources.declarative.models.declarative_component_schema.IterableDecoder, config: Mapping[str, Any], **kwargs: Any) -> airbyte_cdk.sources.declarative.decoders.IterableDecoder:
@staticmethod
def
create_xml_decoder( model: airbyte_cdk.sources.declarative.models.declarative_component_schema.XmlDecoder, config: Mapping[str, Any], **kwargs: Any) -> airbyte_cdk.sources.declarative.decoders.XmlDecoder:
def
create_zipfile_decoder( self, model: airbyte_cdk.sources.declarative.models.declarative_component_schema.ZipfileDecoder, config: Mapping[str, Any], **kwargs: Any) -> airbyte_cdk.sources.declarative.decoders.ZipfileDecoder:
@staticmethod
def
create_json_file_schema_loader( model: airbyte_cdk.sources.declarative.models.declarative_component_schema.JsonFileSchemaLoader, config: Mapping[str, Any], **kwargs: Any) -> airbyte_cdk.JsonFileSchemaLoader:
def
create_jwt_authenticator( self, model: airbyte_cdk.sources.declarative.models.declarative_component_schema.JwtAuthenticator, config: Mapping[str, Any], **kwargs: Any) -> airbyte_cdk.sources.declarative.auth.JwtAuthenticator:
2806 def create_jwt_authenticator( 2807 self, model: JwtAuthenticatorModel, config: Config, **kwargs: Any 2808 ) -> JwtAuthenticator: 2809 jwt_headers = model.jwt_headers or JwtHeadersModel(kid=None, typ="JWT", cty=None) 2810 jwt_payload = model.jwt_payload or JwtPayloadModel(iss=None, sub=None, aud=None) 2811 request_option = ( 2812 self._create_component_from_model(model.request_option, config) 2813 if model.request_option 2814 else None 2815 ) 2816 return JwtAuthenticator( 2817 config=config, 2818 parameters=model.parameters or {}, 2819 algorithm=JwtAlgorithm(model.algorithm.value), 2820 secret_key=model.secret_key, 2821 base64_encode_secret_key=model.base64_encode_secret_key, 2822 token_duration=model.token_duration, 2823 header_prefix=model.header_prefix, 2824 kid=jwt_headers.kid, 2825 typ=jwt_headers.typ, 2826 cty=jwt_headers.cty, 2827 iss=jwt_payload.iss, 2828 sub=jwt_payload.sub, 2829 aud=jwt_payload.aud, 2830 additional_jwt_headers=model.additional_jwt_headers, 2831 additional_jwt_payload=model.additional_jwt_payload, 2832 passphrase=model.passphrase, 2833 request_option=request_option, 2834 )
def
create_list_partition_router( self, model: airbyte_cdk.sources.declarative.models.declarative_component_schema.ListPartitionRouter, config: Mapping[str, Any], **kwargs: Any) -> airbyte_cdk.sources.declarative.partition_routers.ListPartitionRouter:
2836 def create_list_partition_router( 2837 self, model: ListPartitionRouterModel, config: Config, **kwargs: Any 2838 ) -> ListPartitionRouter: 2839 request_option = ( 2840 self._create_component_from_model(model.request_option, config) 2841 if model.request_option 2842 else None 2843 ) 2844 return ListPartitionRouter( 2845 cursor_field=model.cursor_field, 2846 request_option=request_option, 2847 values=model.values, 2848 config=config, 2849 parameters=model.parameters or {}, 2850 )
@staticmethod
def
create_min_max_datetime( model: airbyte_cdk.sources.declarative.models.declarative_component_schema.MinMaxDatetime, config: Mapping[str, Any], **kwargs: Any) -> airbyte_cdk.MinMaxDatetime:
2852 @staticmethod 2853 def create_min_max_datetime( 2854 model: MinMaxDatetimeModel, config: Config, **kwargs: Any 2855 ) -> MinMaxDatetime: 2856 return MinMaxDatetime( 2857 datetime=model.datetime, 2858 datetime_format=model.datetime_format or "", 2859 max_datetime=model.max_datetime or "", 2860 min_datetime=model.min_datetime or "", 2861 parameters=model.parameters or {}, 2862 )
@staticmethod
def
create_no_auth( model: airbyte_cdk.sources.declarative.models.declarative_component_schema.NoAuth, config: Mapping[str, Any], **kwargs: Any) -> airbyte_cdk.NoAuth:
@staticmethod
def
create_no_pagination( model: airbyte_cdk.sources.declarative.models.declarative_component_schema.NoPagination, config: Mapping[str, Any], **kwargs: Any) -> airbyte_cdk.sources.declarative.requesters.paginators.NoPagination:
def
create_oauth_authenticator( self, model: airbyte_cdk.sources.declarative.models.declarative_component_schema.OAuthAuthenticator, config: Mapping[str, Any], **kwargs: Any) -> airbyte_cdk.DeclarativeOauth2Authenticator:
2874 def create_oauth_authenticator( 2875 self, model: OAuthAuthenticatorModel, config: Config, **kwargs: Any 2876 ) -> DeclarativeOauth2Authenticator: 2877 profile_assertion = ( 2878 self._create_component_from_model(model.profile_assertion, config=config) 2879 if model.profile_assertion 2880 else None 2881 ) 2882 2883 refresh_token_error_status_codes, refresh_token_error_key, refresh_token_error_values = ( 2884 self._get_refresh_token_error_information(model) 2885 ) 2886 if model.refresh_token_updater: 2887 # ignore type error because fixing it would have a lot of dependencies, revisit later 2888 return DeclarativeSingleUseRefreshTokenOauth2Authenticator( # type: ignore 2889 config, 2890 InterpolatedString.create( 2891 model.token_refresh_endpoint, # type: ignore 2892 parameters=model.parameters or {}, 2893 ).eval(config), 2894 access_token_name=InterpolatedString.create( 2895 model.access_token_name or "access_token", parameters=model.parameters or {} 2896 ).eval(config), 2897 refresh_token_name=model.refresh_token_updater.refresh_token_name, 2898 expires_in_name=InterpolatedString.create( 2899 model.expires_in_name or "expires_in", parameters=model.parameters or {} 2900 ).eval(config), 2901 client_id_name=InterpolatedString.create( 2902 model.client_id_name or "client_id", parameters=model.parameters or {} 2903 ).eval(config), 2904 client_id=InterpolatedString.create( 2905 model.client_id, parameters=model.parameters or {} 2906 ).eval(config) 2907 if model.client_id 2908 else model.client_id, 2909 client_secret_name=InterpolatedString.create( 2910 model.client_secret_name or "client_secret", parameters=model.parameters or {} 2911 ).eval(config), 2912 client_secret=InterpolatedString.create( 2913 model.client_secret, parameters=model.parameters or {} 2914 ).eval(config) 2915 if model.client_secret 2916 else model.client_secret, 2917 access_token_config_path=model.refresh_token_updater.access_token_config_path, 2918 refresh_token_config_path=model.refresh_token_updater.refresh_token_config_path, 2919 token_expiry_date_config_path=model.refresh_token_updater.token_expiry_date_config_path, 2920 grant_type_name=InterpolatedString.create( 2921 model.grant_type_name or "grant_type", parameters=model.parameters or {} 2922 ).eval(config), 2923 grant_type=InterpolatedString.create( 2924 model.grant_type or "refresh_token", parameters=model.parameters or {} 2925 ).eval(config), 2926 refresh_request_body=InterpolatedMapping( 2927 model.refresh_request_body or {}, parameters=model.parameters or {} 2928 ).eval(config), 2929 refresh_request_headers=InterpolatedMapping( 2930 model.refresh_request_headers or {}, parameters=model.parameters or {} 2931 ).eval(config), 2932 send_refresh_request_as_query_params=bool( 2933 model.send_refresh_request_as_query_params 2934 ), 2935 scopes=model.scopes, 2936 token_expiry_date_format=model.token_expiry_date_format, 2937 token_expiry_is_time_of_expiration=bool(model.token_expiry_date_format), 2938 message_repository=self._message_repository, 2939 refresh_token_error_status_codes=refresh_token_error_status_codes, 2940 refresh_token_error_key=refresh_token_error_key, 2941 refresh_token_error_values=refresh_token_error_values, 2942 ) 2943 # ignore type error because fixing it would have a lot of dependencies, revisit later 2944 return DeclarativeOauth2Authenticator( # type: ignore 2945 access_token_name=model.access_token_name or "access_token", 2946 access_token_value=model.access_token_value, 2947 client_id_name=model.client_id_name or "client_id", 2948 client_id=model.client_id, 2949 client_secret_name=model.client_secret_name or "client_secret", 2950 client_secret=model.client_secret, 2951 expires_in_name=model.expires_in_name or "expires_in", 2952 grant_type_name=model.grant_type_name or "grant_type", 2953 grant_type=model.grant_type or "refresh_token", 2954 refresh_request_body=model.refresh_request_body, 2955 refresh_request_headers=model.refresh_request_headers, 2956 send_refresh_request_as_query_params=bool(model.send_refresh_request_as_query_params), 2957 refresh_token_name=model.refresh_token_name or "refresh_token", 2958 refresh_token=model.refresh_token, 2959 scopes=model.scopes, 2960 token_expiry_date=model.token_expiry_date, 2961 token_expiry_date_format=model.token_expiry_date_format, 2962 token_expiry_is_time_of_expiration=bool(model.token_expiry_date_format), 2963 token_refresh_endpoint=model.token_refresh_endpoint, 2964 config=config, 2965 parameters=model.parameters or {}, 2966 message_repository=self._message_repository, 2967 profile_assertion=profile_assertion, 2968 use_profile_assertion=model.use_profile_assertion, 2969 refresh_token_error_status_codes=refresh_token_error_status_codes, 2970 refresh_token_error_key=refresh_token_error_key, 2971 refresh_token_error_values=refresh_token_error_values, 2972 )
def
create_offset_increment( self, model: airbyte_cdk.sources.declarative.models.declarative_component_schema.OffsetIncrement, config: Mapping[str, Any], decoder: airbyte_cdk.Decoder, extractor_model: Union[airbyte_cdk.sources.declarative.models.declarative_component_schema.DpathExtractor, airbyte_cdk.sources.declarative.models.declarative_component_schema.CustomRecordExtractor, NoneType] = None, **kwargs: Any) -> airbyte_cdk.OffsetIncrement:
3022 def create_offset_increment( 3023 self, 3024 model: OffsetIncrementModel, 3025 config: Config, 3026 decoder: Decoder, 3027 extractor_model: Optional[Union[CustomRecordExtractorModel, DpathExtractorModel]] = None, 3028 **kwargs: Any, 3029 ) -> OffsetIncrement: 3030 if isinstance(decoder, PaginationDecoderDecorator): 3031 inner_decoder = decoder.decoder 3032 else: 3033 inner_decoder = decoder 3034 decoder = PaginationDecoderDecorator(decoder=decoder) 3035 3036 if self._is_supported_decoder_for_pagination(inner_decoder): 3037 decoder_to_use = decoder 3038 else: 3039 raise ValueError( 3040 self._UNSUPPORTED_DECODER_ERROR.format(decoder_type=type(inner_decoder)) 3041 ) 3042 3043 # Ideally we would instantiate the runtime extractor from highest most level (in this case the SimpleRetriever) 3044 # so that it can be shared by OffSetIncrement and RecordSelector. However, due to how we instantiate the 3045 # decoder with various decorators here, but not in create_record_selector, it is simpler to retain existing 3046 # behavior by having two separate extractors with identical behavior since they use the same extractor model. 3047 # When we have more time to investigate we can look into reusing the same component. 3048 extractor = ( 3049 self._create_component_from_model( 3050 model=extractor_model, config=config, decoder=decoder_to_use 3051 ) 3052 if extractor_model 3053 else None 3054 ) 3055 3056 # Pydantic v1 Union type coercion can convert int to string depending on Union order. 3057 # If page_size is a string that represents an integer (not an interpolation), convert it back. 3058 page_size = model.page_size 3059 if isinstance(page_size, str) and page_size.isdigit(): 3060 page_size = int(page_size) 3061 3062 return OffsetIncrement( 3063 page_size=page_size, 3064 config=config, 3065 decoder=decoder_to_use, 3066 extractor=extractor, 3067 inject_on_first_request=model.inject_on_first_request or False, 3068 parameters=model.parameters or {}, 3069 )
def
create_page_increment( self, model: airbyte_cdk.sources.declarative.models.declarative_component_schema.PageIncrement, config: Mapping[str, Any], decoder: Optional[airbyte_cdk.Decoder] = None, extractor_model: Union[airbyte_cdk.sources.declarative.models.declarative_component_schema.DpathExtractor, airbyte_cdk.sources.declarative.models.declarative_component_schema.CustomRecordExtractor, NoneType] = None, **kwargs: Any) -> airbyte_cdk.PageIncrement:
3071 def create_page_increment( 3072 self, 3073 model: PageIncrementModel, 3074 config: Config, 3075 decoder: Optional[Decoder] = None, 3076 extractor_model: Optional[Union[CustomRecordExtractorModel, DpathExtractorModel]] = None, 3077 **kwargs: Any, 3078 ) -> PageIncrement: 3079 # Like OffsetIncrement, we instantiate a separate extractor with identical behavior to the 3080 # RecordSelector's so the strategy can count the raw records in the response. This ensures 3081 # pagination is driven by the API's page size, not the post-filter record count. 3082 extractor = ( 3083 self._create_component_from_model(model=extractor_model, config=config, decoder=decoder) 3084 if extractor_model 3085 else None 3086 ) 3087 3088 # Pydantic v1 Union type coercion can convert int to string depending on Union order. 3089 # If page_size is a string that represents an integer (not an interpolation), convert it back. 3090 page_size = model.page_size 3091 if isinstance(page_size, str) and page_size.isdigit(): 3092 page_size = int(page_size) 3093 3094 return PageIncrement( 3095 page_size=page_size, 3096 config=config, 3097 start_from_page=model.start_from_page or 0, 3098 inject_on_first_request=model.inject_on_first_request or False, 3099 extractor=extractor, 3100 parameters=model.parameters or {}, 3101 )
def
create_parent_stream_config( self, model: airbyte_cdk.sources.declarative.models.declarative_component_schema.ParentStreamConfig, config: Mapping[str, Any], *, stream_name: str, **kwargs: Any) -> airbyte_cdk.ParentStreamConfig:
3103 def create_parent_stream_config( 3104 self, model: ParentStreamConfigModel, config: Config, *, stream_name: str, **kwargs: Any 3105 ) -> ParentStreamConfig: 3106 declarative_stream = self._create_component_from_model( 3107 model.stream, 3108 config=config, 3109 is_parent=True, 3110 **kwargs, 3111 ) 3112 request_option = ( 3113 self._create_component_from_model(model.request_option, config=config) 3114 if model.request_option 3115 else None 3116 ) 3117 3118 if model.lazy_read_pointer and any("*" in pointer for pointer in model.lazy_read_pointer): 3119 raise ValueError( 3120 "The '*' wildcard in 'lazy_read_pointer' is not supported — only direct paths are allowed." 3121 ) 3122 3123 model_lazy_read_pointer: List[Union[InterpolatedString, str]] = ( 3124 [x for x in model.lazy_read_pointer] if model.lazy_read_pointer else [] 3125 ) 3126 3127 return ParentStreamConfig( 3128 parent_key=model.parent_key, 3129 request_option=request_option, 3130 stream=declarative_stream, 3131 partition_field=model.partition_field, 3132 config=config, 3133 incremental_dependency=model.incremental_dependency or False, 3134 parameters=model.parameters or {}, 3135 extra_fields=model.extra_fields, 3136 lazy_read_pointer=model_lazy_read_pointer, 3137 )
def
create_properties_from_endpoint( self, model: airbyte_cdk.sources.declarative.models.declarative_component_schema.PropertiesFromEndpoint, config: Mapping[str, Any], **kwargs: Any) -> airbyte_cdk.sources.declarative.requesters.query_properties.PropertiesFromEndpoint:
3139 def create_properties_from_endpoint( 3140 self, model: PropertiesFromEndpointModel, config: Config, **kwargs: Any 3141 ) -> PropertiesFromEndpoint: 3142 retriever = self._create_component_from_model( 3143 model=model.retriever, 3144 config=config, 3145 name="dynamic_properties", 3146 primary_key=None, 3147 stream_slicer=None, 3148 transformations=[], 3149 use_cache=True, # Enable caching on the HttpRequester/HttpClient because the properties endpoint will be called for every slice being processed, and it is highly unlikely for the response to different 3150 ) 3151 return PropertiesFromEndpoint( 3152 property_field_path=model.property_field_path, 3153 retriever=retriever, 3154 config=config, 3155 parameters=model.parameters or {}, 3156 )
def
create_property_chunking( self, model: airbyte_cdk.sources.declarative.models.declarative_component_schema.PropertyChunking, config: Mapping[str, Any], **kwargs: Any) -> airbyte_cdk.sources.declarative.requesters.query_properties.PropertyChunking:
3158 def create_property_chunking( 3159 self, model: PropertyChunkingModel, config: Config, **kwargs: Any 3160 ) -> PropertyChunking: 3161 record_merge_strategy = ( 3162 self._create_component_from_model( 3163 model=model.record_merge_strategy, config=config, **kwargs 3164 ) 3165 if model.record_merge_strategy 3166 else None 3167 ) 3168 3169 property_limit_type: PropertyLimitType 3170 match model.property_limit_type: 3171 case PropertyLimitTypeModel.property_count: 3172 property_limit_type = PropertyLimitType.property_count 3173 case PropertyLimitTypeModel.characters: 3174 property_limit_type = PropertyLimitType.characters 3175 case _: 3176 raise ValueError(f"Invalid PropertyLimitType {property_limit_type}") 3177 3178 return PropertyChunking( 3179 property_limit_type=property_limit_type, 3180 property_limit=model.property_limit, 3181 record_merge_strategy=record_merge_strategy, 3182 config=config, 3183 parameters=model.parameters or {}, 3184 )
def
create_query_properties( self, model: airbyte_cdk.sources.declarative.models.declarative_component_schema.QueryProperties, config: Mapping[str, Any], *, stream_name: str, **kwargs: Any) -> airbyte_cdk.sources.declarative.requesters.query_properties.QueryProperties:
3186 def create_query_properties( 3187 self, model: QueryPropertiesModel, config: Config, *, stream_name: str, **kwargs: Any 3188 ) -> QueryProperties: 3189 if isinstance(model.property_list, list): 3190 property_list = model.property_list 3191 else: 3192 property_list = self._create_component_from_model( 3193 model=model.property_list, config=config, **kwargs 3194 ) 3195 3196 property_chunking = ( 3197 self._create_component_from_model( 3198 model=model.property_chunking, config=config, **kwargs 3199 ) 3200 if model.property_chunking 3201 else None 3202 ) 3203 3204 property_selector = ( 3205 self._create_component_from_model( 3206 model=model.property_selector, config=config, stream_name=stream_name, **kwargs 3207 ) 3208 if model.property_selector 3209 else None 3210 ) 3211 3212 return QueryProperties( 3213 property_list=property_list, 3214 always_include_properties=model.always_include_properties, 3215 property_chunking=property_chunking, 3216 property_selector=property_selector, 3217 config=config, 3218 parameters=model.parameters or {}, 3219 )
def
create_json_schema_property_selector( self, model: airbyte_cdk.sources.declarative.models.declarative_component_schema.JsonSchemaPropertySelector, config: Mapping[str, Any], *, stream_name: str, **kwargs: Any) -> airbyte_cdk.sources.declarative.requesters.query_properties.property_selector.JsonSchemaPropertySelector:
3221 def create_json_schema_property_selector( 3222 self, 3223 model: JsonSchemaPropertySelectorModel, 3224 config: Config, 3225 *, 3226 stream_name: str, 3227 **kwargs: Any, 3228 ) -> JsonSchemaPropertySelector: 3229 configured_stream = self._stream_name_to_configured_stream.get(stream_name) 3230 3231 transformations = [] 3232 if model.transformations: 3233 for transformation_model in model.transformations: 3234 transformations.append( 3235 self._create_component_from_model(model=transformation_model, config=config) 3236 ) 3237 3238 return JsonSchemaPropertySelector( 3239 configured_stream=configured_stream, 3240 properties_transformations=transformations, 3241 config=config, 3242 parameters=model.parameters or {}, 3243 )
@staticmethod
def
create_record_filter( model: airbyte_cdk.sources.declarative.models.declarative_component_schema.RecordFilter, config: Mapping[str, Any], **kwargs: Any) -> airbyte_cdk.RecordFilter:
@staticmethod
def
create_request_path( model: airbyte_cdk.sources.declarative.models.declarative_component_schema.RequestPath, config: Mapping[str, Any], **kwargs: Any) -> airbyte_cdk.sources.declarative.requesters.request_path.RequestPath:
@staticmethod
def
create_request_option( model: airbyte_cdk.sources.declarative.models.declarative_component_schema.RequestOption, config: Mapping[str, Any], **kwargs: Any) -> airbyte_cdk.RequestOption:
3257 @staticmethod 3258 def create_request_option( 3259 model: RequestOptionModel, config: Config, **kwargs: Any 3260 ) -> RequestOption: 3261 inject_into = RequestOptionType(model.inject_into.value) 3262 field_path: Optional[List[Union[InterpolatedString, str]]] = ( 3263 [ 3264 InterpolatedString.create(segment, parameters=kwargs.get("parameters", {})) 3265 for segment in model.field_path 3266 ] 3267 if model.field_path 3268 else None 3269 ) 3270 field_name = ( 3271 InterpolatedString.create(model.field_name, parameters=kwargs.get("parameters", {})) 3272 if model.field_name 3273 else None 3274 ) 3275 return RequestOption( 3276 field_name=field_name, 3277 field_path=field_path, 3278 inject_into=inject_into, 3279 parameters=kwargs.get("parameters", {}), 3280 )
def
create_record_selector( self, model: airbyte_cdk.sources.declarative.models.declarative_component_schema.RecordSelector, config: Mapping[str, Any], *, name: str, transformations: Optional[List[airbyte_cdk.RecordTransformation]] = None, decoder: airbyte_cdk.Decoder | None = None, client_side_incremental_sync_cursor: Optional[airbyte_cdk.Cursor] = None, file_uploader: Optional[airbyte_cdk.sources.declarative.retrievers.file_uploader.DefaultFileUploader] = None, **kwargs: Any) -> airbyte_cdk.RecordSelector:
3282 def create_record_selector( 3283 self, 3284 model: RecordSelectorModel, 3285 config: Config, 3286 *, 3287 name: str, 3288 transformations: List[RecordTransformation] | None = None, 3289 decoder: Decoder | None = None, 3290 client_side_incremental_sync_cursor: Optional[Cursor] = None, 3291 file_uploader: Optional[DefaultFileUploader] = None, 3292 **kwargs: Any, 3293 ) -> RecordSelector: 3294 extractor = self._create_component_from_model( 3295 model=model.extractor, decoder=decoder, config=config 3296 ) 3297 record_filter = ( 3298 self._create_component_from_model(model.record_filter, config=config) 3299 if model.record_filter 3300 else None 3301 ) 3302 3303 transform_before_filtering = ( 3304 False if model.transform_before_filtering is None else model.transform_before_filtering 3305 ) 3306 if client_side_incremental_sync_cursor: 3307 record_filter = ClientSideIncrementalRecordFilterDecorator( 3308 config=config, 3309 parameters=model.parameters, 3310 condition=model.record_filter.condition 3311 if (model.record_filter and hasattr(model.record_filter, "condition")) 3312 else None, 3313 cursor=client_side_incremental_sync_cursor, 3314 ) 3315 transform_before_filtering = ( 3316 True 3317 if model.transform_before_filtering is None 3318 else model.transform_before_filtering 3319 ) 3320 3321 if model.schema_normalization is None: 3322 # default to no schema normalization if not set 3323 model.schema_normalization = SchemaNormalizationModel.None_ 3324 3325 schema_normalization = ( 3326 TypeTransformer(SCHEMA_TRANSFORMER_TYPE_MAPPING[model.schema_normalization]) 3327 if isinstance(model.schema_normalization, SchemaNormalizationModel) 3328 else self._create_component_from_model(model.schema_normalization, config=config) # type: ignore[arg-type] # custom normalization model expected here 3329 ) 3330 3331 return RecordSelector( 3332 extractor=extractor, 3333 name=name, 3334 config=config, 3335 record_filter=record_filter, 3336 transformations=transformations or [], 3337 file_uploader=file_uploader, 3338 schema_normalization=schema_normalization, 3339 parameters=model.parameters or {}, 3340 transform_before_filtering=transform_before_filtering, 3341 )
@staticmethod
def
create_remove_fields( model: airbyte_cdk.sources.declarative.models.declarative_component_schema.RemoveFields, config: Mapping[str, Any], **kwargs: Any) -> airbyte_cdk.sources.declarative.transformations.RemoveFields:
def
create_selective_authenticator( self, model: airbyte_cdk.sources.declarative.models.declarative_component_schema.SelectiveAuthenticator, config: Mapping[str, Any], **kwargs: Any) -> airbyte_cdk.DeclarativeAuthenticator:
3351 def create_selective_authenticator( 3352 self, model: SelectiveAuthenticatorModel, config: Config, **kwargs: Any 3353 ) -> DeclarativeAuthenticator: 3354 authenticators = { 3355 name: self._create_component_from_model(model=auth, config=config) 3356 for name, auth in model.authenticators.items() 3357 } 3358 # SelectiveAuthenticator will return instance of DeclarativeAuthenticator or raise ValueError error 3359 return SelectiveAuthenticator( # type: ignore[abstract] 3360 config=config, 3361 authenticators=authenticators, 3362 authenticator_selection_path=model.authenticator_selection_path, 3363 **kwargs, 3364 )
@staticmethod
def
create_legacy_session_token_authenticator( model: airbyte_cdk.sources.declarative.models.declarative_component_schema.LegacySessionTokenAuthenticator, config: Mapping[str, Any], *, url_base: str, **kwargs: Any) -> airbyte_cdk.sources.declarative.auth.token.LegacySessionTokenAuthenticator:
3366 @staticmethod 3367 def create_legacy_session_token_authenticator( 3368 model: LegacySessionTokenAuthenticatorModel, config: Config, *, url_base: str, **kwargs: Any 3369 ) -> LegacySessionTokenAuthenticator: 3370 return LegacySessionTokenAuthenticator( 3371 api_url=url_base, 3372 header=model.header, 3373 login_url=model.login_url, 3374 password=model.password or "", 3375 session_token=model.session_token or "", 3376 session_token_response_key=model.session_token_response_key or "", 3377 username=model.username or "", 3378 validate_session_url=model.validate_session_url, 3379 config=config, 3380 parameters=model.parameters or {}, 3381 )
def
create_simple_retriever( self, model: airbyte_cdk.sources.declarative.models.declarative_component_schema.SimpleRetriever, config: Mapping[str, Any], *, name: str, primary_key: Union[str, List[str], List[List[str]], NoneType], request_options_provider: Optional[airbyte_cdk.sources.declarative.requesters.request_options.RequestOptionsProvider] = None, cursor: Optional[airbyte_cdk.Cursor] = None, has_stop_condition_cursor: bool = False, is_client_side_incremental_sync: bool = False, transformations: List[airbyte_cdk.RecordTransformation], file_uploader: Optional[airbyte_cdk.sources.declarative.retrievers.file_uploader.DefaultFileUploader] = None, incremental_sync: Union[airbyte_cdk.sources.declarative.models.declarative_component_schema.IncrementingCountCursor, airbyte_cdk.sources.declarative.models.declarative_component_schema.DatetimeBasedCursor, NoneType] = None, use_cache: Optional[bool] = None, log_formatter: Optional[Callable[[requests.models.Response], Any]] = None, partition_router: Optional[airbyte_cdk.sources.declarative.partition_routers.PartitionRouter] = None, **kwargs: Any) -> airbyte_cdk.SimpleRetriever:
3383 def create_simple_retriever( 3384 self, 3385 model: SimpleRetrieverModel, 3386 config: Config, 3387 *, 3388 name: str, 3389 primary_key: Optional[Union[str, List[str], List[List[str]]]], 3390 request_options_provider: Optional[RequestOptionsProvider] = None, 3391 cursor: Optional[Cursor] = None, 3392 has_stop_condition_cursor: bool = False, 3393 is_client_side_incremental_sync: bool = False, 3394 transformations: List[RecordTransformation], 3395 file_uploader: Optional[DefaultFileUploader] = None, 3396 incremental_sync: Optional[ 3397 Union[IncrementingCountCursorModel, DatetimeBasedCursorModel] 3398 ] = None, 3399 use_cache: Optional[bool] = None, 3400 log_formatter: Optional[Callable[[Response], Any]] = None, 3401 partition_router: Optional[PartitionRouter] = None, 3402 **kwargs: Any, 3403 ) -> SimpleRetriever: 3404 def _get_url(req: Requester) -> str: 3405 """ 3406 Closure to get the URL from the requester. This is used to get the URL in the case of a lazy retriever. 3407 This is needed because the URL is not set until the requester is created. 3408 """ 3409 3410 _url: str = ( 3411 model.requester.url 3412 if hasattr(model.requester, "url") and model.requester.url is not None 3413 else req.get_url(stream_state=None, stream_slice=None, next_page_token=None) 3414 ) 3415 _url_base: str = ( 3416 model.requester.url_base 3417 if hasattr(model.requester, "url_base") and model.requester.url_base is not None 3418 else req.get_url_base(stream_state=None, stream_slice=None, next_page_token=None) 3419 ) 3420 3421 return _url or _url_base 3422 3423 if cursor is None: 3424 cursor = FinalStateCursor(name, None, self._message_repository) 3425 3426 decoder = ( 3427 self._create_component_from_model(model=model.decoder, config=config) 3428 if model.decoder 3429 else JsonDecoder(parameters={}) 3430 ) 3431 record_selector = self._create_component_from_model( 3432 model=model.record_selector, 3433 name=name, 3434 config=config, 3435 decoder=decoder, 3436 transformations=transformations, 3437 client_side_incremental_sync_cursor=cursor if is_client_side_incremental_sync else None, 3438 file_uploader=file_uploader, 3439 ) 3440 3441 query_properties: Optional[QueryProperties] = None 3442 query_properties_key: Optional[str] = None 3443 self._ensure_query_properties_to_model(model.requester) 3444 if self._has_query_properties_in_request_parameters(model.requester): 3445 # It is better to be explicit about an error if PropertiesFromEndpoint is defined in multiple 3446 # places instead of default to request_parameters which isn't clearly documented 3447 if ( 3448 hasattr(model.requester, "fetch_properties_from_endpoint") 3449 and model.requester.fetch_properties_from_endpoint 3450 ): 3451 raise ValueError( 3452 f"PropertiesFromEndpoint should only be specified once per stream, but found in {model.requester.type}.fetch_properties_from_endpoint and {model.requester.type}.request_parameters" 3453 ) 3454 3455 query_properties_definitions = [] 3456 for key, request_parameter in model.requester.request_parameters.items(): # type: ignore # request_parameters is already validated to be a Mapping using _has_query_properties_in_request_parameters() 3457 if isinstance(request_parameter, QueryPropertiesModel): 3458 query_properties_key = key 3459 query_properties_definitions.append(request_parameter) 3460 3461 if len(query_properties_definitions) > 1: 3462 raise ValueError( 3463 f"request_parameters only supports defining one QueryProperties field, but found {len(query_properties_definitions)} usages" 3464 ) 3465 3466 if len(query_properties_definitions) == 1: 3467 query_properties = self._create_component_from_model( 3468 model=query_properties_definitions[0], stream_name=name, config=config 3469 ) 3470 3471 # Removes QueryProperties components from the interpolated mappings because it has been designed 3472 # to be used by the SimpleRetriever and will be resolved from the provider from the slice directly 3473 # instead of through jinja interpolation 3474 if hasattr(model.requester, "request_parameters") and isinstance( 3475 model.requester.request_parameters, Mapping 3476 ): 3477 model.requester.request_parameters = self._remove_query_properties( 3478 model.requester.request_parameters 3479 ) 3480 elif ( 3481 hasattr(model.requester, "fetch_properties_from_endpoint") 3482 and model.requester.fetch_properties_from_endpoint 3483 ): 3484 # todo: Deprecate this condition once dependent connectors migrate to query_properties 3485 query_properties_definition = QueryPropertiesModel( 3486 type="QueryProperties", 3487 property_list=model.requester.fetch_properties_from_endpoint, 3488 always_include_properties=None, 3489 property_chunking=None, 3490 ) # type: ignore # $parameters has a default value 3491 3492 query_properties = self.create_query_properties( 3493 model=query_properties_definition, 3494 stream_name=name, 3495 config=config, 3496 ) 3497 elif hasattr(model.requester, "query_properties") and model.requester.query_properties: 3498 query_properties = self.create_query_properties( 3499 model=model.requester.query_properties, 3500 stream_name=name, 3501 config=config, 3502 ) 3503 3504 requester = self._create_component_from_model( 3505 model=model.requester, 3506 decoder=decoder, 3507 name=name, 3508 query_properties_key=query_properties_key, 3509 use_cache=use_cache, 3510 config=config, 3511 ) 3512 3513 if not request_options_provider: 3514 request_options_provider = DefaultRequestOptionsProvider(parameters={}) 3515 if isinstance(request_options_provider, DefaultRequestOptionsProvider) and isinstance( 3516 partition_router, PartitionRouter 3517 ): 3518 request_options_provider = partition_router 3519 3520 paginator = ( 3521 self._create_component_from_model( 3522 model=model.paginator, 3523 config=config, 3524 url_base=_get_url(requester), 3525 extractor_model=model.record_selector.extractor, 3526 decoder=decoder, 3527 cursor_used_for_stop_condition=cursor if has_stop_condition_cursor else None, 3528 ) 3529 if model.paginator 3530 else NoPagination(parameters={}) 3531 ) 3532 3533 ignore_stream_slicer_parameters_on_paginated_requests = ( 3534 model.ignore_stream_slicer_parameters_on_paginated_requests or False 3535 ) 3536 3537 if ( 3538 model.partition_router 3539 and isinstance(model.partition_router, SubstreamPartitionRouterModel) 3540 and not bool(self._connector_state_manager.get_stream_state(name, None)) 3541 and any( 3542 parent_stream_config.lazy_read_pointer 3543 for parent_stream_config in model.partition_router.parent_stream_configs 3544 ) 3545 ): 3546 if incremental_sync: 3547 if incremental_sync.type != "DatetimeBasedCursor": 3548 raise ValueError( 3549 f"LazySimpleRetriever only supports DatetimeBasedCursor. Found: {incremental_sync.type}." 3550 ) 3551 3552 elif incremental_sync.step or incremental_sync.cursor_granularity: 3553 raise ValueError( 3554 f"Found more that one slice per parent. LazySimpleRetriever only supports single slice read for stream - {name}." 3555 ) 3556 3557 if model.decoder and model.decoder.type != "JsonDecoder": 3558 raise ValueError( 3559 f"LazySimpleRetriever only supports JsonDecoder. Found: {model.decoder.type}." 3560 ) 3561 3562 return LazySimpleRetriever( 3563 name=name, 3564 paginator=paginator, 3565 primary_key=primary_key, 3566 requester=requester, 3567 record_selector=record_selector, 3568 stream_slicer=_NO_STREAM_SLICING, 3569 request_option_provider=request_options_provider, 3570 config=config, 3571 ignore_stream_slicer_parameters_on_paginated_requests=ignore_stream_slicer_parameters_on_paginated_requests, 3572 parameters=model.parameters or {}, 3573 ) 3574 3575 if ( 3576 model.record_selector.record_filter 3577 and model.pagination_reset 3578 and model.pagination_reset.limits 3579 ): 3580 raise ValueError("PaginationResetLimits are not supported while having record filter.") 3581 3582 return SimpleRetriever( 3583 name=name, 3584 paginator=paginator, 3585 primary_key=primary_key, 3586 requester=requester, 3587 record_selector=record_selector, 3588 stream_slicer=_NO_STREAM_SLICING, 3589 request_option_provider=request_options_provider, 3590 config=config, 3591 ignore_stream_slicer_parameters_on_paginated_requests=ignore_stream_slicer_parameters_on_paginated_requests, 3592 additional_query_properties=query_properties, 3593 log_formatter=self._get_log_formatter(log_formatter, name), 3594 pagination_tracker_factory=self._create_pagination_tracker_factory( 3595 model.pagination_reset, cursor 3596 ), 3597 parameters=model.parameters or {}, 3598 )
def
create_state_delegating_stream( self, model: airbyte_cdk.sources.declarative.models.declarative_component_schema.StateDelegatingStream, config: Mapping[str, Any], **kwargs: Any) -> airbyte_cdk.sources.streams.concurrent.default_stream.DefaultStream:
3676 def create_state_delegating_stream( 3677 self, 3678 model: StateDelegatingStreamModel, 3679 config: Config, 3680 **kwargs: Any, 3681 ) -> DefaultStream: 3682 if ( 3683 model.full_refresh_stream.name != model.name 3684 or model.name != model.incremental_stream.name 3685 ): 3686 raise ValueError( 3687 f"state_delegating_stream, full_refresh_stream name and incremental_stream must have equal names. Instead has {model.name}, {model.full_refresh_stream.name} and {model.incremental_stream.name}." 3688 ) 3689 3690 # Resolve api_retention_period with config context (supports Jinja2 interpolation) 3691 resolved_retention_period: Optional[str] = None 3692 if model.api_retention_period: 3693 interpolated_retention = InterpolatedString.create( 3694 model.api_retention_period, parameters=model.parameters or {} 3695 ) 3696 resolved_value = interpolated_retention.eval(config=config) 3697 if resolved_value: 3698 resolved_retention_period = str(resolved_value) 3699 3700 if resolved_retention_period: 3701 for stream_model in (model.full_refresh_stream, model.incremental_stream): 3702 if isinstance(stream_model.incremental_sync, IncrementingCountCursorModel): 3703 raise ValueError( 3704 f"Stream '{model.name}' uses IncrementingCountCursor which is not supported " 3705 f"with api_retention_period. IncrementingCountCursor does not use datetime-based " 3706 f"cursors, so cursor age validation cannot be performed." 3707 ) 3708 3709 stream_state = self._connector_state_manager.get_stream_state(model.name, None) 3710 3711 if not stream_state: 3712 return self._create_component_from_model( # type: ignore[no-any-return] 3713 model.full_refresh_stream, config=config, **kwargs 3714 ) 3715 3716 incremental_stream: DefaultStream = self._create_component_from_model( 3717 model.incremental_stream, config=config, **kwargs 3718 ) # type: ignore[assignment] 3719 3720 # Only run cursor age validation for streams that are in the configured 3721 # catalog (or when no catalog was provided, e.g. during discover / connector 3722 # builder). Streams not selected by the user but instantiated as parent-stream 3723 # dependencies must not go through this path because it emits state messages 3724 # that the destination does not know about, causing "Stream not found" crashes. 3725 stream_is_in_catalog = ( 3726 not self._stream_name_to_configured_stream # no catalog → validate by default 3727 or model.name in self._stream_name_to_configured_stream 3728 ) 3729 if resolved_retention_period and stream_is_in_catalog: 3730 full_refresh_stream: DefaultStream = self._create_component_from_model( 3731 model.full_refresh_stream, config=config, **kwargs 3732 ) # type: ignore[assignment] 3733 if self._is_cursor_older_than_retention_period( 3734 stream_state, 3735 full_refresh_stream.cursor, 3736 incremental_stream.cursor, 3737 resolved_retention_period, 3738 model.name, 3739 ): 3740 # Clear state BEFORE constructing the full_refresh_stream so that 3741 # its cursor starts from start_date instead of the stale cursor. 3742 self._connector_state_manager.update_state_for_stream(model.name, None, {}) 3743 state_message = self._connector_state_manager.create_state_message(model.name, None) 3744 self._message_repository.emit_message(state_message) 3745 return self._create_component_from_model( # type: ignore[no-any-return] 3746 model.full_refresh_stream, config=config, **kwargs 3747 ) 3748 3749 return incremental_stream
def
create_async_retriever( self, model: airbyte_cdk.sources.declarative.models.declarative_component_schema.AsyncRetriever, config: Mapping[str, Any], *, name: str, primary_key: Union[str, List[str], List[List[str]], NoneType], stream_slicer: Optional[airbyte_cdk.sources.declarative.stream_slicers.StreamSlicer], client_side_incremental_sync: Optional[Dict[str, Any]] = None, transformations: List[airbyte_cdk.RecordTransformation], **kwargs: Any) -> airbyte_cdk.sources.declarative.retrievers.AsyncRetriever:
3848 def create_async_retriever( 3849 self, 3850 model: AsyncRetrieverModel, 3851 config: Config, 3852 *, 3853 name: str, 3854 primary_key: Optional[ 3855 Union[str, List[str], List[List[str]]] 3856 ], # this seems to be needed to match create_simple_retriever 3857 stream_slicer: Optional[StreamSlicer], 3858 client_side_incremental_sync: Optional[Dict[str, Any]] = None, 3859 transformations: List[RecordTransformation], 3860 **kwargs: Any, 3861 ) -> AsyncRetriever: 3862 if model.download_target_requester and not model.download_target_extractor: 3863 raise ValueError( 3864 f"`download_target_extractor` required if using a `download_target_requester`" 3865 ) 3866 3867 def _get_download_retriever( 3868 requester: Requester, extractor: RecordExtractor, _decoder: Decoder 3869 ) -> SimpleRetriever: 3870 # We create a record selector for the download retriever 3871 # with no schema normalization and no transformations, neither record filter 3872 # as all this occurs in the record_selector of the AsyncRetriever 3873 record_selector = RecordSelector( 3874 extractor=extractor, 3875 name=name, 3876 record_filter=None, 3877 transformations=[], 3878 schema_normalization=TypeTransformer(TransformConfig.NoTransform), 3879 config=config, 3880 parameters={}, 3881 ) 3882 paginator = ( 3883 self._create_component_from_model( 3884 model=model.download_paginator, 3885 decoder=_decoder, 3886 config=config, 3887 url_base="", 3888 ) 3889 if model.download_paginator 3890 else NoPagination(parameters={}) 3891 ) 3892 3893 return SimpleRetriever( 3894 requester=requester, 3895 record_selector=record_selector, 3896 primary_key=None, 3897 name=name, 3898 paginator=paginator, 3899 config=config, 3900 parameters={}, 3901 log_formatter=self._get_log_formatter(None, name), 3902 ) 3903 3904 def _get_job_timeout() -> datetime.timedelta: 3905 user_defined_timeout: Optional[int] = ( 3906 int( 3907 InterpolatedString.create( 3908 str(model.polling_job_timeout), 3909 parameters={}, 3910 ).eval(config) 3911 ) 3912 if model.polling_job_timeout 3913 else None 3914 ) 3915 3916 # check for user defined timeout during the test read or 15 minutes 3917 test_read_timeout = datetime.timedelta(minutes=user_defined_timeout or 15) 3918 # default value for non-connector builder is 60 minutes. 3919 default_sync_timeout = datetime.timedelta(minutes=user_defined_timeout or 60) 3920 3921 return ( 3922 test_read_timeout if self._emit_connector_builder_messages else default_sync_timeout 3923 ) 3924 3925 decoder = ( 3926 self._create_component_from_model(model=model.decoder, config=config) 3927 if model.decoder 3928 else JsonDecoder(parameters={}) 3929 ) 3930 record_selector = self._create_component_from_model( 3931 model=model.record_selector, 3932 config=config, 3933 decoder=decoder, 3934 name=name, 3935 transformations=transformations, 3936 client_side_incremental_sync=client_side_incremental_sync, 3937 ) 3938 3939 stream_slicer = stream_slicer or SinglePartitionRouter(parameters={}) 3940 if self._should_limit_slices_fetched(): 3941 stream_slicer = cast( 3942 StreamSlicer, 3943 StreamSlicerTestReadDecorator( 3944 wrapped_slicer=stream_slicer, 3945 maximum_number_of_slices=self._limit_slices_fetched or 5, 3946 ), 3947 ) 3948 3949 creation_requester = self._create_component_from_model( 3950 model=model.creation_requester, 3951 decoder=decoder, 3952 config=config, 3953 name=f"job creation - {name}", 3954 ) 3955 polling_requester = self._create_component_from_model( 3956 model=model.polling_requester, 3957 decoder=decoder, 3958 config=config, 3959 name=f"job polling - {name}", 3960 ) 3961 job_download_components_name = f"job download - {name}" 3962 download_decoder = ( 3963 self._create_component_from_model(model=model.download_decoder, config=config) 3964 if model.download_decoder 3965 else JsonDecoder(parameters={}) 3966 ) 3967 download_extractor = ( 3968 self._create_component_from_model( 3969 model=model.download_extractor, 3970 config=config, 3971 decoder=download_decoder, 3972 parameters=model.parameters, 3973 ) 3974 if model.download_extractor 3975 else DpathExtractor( 3976 [], 3977 config=config, 3978 decoder=download_decoder, 3979 parameters=model.parameters or {}, 3980 ) 3981 ) 3982 download_requester = self._create_component_from_model( 3983 model=model.download_requester, 3984 decoder=download_decoder, 3985 config=config, 3986 name=job_download_components_name, 3987 ) 3988 download_retriever = _get_download_retriever( 3989 download_requester, download_extractor, download_decoder 3990 ) 3991 abort_requester = ( 3992 self._create_component_from_model( 3993 model=model.abort_requester, 3994 decoder=decoder, 3995 config=config, 3996 name=f"job abort - {name}", 3997 ) 3998 if model.abort_requester 3999 else None 4000 ) 4001 delete_requester = ( 4002 self._create_component_from_model( 4003 model=model.delete_requester, 4004 decoder=decoder, 4005 config=config, 4006 name=f"job delete - {name}", 4007 ) 4008 if model.delete_requester 4009 else None 4010 ) 4011 download_target_requester = ( 4012 self._create_component_from_model( 4013 model=model.download_target_requester, 4014 decoder=decoder, 4015 config=config, 4016 name=f"job extract_url - {name}", 4017 ) 4018 if model.download_target_requester 4019 else None 4020 ) 4021 status_extractor = self._create_component_from_model( 4022 model=model.status_extractor, decoder=decoder, config=config, name=name 4023 ) 4024 download_target_extractor = ( 4025 self._create_component_from_model( 4026 model=model.download_target_extractor, 4027 decoder=decoder, 4028 config=config, 4029 name=name, 4030 ) 4031 if model.download_target_extractor 4032 else None 4033 ) 4034 4035 job_repository: AsyncJobRepository = AsyncHttpJobRepository( 4036 creation_requester=creation_requester, 4037 polling_requester=polling_requester, 4038 download_retriever=download_retriever, 4039 download_target_requester=download_target_requester, 4040 abort_requester=abort_requester, 4041 delete_requester=delete_requester, 4042 status_extractor=status_extractor, 4043 status_mapping=self._create_async_job_status_mapping(model.status_mapping, config), 4044 download_target_extractor=download_target_extractor, 4045 job_timeout=_get_job_timeout(), 4046 ) 4047 4048 failed_retry_wait_time_in_seconds: Optional[int] = ( 4049 int( 4050 InterpolatedString.create( 4051 str(model.failed_retry_wait_time_in_seconds), 4052 parameters={}, 4053 ).eval(config) 4054 ) 4055 if model.failed_retry_wait_time_in_seconds 4056 else None 4057 ) 4058 4059 async_job_partition_router = AsyncJobPartitionRouter( 4060 job_orchestrator_factory=lambda stream_slices: AsyncJobOrchestrator( 4061 job_repository, 4062 stream_slices, 4063 self._job_tracker, 4064 self._message_repository, 4065 # FIXME work would need to be done here in order to detect if a stream as a parent stream that is bulk 4066 has_bulk_parent=False, 4067 # set the `job_max_retry` to 1 for the `Connector Builder`` use-case. 4068 # `None` == default retry is set to 3 attempts, under the hood. 4069 job_max_retry=1 if self._emit_connector_builder_messages else None, 4070 failed_retry_wait_time_in_seconds=failed_retry_wait_time_in_seconds, 4071 ), 4072 stream_slicer=stream_slicer, 4073 config=config, 4074 parameters=model.parameters or {}, 4075 ) 4076 4077 return AsyncRetriever( 4078 record_selector=record_selector, 4079 stream_slicer=async_job_partition_router, 4080 config=config, 4081 parameters=model.parameters or {}, 4082 )
def
create_spec( self, model: airbyte_cdk.sources.declarative.models.declarative_component_schema.Spec, config: Mapping[str, Any], **kwargs: Any) -> airbyte_cdk.sources.declarative.spec.Spec:
4084 def create_spec(self, model: SpecModel, config: Config, **kwargs: Any) -> Spec: 4085 config_migrations = [ 4086 self._create_component_from_model(migration, config) 4087 for migration in ( 4088 model.config_normalization_rules.config_migrations 4089 if ( 4090 model.config_normalization_rules 4091 and model.config_normalization_rules.config_migrations 4092 ) 4093 else [] 4094 ) 4095 ] 4096 config_transformations = [ 4097 self._create_component_from_model(transformation, config) 4098 for transformation in ( 4099 model.config_normalization_rules.transformations 4100 if ( 4101 model.config_normalization_rules 4102 and model.config_normalization_rules.transformations 4103 ) 4104 else [] 4105 ) 4106 ] 4107 config_validations = [ 4108 self._create_component_from_model(validation, config) 4109 for validation in ( 4110 model.config_normalization_rules.validations 4111 if ( 4112 model.config_normalization_rules 4113 and model.config_normalization_rules.validations 4114 ) 4115 else [] 4116 ) 4117 ] 4118 4119 return Spec( 4120 connection_specification=model.connection_specification, 4121 documentation_url=model.documentation_url, 4122 advanced_auth=model.advanced_auth, 4123 parameters={}, 4124 config_migrations=config_migrations, 4125 config_transformations=config_transformations, 4126 config_validations=config_validations, 4127 )
def
create_substream_partition_router( self, model: airbyte_cdk.sources.declarative.models.declarative_component_schema.SubstreamPartitionRouter, config: Mapping[str, Any], *, stream_name: str, **kwargs: Any) -> airbyte_cdk.SubstreamPartitionRouter:
4129 def create_substream_partition_router( 4130 self, 4131 model: SubstreamPartitionRouterModel, 4132 config: Config, 4133 *, 4134 stream_name: str, 4135 **kwargs: Any, 4136 ) -> SubstreamPartitionRouter: 4137 parent_stream_configs = [] 4138 if model.parent_stream_configs: 4139 parent_stream_configs.extend( 4140 [ 4141 self.create_parent_stream_config_with_substream_wrapper( 4142 model=parent_stream_config, config=config, stream_name=stream_name, **kwargs 4143 ) 4144 for parent_stream_config in model.parent_stream_configs 4145 ] 4146 ) 4147 4148 return SubstreamPartitionRouter( 4149 parent_stream_configs=parent_stream_configs, 4150 parameters=model.parameters or {}, 4151 config=config, 4152 )
def
create_parent_stream_config_with_substream_wrapper( self, model: airbyte_cdk.sources.declarative.models.declarative_component_schema.ParentStreamConfig, config: Mapping[str, Any], *, stream_name: str, **kwargs: Any) -> Any:
4154 def create_parent_stream_config_with_substream_wrapper( 4155 self, model: ParentStreamConfigModel, config: Config, *, stream_name: str, **kwargs: Any 4156 ) -> Any: 4157 child_state = self._connector_state_manager.get_stream_state(stream_name, None) 4158 4159 parent_state: Optional[Mapping[str, Any]] = ( 4160 child_state if model.incremental_dependency and child_state else None 4161 ) 4162 connector_state_manager = self._instantiate_parent_stream_state_manager( 4163 child_state, config, model, parent_state 4164 ) 4165 4166 substream_factory = ModelToComponentFactory( 4167 custom_components_trusted=self._custom_components_trusted, 4168 connector_state_manager=connector_state_manager, 4169 limit_pages_fetched_per_slice=self._limit_pages_fetched_per_slice, 4170 limit_slices_fetched=self._limit_slices_fetched, 4171 emit_connector_builder_messages=self._emit_connector_builder_messages, 4172 disable_retries=self._disable_retries, 4173 disable_cache=self._disable_cache, 4174 message_repository=StateFilteringMessageRepository( 4175 LogAppenderMessageRepositoryDecorator( 4176 { 4177 "airbyte_cdk": {"stream": {"is_substream": True}}, 4178 "http": {"is_auxiliary": True}, 4179 }, 4180 self._message_repository, 4181 self._evaluate_log_level(self._emit_connector_builder_messages), 4182 ), 4183 ), 4184 api_budget=self._api_budget, 4185 # Share the authenticator registry so parent and child streams draw from the 4186 # same token quota counters 4187 rate_limited_authenticators=self._rate_limited_authenticators, 4188 ) 4189 4190 return substream_factory.create_parent_stream_config( 4191 model=model, config=config, stream_name=stream_name, **kwargs 4192 )
@staticmethod
def
create_wait_time_from_header( model: airbyte_cdk.sources.declarative.models.declarative_component_schema.WaitTimeFromHeader, config: Mapping[str, Any], **kwargs: Any) -> airbyte_cdk.sources.declarative.requesters.error_handlers.backoff_strategies.WaitTimeFromHeaderBackoffStrategy:
4252 @staticmethod 4253 def create_wait_time_from_header( 4254 model: WaitTimeFromHeaderModel, config: Config, **kwargs: Any 4255 ) -> WaitTimeFromHeaderBackoffStrategy: 4256 return WaitTimeFromHeaderBackoffStrategy( 4257 header=model.header, 4258 parameters=model.parameters or {}, 4259 config=config, 4260 regex=model.regex, 4261 max_waiting_time_in_seconds=model.max_waiting_time_in_seconds 4262 if model.max_waiting_time_in_seconds is not None 4263 else None, 4264 )
@staticmethod
def
create_wait_until_time_from_header( model: airbyte_cdk.sources.declarative.models.declarative_component_schema.WaitUntilTimeFromHeader, config: Mapping[str, Any], **kwargs: Any) -> airbyte_cdk.sources.declarative.requesters.error_handlers.backoff_strategies.WaitUntilTimeFromHeaderBackoffStrategy:
4266 @staticmethod 4267 def create_wait_until_time_from_header( 4268 model: WaitUntilTimeFromHeaderModel, config: Config, **kwargs: Any 4269 ) -> WaitUntilTimeFromHeaderBackoffStrategy: 4270 return WaitUntilTimeFromHeaderBackoffStrategy( 4271 header=model.header, 4272 parameters=model.parameters or {}, 4273 config=config, 4274 min_wait=model.min_wait, 4275 regex=model.regex, 4276 )
@staticmethod
def
create_components_mapping_definition( model: airbyte_cdk.sources.declarative.models.declarative_component_schema.ComponentMappingDefinition, config: Mapping[str, Any], **kwargs: Any) -> airbyte_cdk.sources.declarative.resolvers.ComponentMappingDefinition:
4284 @staticmethod 4285 def create_components_mapping_definition( 4286 model: ComponentMappingDefinitionModel, config: Config, **kwargs: Any 4287 ) -> ComponentMappingDefinition: 4288 interpolated_value = InterpolatedString.create( 4289 model.value, parameters=model.parameters or {} 4290 ) 4291 field_path = [ 4292 InterpolatedString.create(path, parameters=model.parameters or {}) 4293 for path in model.field_path 4294 ] 4295 return ComponentMappingDefinition( 4296 field_path=field_path, # type: ignore[arg-type] # field_path can be str and InterpolatedString 4297 value=interpolated_value, 4298 value_type=ModelToComponentFactory._json_schema_type_name_to_type(model.value_type), 4299 create_or_update=model.create_or_update, 4300 condition=model.condition, 4301 parameters=model.parameters or {}, 4302 )
def
create_http_components_resolver( self, model: airbyte_cdk.sources.declarative.models.declarative_component_schema.HttpComponentsResolver, config: Mapping[str, Any], stream_name: Optional[str] = None) -> Any:
4304 def create_http_components_resolver( 4305 self, model: HttpComponentsResolverModel, config: Config, stream_name: Optional[str] = None 4306 ) -> Any: 4307 retriever = self._create_component_from_model( 4308 model=model.retriever, 4309 config=config, 4310 name=f"{stream_name if stream_name else '__http_components_resolver'}", 4311 primary_key=None, 4312 stream_slicer=self._build_stream_slicer_from_partition_router(model.retriever, config), 4313 transformations=[], 4314 ) 4315 4316 components_mapping = [] 4317 for component_mapping_definition_model in model.components_mapping: 4318 if component_mapping_definition_model.condition: 4319 raise ValueError("`condition` is only supported for `ConfigComponentsResolver`") 4320 components_mapping.append( 4321 self._create_component_from_model( 4322 model=component_mapping_definition_model, 4323 value_type=ModelToComponentFactory._json_schema_type_name_to_type( 4324 component_mapping_definition_model.value_type 4325 ), 4326 config=config, 4327 ) 4328 ) 4329 4330 return HttpComponentsResolver( 4331 retriever=retriever, 4332 stream_slicer=self._build_stream_slicer_from_partition_router(model.retriever, config), 4333 config=config, 4334 components_mapping=components_mapping, 4335 parameters=model.parameters or {}, 4336 )
@staticmethod
def
create_stream_config( model: airbyte_cdk.sources.declarative.models.declarative_component_schema.StreamConfig, config: Mapping[str, Any], **kwargs: Any) -> airbyte_cdk.sources.declarative.resolvers.StreamConfig:
4338 @staticmethod 4339 def create_stream_config( 4340 model: StreamConfigModel, config: Config, **kwargs: Any 4341 ) -> StreamConfig: 4342 model_configs_pointer: List[Union[InterpolatedString, str]] = ( 4343 [x for x in model.configs_pointer] if model.configs_pointer else [] 4344 ) 4345 4346 return StreamConfig( 4347 configs_pointer=model_configs_pointer, 4348 default_values=model.default_values, 4349 parameters=model.parameters or {}, 4350 )
def
create_config_components_resolver( self, model: airbyte_cdk.sources.declarative.models.declarative_component_schema.ConfigComponentsResolver, config: Mapping[str, Any]) -> Any:
4352 def create_config_components_resolver( 4353 self, 4354 model: ConfigComponentsResolverModel, 4355 config: Config, 4356 ) -> Any: 4357 model_stream_configs = ( 4358 model.stream_config if isinstance(model.stream_config, list) else [model.stream_config] 4359 ) 4360 4361 stream_configs = [ 4362 self._create_component_from_model( 4363 stream_config, config=config, parameters=model.parameters or {} 4364 ) 4365 for stream_config in model_stream_configs 4366 ] 4367 4368 components_mapping = [ 4369 self._create_component_from_model( 4370 model=components_mapping_definition_model, 4371 value_type=ModelToComponentFactory._json_schema_type_name_to_type( 4372 components_mapping_definition_model.value_type 4373 ), 4374 config=config, 4375 parameters=model.parameters, 4376 ) 4377 for components_mapping_definition_model in model.components_mapping 4378 ] 4379 4380 return ConfigComponentsResolver( 4381 stream_configs=stream_configs, 4382 config=config, 4383 components_mapping=components_mapping, 4384 parameters=model.parameters or {}, 4385 )
def
create_parametrized_components_resolver( self, model: airbyte_cdk.sources.declarative.models.declarative_component_schema.ParametrizedComponentsResolver, config: Mapping[str, Any]) -> airbyte_cdk.sources.declarative.resolvers.ParametrizedComponentsResolver:
4387 def create_parametrized_components_resolver( 4388 self, 4389 model: ParametrizedComponentsResolverModel, 4390 config: Config, 4391 ) -> ParametrizedComponentsResolver: 4392 stream_parameters = StreamParametersDefinition( 4393 list_of_parameters_for_stream=model.stream_parameters.list_of_parameters_for_stream 4394 ) 4395 4396 components_mapping = [] 4397 for components_mapping_definition_model in model.components_mapping: 4398 if components_mapping_definition_model.condition: 4399 raise ValueError("`condition` is only supported for `ConfigComponentsResolver`") 4400 components_mapping.append( 4401 self._create_component_from_model( 4402 model=components_mapping_definition_model, 4403 value_type=ModelToComponentFactory._json_schema_type_name_to_type( 4404 components_mapping_definition_model.value_type 4405 ), 4406 config=config, 4407 ) 4408 ) 4409 return ParametrizedComponentsResolver( 4410 stream_parameters=stream_parameters, 4411 config=config, 4412 components_mapping=components_mapping, 4413 parameters=model.parameters or {}, 4414 )
def
create_http_api_budget( self, model: airbyte_cdk.sources.declarative.models.declarative_component_schema.HTTPAPIBudget, config: Mapping[str, Any], **kwargs: Any) -> airbyte_cdk.HttpAPIBudget:
4438 def create_http_api_budget( 4439 self, model: HTTPAPIBudgetModel, config: Config, **kwargs: Any 4440 ) -> HttpAPIBudget: 4441 policies = [ 4442 self._create_component_from_model(model=policy, config=config) 4443 for policy in model.policies 4444 ] 4445 4446 return HttpAPIBudget( 4447 policies=policies, 4448 ratelimit_reset_header=model.ratelimit_reset_header or "ratelimit-reset", 4449 ratelimit_remaining_header=model.ratelimit_remaining_header or "ratelimit-remaining", 4450 status_codes_for_ratelimit_hit=model.status_codes_for_ratelimit_hit or [429], 4451 )
def
create_fixed_window_call_rate_policy( self, model: airbyte_cdk.sources.declarative.models.declarative_component_schema.FixedWindowCallRatePolicy, config: Mapping[str, Any], **kwargs: Any) -> airbyte_cdk.sources.streams.call_rate.FixedWindowCallRatePolicy:
4453 def create_fixed_window_call_rate_policy( 4454 self, model: FixedWindowCallRatePolicyModel, config: Config, **kwargs: Any 4455 ) -> FixedWindowCallRatePolicy: 4456 matchers = [ 4457 self._create_component_from_model(model=matcher, config=config) 4458 for matcher in model.matchers 4459 ] 4460 4461 # Set the initial reset timestamp to 10 days from now. 4462 # This value will be updated by the first request. 4463 return FixedWindowCallRatePolicy( 4464 next_reset_ts=datetime.datetime.now() + datetime.timedelta(days=10), 4465 period=parse_duration(model.period), 4466 call_limit=model.call_limit, 4467 matchers=matchers, 4468 )
def
create_file_uploader( self, model: airbyte_cdk.sources.declarative.models.declarative_component_schema.FileUploader, config: Mapping[str, Any], **kwargs: Any) -> airbyte_cdk.sources.declarative.retrievers.file_uploader.FileUploader:
4470 def create_file_uploader( 4471 self, model: FileUploaderModel, config: Config, **kwargs: Any 4472 ) -> FileUploader: 4473 name = "File Uploader" 4474 requester = self._create_component_from_model( 4475 model=model.requester, 4476 config=config, 4477 name=name, 4478 **kwargs, 4479 ) 4480 download_target_extractor = self._create_component_from_model( 4481 model=model.download_target_extractor, 4482 config=config, 4483 name=name, 4484 **kwargs, 4485 ) 4486 emit_connector_builder_messages = self._emit_connector_builder_messages 4487 file_uploader = DefaultFileUploader( 4488 requester=requester, 4489 download_target_extractor=download_target_extractor, 4490 config=config, 4491 file_writer=NoopFileWriter() 4492 if emit_connector_builder_messages 4493 else LocalFileSystemFileWriter(), 4494 parameters=model.parameters or {}, 4495 filename_extractor=model.filename_extractor if model.filename_extractor else None, 4496 ) 4497 4498 return ( 4499 ConnectorBuilderFileUploader(file_uploader) 4500 if emit_connector_builder_messages 4501 else file_uploader 4502 )
def
create_moving_window_call_rate_policy( self, model: airbyte_cdk.sources.declarative.models.declarative_component_schema.MovingWindowCallRatePolicy, config: Mapping[str, Any], **kwargs: Any) -> airbyte_cdk.MovingWindowCallRatePolicy:
4504 def create_moving_window_call_rate_policy( 4505 self, model: MovingWindowCallRatePolicyModel, config: Config, **kwargs: Any 4506 ) -> MovingWindowCallRatePolicy: 4507 rates = [ 4508 self._create_component_from_model(model=rate, config=config) for rate in model.rates 4509 ] 4510 matchers = [ 4511 self._create_component_from_model(model=matcher, config=config) 4512 for matcher in model.matchers 4513 ] 4514 return MovingWindowCallRatePolicy( 4515 rates=rates, 4516 matchers=matchers, 4517 )
def
create_unlimited_call_rate_policy( self, model: airbyte_cdk.sources.declarative.models.declarative_component_schema.UnlimitedCallRatePolicy, config: Mapping[str, Any], **kwargs: Any) -> airbyte_cdk.sources.streams.call_rate.UnlimitedCallRatePolicy:
4519 def create_unlimited_call_rate_policy( 4520 self, model: UnlimitedCallRatePolicyModel, config: Config, **kwargs: Any 4521 ) -> UnlimitedCallRatePolicy: 4522 matchers = [ 4523 self._create_component_from_model(model=matcher, config=config) 4524 for matcher in model.matchers 4525 ] 4526 4527 return UnlimitedCallRatePolicy( 4528 matchers=matchers, 4529 )
def
create_rate( self, model: airbyte_cdk.sources.declarative.models.declarative_component_schema.Rate, config: Mapping[str, Any], **kwargs: Any) -> airbyte_cdk.Rate:
def
create_http_request_matcher( self, model: airbyte_cdk.sources.declarative.models.declarative_component_schema.HttpRequestRegexMatcher, config: Mapping[str, Any], **kwargs: Any) -> airbyte_cdk.sources.streams.call_rate.HttpRequestRegexMatcher:
4538 def create_http_request_matcher( 4539 self, model: HttpRequestRegexMatcherModel, config: Config, **kwargs: Any 4540 ) -> HttpRequestRegexMatcher: 4541 weight = model.weight 4542 if weight is not None: 4543 if isinstance(weight, str): 4544 weight = int(InterpolatedString.create(weight, parameters={}).eval(config)) 4545 else: 4546 weight = int(weight) 4547 if weight < 1: 4548 raise ValueError(f"weight must be >= 1, got {weight}") 4549 return HttpRequestRegexMatcher( 4550 method=model.method, 4551 url_base=model.url_base, 4552 url_path_pattern=model.url_path_pattern, 4553 params=model.params, 4554 headers=model.headers, 4555 weight=weight, 4556 )
def
create_rate_limited_multiple_token_authenticator( self, model: airbyte_cdk.sources.declarative.models.declarative_component_schema.RateLimitedMultipleTokenAuthenticator, config: Mapping[str, Any], **kwargs: Any) -> airbyte_cdk.sources.declarative.auth.RateLimitedMultipleTokenAuthenticator:
4558 def create_rate_limited_multiple_token_authenticator( 4559 self, 4560 model: RateLimitedMultipleTokenAuthenticatorModel, 4561 config: Config, 4562 **kwargs: Any, 4563 ) -> RateLimitedMultipleTokenAuthenticator: 4564 if isinstance(model.tokens, str): 4565 tokens_value = InterpolatedString.create(model.tokens, parameters={}).eval(config) 4566 delimiter = model.token_delimiter or "," 4567 tokens = [ 4568 token.strip() for token in str(tokens_value).split(delimiter) if token.strip() 4569 ] 4570 else: 4571 tokens = [ 4572 token_value 4573 for token in model.tokens 4574 if ( 4575 token_value := str( 4576 InterpolatedString.create(token, parameters={}).eval(config) 4577 ).strip() 4578 ) 4579 ] 4580 4581 quota_specs = [ 4582 { 4583 "name": quota_model.name, 4584 "remaining_path": quota_model.remaining_path, 4585 "reset_path": quota_model.reset_path, 4586 "limit_path": quota_model.limit_path, 4587 "matchers": [ 4588 { 4589 "method": matcher_model.method, 4590 "url_base": matcher_model.url_base, 4591 "url_path_pattern": matcher_model.url_path_pattern, 4592 "params": matcher_model.params, 4593 "headers": matcher_model.headers, 4594 "weight": matcher_model.weight, 4595 } 4596 for matcher_model in quota_model.matchers or [] 4597 ], 4598 } 4599 for quota_model in model.quotas 4600 ] 4601 4602 quota_status_url = str( 4603 InterpolatedString.create(model.quota_status_source.url, parameters={}).eval(config) 4604 ) 4605 quota_status_http_method = ( 4606 model.quota_status_source.http_method.value 4607 if model.quota_status_source.http_method 4608 else "GET" 4609 ) 4610 quota_status_headers = { 4611 key: str(InterpolatedString.create(value, parameters={}).eval(config)) 4612 for key, value in (model.quota_status_source.request_headers or {}).items() 4613 } 4614 auth_method = model.auth_method or "Bearer" 4615 header = model.header or "Authorization" 4616 max_wait_time_str = str( 4617 InterpolatedString.create(model.max_wait_time or "PT2H", parameters={}).eval(config) 4618 ) 4619 max_wait_time = parse_duration(max_wait_time_str) 4620 if not isinstance(max_wait_time, datetime.timedelta): 4621 raise ValueError( 4622 f"max_wait_time must be a fixed-length ISO 8601 duration (e.g. 'PT2H'); " 4623 f"calendar-unit durations like '{max_wait_time_str}' are not supported" 4624 ) 4625 budget_reserve_fraction = ( 4626 model.budget_reserve_fraction if model.budget_reserve_fraction is not None else 0.1 4627 ) 4628 budget_min_reserve = ( 4629 model.budget_min_reserve if model.budget_min_reserve is not None else 50 4630 ) 4631 4632 # Reuse the same instance for identical definitions so that all streams share the 4633 # same token quota counters (similar to how api_budget is shared). The key is built 4634 # from the resolved constructor arguments rather than the raw model so that 4635 # stream-specific `$parameters` propagated onto the model (and its nested components) 4636 # cannot break instance sharing. 4637 cache_key = json.dumps( 4638 { 4639 "tokens": tokens, 4640 "quotas": quota_specs, 4641 "quota_status_url": quota_status_url, 4642 "quota_status_http_method": quota_status_http_method, 4643 "quota_status_headers": quota_status_headers, 4644 "auth_method": auth_method, 4645 "header": header, 4646 "max_wait_time": max_wait_time.total_seconds(), 4647 "budget_reserve_fraction": budget_reserve_fraction, 4648 "budget_min_reserve": budget_min_reserve, 4649 }, 4650 sort_keys=True, 4651 ) 4652 if cache_key in self._rate_limited_authenticators: 4653 return self._rate_limited_authenticators[cache_key] 4654 4655 quotas = [ 4656 TokenQuota( 4657 name=quota_model.name, 4658 remaining_path=quota_model.remaining_path, 4659 reset_path=quota_model.reset_path, 4660 limit_path=quota_model.limit_path, 4661 matchers=[ 4662 self.create_http_request_matcher(matcher_model, config) 4663 for matcher_model in quota_model.matchers or [] 4664 ], 4665 ) 4666 for quota_model in model.quotas 4667 ] 4668 4669 authenticator = RateLimitedMultipleTokenAuthenticator( 4670 tokens=tokens, 4671 quotas=quotas, 4672 quota_status_url=quota_status_url, 4673 quota_status_http_method=quota_status_http_method, 4674 quota_status_headers=quota_status_headers, 4675 auth_method=auth_method, 4676 header=header, 4677 max_wait_time=max_wait_time, 4678 budget_reserve_fraction=budget_reserve_fraction, 4679 budget_min_reserve=budget_min_reserve, 4680 ) 4681 self._rate_limited_authenticators[cache_key] = authenticator 4682 return authenticator
def
set_api_budget( self, component_definition: Mapping[str, Any], config: Mapping[str, Any]) -> None:
def
create_grouping_partition_router( self, model: airbyte_cdk.sources.declarative.models.declarative_component_schema.GroupingPartitionRouter, config: Mapping[str, Any], *, stream_name: str, **kwargs: Any) -> airbyte_cdk.sources.declarative.partition_routers.GroupingPartitionRouter:
4689 def create_grouping_partition_router( 4690 self, 4691 model: GroupingPartitionRouterModel, 4692 config: Config, 4693 *, 4694 stream_name: str, 4695 **kwargs: Any, 4696 ) -> GroupingPartitionRouter: 4697 underlying_router = self._create_component_from_model( 4698 model=model.underlying_partition_router, 4699 config=config, 4700 stream_name=stream_name, 4701 **kwargs, 4702 ) 4703 if model.group_size < 1: 4704 raise ValueError(f"Group size must be greater than 0, got {model.group_size}") 4705 4706 # Request options in underlying partition routers are not supported for GroupingPartitionRouter 4707 # because they are specific to individual partitions and cannot be aggregated or handled 4708 # when grouping, potentially leading to incorrect API calls. Any request customization 4709 # should be managed at the stream level through the requester's configuration. 4710 if isinstance(underlying_router, SubstreamPartitionRouter): 4711 if any( 4712 parent_config.request_option 4713 for parent_config in underlying_router.parent_stream_configs 4714 ): 4715 raise ValueError("Request options are not supported for GroupingPartitionRouter.") 4716 4717 if isinstance(underlying_router, ListPartitionRouter): 4718 if underlying_router.request_option: 4719 raise ValueError("Request options are not supported for GroupingPartitionRouter.") 4720 4721 return GroupingPartitionRouter( 4722 group_size=model.group_size, 4723 underlying_partition_router=underlying_router, 4724 deduplicate=model.deduplicate if model.deduplicate is not None else True, 4725 config=config, 4726 )