airbyte_cdk.sources.declarative.parsers.model_to_component_factory
1# 2# Copyright (c) 2025 Airbyte, Inc., all rights reserved. 3# 4 5from __future__ import annotations 6 7import datetime 8import importlib 9import inspect 10import logging 11import re 12from functools import partial 13from typing import ( 14 TYPE_CHECKING, 15 Any, 16 Callable, 17 Dict, 18 List, 19 Mapping, 20 MutableMapping, 21 Optional, 22 Tuple, 23 Type, 24 Union, 25 cast, 26 get_args, 27 get_origin, 28 get_type_hints, 29) 30 31if TYPE_CHECKING: 32 from airbyte_cdk.legacy.sources.declarative.incremental.datetime_based_cursor import ( 33 DatetimeBasedCursor, 34 ) 35 36from airbyte_protocol_dataclasses.models import ConfiguredAirbyteStream 37from isodate import parse_duration 38from pydantic.v1 import BaseModel 39from requests import Response 40 41from airbyte_cdk.connector_builder.models import ( 42 LogMessage as ConnectorBuilderLogMessage, 43) 44from airbyte_cdk.models import ( 45 AirbyteStateBlob, 46 AirbyteStateMessage, 47 AirbyteStateType, 48 AirbyteStreamState, 49 ConfiguredAirbyteCatalog, 50 FailureType, 51 Level, 52 StreamDescriptor, 53) 54from airbyte_cdk.sources.connector_state_manager import ConnectorStateManager 55from airbyte_cdk.sources.declarative.async_job.job_orchestrator import AsyncJobOrchestrator 56from airbyte_cdk.sources.declarative.async_job.job_tracker import JobTracker 57from airbyte_cdk.sources.declarative.async_job.repository import AsyncJobRepository 58from airbyte_cdk.sources.declarative.async_job.status import AsyncJobStatus 59from airbyte_cdk.sources.declarative.auth import DeclarativeOauth2Authenticator, JwtAuthenticator 60from airbyte_cdk.sources.declarative.auth.declarative_authenticator import ( 61 DeclarativeAuthenticator, 62 NoAuth, 63) 64from airbyte_cdk.sources.declarative.auth.jwt import JwtAlgorithm 65from airbyte_cdk.sources.declarative.auth.oauth import ( 66 DeclarativeSingleUseRefreshTokenOauth2Authenticator, 67) 68from airbyte_cdk.sources.declarative.auth.selective_authenticator import SelectiveAuthenticator 69from airbyte_cdk.sources.declarative.auth.token import ( 70 ApiKeyAuthenticator, 71 BasicHttpAuthenticator, 72 BearerAuthenticator, 73 LegacySessionTokenAuthenticator, 74) 75from airbyte_cdk.sources.declarative.auth.token_provider import ( 76 InterpolatedSessionTokenProvider, 77 InterpolatedStringTokenProvider, 78 SessionTokenProvider, 79 TokenProvider, 80) 81from airbyte_cdk.sources.declarative.checks import ( 82 CheckDynamicStream, 83 CheckStream, 84 DynamicStreamCheckConfig, 85) 86from airbyte_cdk.sources.declarative.concurrency_level import ConcurrencyLevel 87from airbyte_cdk.sources.declarative.datetime.min_max_datetime import MinMaxDatetime 88from airbyte_cdk.sources.declarative.decoders import ( 89 Decoder, 90 IterableDecoder, 91 JsonDecoder, 92 PaginationDecoderDecorator, 93 XmlDecoder, 94 ZipfileDecoder, 95) 96from airbyte_cdk.sources.declarative.decoders.composite_raw_decoder import ( 97 CompositeRawDecoder, 98 CsvParser, 99 GzipParser, 100 JsonItemsParser, 101 JsonLineParser, 102 JsonParser, 103 Parser, 104) 105from airbyte_cdk.sources.declarative.expanders.record_expander import ( 106 OnNoRecords, 107 RecordExpander, 108) 109from airbyte_cdk.sources.declarative.extractors import ( 110 DpathExtractor, 111 RecordFilter, 112 RecordSelector, 113 ResponseToFileExtractor, 114) 115from airbyte_cdk.sources.declarative.extractors.record_extractor import RecordExtractor 116from airbyte_cdk.sources.declarative.extractors.record_filter import ( 117 ClientSideIncrementalRecordFilterDecorator, 118) 119from airbyte_cdk.sources.declarative.incremental import ( 120 ConcurrentCursorFactory, 121 ConcurrentPerPartitionCursor, 122) 123from airbyte_cdk.sources.declarative.interpolation import InterpolatedString 124from airbyte_cdk.sources.declarative.interpolation.interpolated_mapping import InterpolatedMapping 125from airbyte_cdk.sources.declarative.migrations.legacy_to_per_partition_state_migration import ( 126 LegacyToPerPartitionStateMigration, 127) 128from airbyte_cdk.sources.declarative.models import ( 129 CustomStateMigration, 130 PaginationResetLimits, 131) 132from airbyte_cdk.sources.declarative.models.base_model_with_deprecations import ( 133 DEPRECATION_LOGS_TAG, 134 BaseModelWithDeprecations, 135) 136from airbyte_cdk.sources.declarative.models.declarative_component_schema import ( 137 Action1 as PaginationResetActionModel, 138) 139from airbyte_cdk.sources.declarative.models.declarative_component_schema import ( 140 AddedFieldDefinition as AddedFieldDefinitionModel, 141) 142from airbyte_cdk.sources.declarative.models.declarative_component_schema import ( 143 AddFields as AddFieldsModel, 144) 145from airbyte_cdk.sources.declarative.models.declarative_component_schema import ( 146 ApiKeyAuthenticator as ApiKeyAuthenticatorModel, 147) 148from airbyte_cdk.sources.declarative.models.declarative_component_schema import ( 149 AsyncJobStatusMap as AsyncJobStatusMapModel, 150) 151from airbyte_cdk.sources.declarative.models.declarative_component_schema import ( 152 AsyncRetriever as AsyncRetrieverModel, 153) 154from airbyte_cdk.sources.declarative.models.declarative_component_schema import ( 155 BasicHttpAuthenticator as BasicHttpAuthenticatorModel, 156) 157from airbyte_cdk.sources.declarative.models.declarative_component_schema import ( 158 BearerAuthenticator as BearerAuthenticatorModel, 159) 160from airbyte_cdk.sources.declarative.models.declarative_component_schema import ( 161 CheckDynamicStream as CheckDynamicStreamModel, 162) 163from airbyte_cdk.sources.declarative.models.declarative_component_schema import ( 164 CheckStream as CheckStreamModel, 165) 166from airbyte_cdk.sources.declarative.models.declarative_component_schema import ( 167 ComplexFieldType as ComplexFieldTypeModel, 168) 169from airbyte_cdk.sources.declarative.models.declarative_component_schema import ( 170 ComponentMappingDefinition as ComponentMappingDefinitionModel, 171) 172from airbyte_cdk.sources.declarative.models.declarative_component_schema import ( 173 CompositeErrorHandler as CompositeErrorHandlerModel, 174) 175from airbyte_cdk.sources.declarative.models.declarative_component_schema import ( 176 ConcurrencyLevel as ConcurrencyLevelModel, 177) 178from airbyte_cdk.sources.declarative.models.declarative_component_schema import ( 179 ConfigAddFields as ConfigAddFieldsModel, 180) 181from airbyte_cdk.sources.declarative.models.declarative_component_schema import ( 182 ConfigComponentsResolver as ConfigComponentsResolverModel, 183) 184from airbyte_cdk.sources.declarative.models.declarative_component_schema import ( 185 ConfigMigration as ConfigMigrationModel, 186) 187from airbyte_cdk.sources.declarative.models.declarative_component_schema import ( 188 ConfigRemapField as ConfigRemapFieldModel, 189) 190from airbyte_cdk.sources.declarative.models.declarative_component_schema import ( 191 ConfigRemoveFields as ConfigRemoveFieldsModel, 192) 193from airbyte_cdk.sources.declarative.models.declarative_component_schema import ( 194 ConstantBackoffStrategy as ConstantBackoffStrategyModel, 195) 196from airbyte_cdk.sources.declarative.models.declarative_component_schema import ( 197 CsvDecoder as CsvDecoderModel, 198) 199from airbyte_cdk.sources.declarative.models.declarative_component_schema import ( 200 CursorPagination as CursorPaginationModel, 201) 202from airbyte_cdk.sources.declarative.models.declarative_component_schema import ( 203 CustomAuthenticator as CustomAuthenticatorModel, 204) 205from airbyte_cdk.sources.declarative.models.declarative_component_schema import ( 206 CustomBackoffStrategy as CustomBackoffStrategyModel, 207) 208from airbyte_cdk.sources.declarative.models.declarative_component_schema import ( 209 CustomConfigTransformation as CustomConfigTransformationModel, 210) 211from airbyte_cdk.sources.declarative.models.declarative_component_schema import ( 212 CustomDecoder as CustomDecoderModel, 213) 214from airbyte_cdk.sources.declarative.models.declarative_component_schema import ( 215 CustomErrorHandler as CustomErrorHandlerModel, 216) 217from airbyte_cdk.sources.declarative.models.declarative_component_schema import ( 218 CustomPaginationStrategy as CustomPaginationStrategyModel, 219) 220from airbyte_cdk.sources.declarative.models.declarative_component_schema import ( 221 CustomPartitionRouter as CustomPartitionRouterModel, 222) 223from airbyte_cdk.sources.declarative.models.declarative_component_schema import ( 224 CustomRecordExtractor as CustomRecordExtractorModel, 225) 226from airbyte_cdk.sources.declarative.models.declarative_component_schema import ( 227 CustomRecordFilter as CustomRecordFilterModel, 228) 229from airbyte_cdk.sources.declarative.models.declarative_component_schema import ( 230 CustomRequester as CustomRequesterModel, 231) 232from airbyte_cdk.sources.declarative.models.declarative_component_schema import ( 233 CustomRetriever as CustomRetrieverModel, 234) 235from airbyte_cdk.sources.declarative.models.declarative_component_schema import ( 236 CustomSchemaLoader as CustomSchemaLoader, 237) 238from airbyte_cdk.sources.declarative.models.declarative_component_schema import ( 239 CustomSchemaNormalization as CustomSchemaNormalizationModel, 240) 241from airbyte_cdk.sources.declarative.models.declarative_component_schema import ( 242 CustomTransformation as CustomTransformationModel, 243) 244from airbyte_cdk.sources.declarative.models.declarative_component_schema import ( 245 CustomValidationStrategy as CustomValidationStrategyModel, 246) 247from airbyte_cdk.sources.declarative.models.declarative_component_schema import ( 248 DatetimeBasedCursor as DatetimeBasedCursorModel, 249) 250from airbyte_cdk.sources.declarative.models.declarative_component_schema import ( 251 DeclarativeStream as DeclarativeStreamModel, 252) 253from airbyte_cdk.sources.declarative.models.declarative_component_schema import ( 254 DefaultErrorHandler as DefaultErrorHandlerModel, 255) 256from airbyte_cdk.sources.declarative.models.declarative_component_schema import ( 257 DefaultPaginator as DefaultPaginatorModel, 258) 259from airbyte_cdk.sources.declarative.models.declarative_component_schema import ( 260 DpathExtractor as DpathExtractorModel, 261) 262from airbyte_cdk.sources.declarative.models.declarative_component_schema import ( 263 DpathFlattenFields as DpathFlattenFieldsModel, 264) 265from airbyte_cdk.sources.declarative.models.declarative_component_schema import ( 266 DpathValidator as DpathValidatorModel, 267) 268from airbyte_cdk.sources.declarative.models.declarative_component_schema import ( 269 DynamicSchemaLoader as DynamicSchemaLoaderModel, 270) 271from airbyte_cdk.sources.declarative.models.declarative_component_schema import ( 272 DynamicStreamCheckConfig as DynamicStreamCheckConfigModel, 273) 274from airbyte_cdk.sources.declarative.models.declarative_component_schema import ( 275 ExponentialBackoffStrategy as ExponentialBackoffStrategyModel, 276) 277from airbyte_cdk.sources.declarative.models.declarative_component_schema import ( 278 FileUploader as FileUploaderModel, 279) 280from airbyte_cdk.sources.declarative.models.declarative_component_schema import ( 281 FixedWindowCallRatePolicy as FixedWindowCallRatePolicyModel, 282) 283from airbyte_cdk.sources.declarative.models.declarative_component_schema import ( 284 FlattenFields as FlattenFieldsModel, 285) 286from airbyte_cdk.sources.declarative.models.declarative_component_schema import ( 287 GroupByKeyMergeStrategy as GroupByKeyMergeStrategyModel, 288) 289from airbyte_cdk.sources.declarative.models.declarative_component_schema import ( 290 GroupingPartitionRouter as GroupingPartitionRouterModel, 291) 292from airbyte_cdk.sources.declarative.models.declarative_component_schema import ( 293 GzipDecoder as GzipDecoderModel, 294) 295from airbyte_cdk.sources.declarative.models.declarative_component_schema import ( 296 HTTPAPIBudget as HTTPAPIBudgetModel, 297) 298from airbyte_cdk.sources.declarative.models.declarative_component_schema import ( 299 HttpComponentsResolver as HttpComponentsResolverModel, 300) 301from airbyte_cdk.sources.declarative.models.declarative_component_schema import ( 302 HttpRequester as HttpRequesterModel, 303) 304from airbyte_cdk.sources.declarative.models.declarative_component_schema import ( 305 HttpRequestRegexMatcher as HttpRequestRegexMatcherModel, 306) 307from airbyte_cdk.sources.declarative.models.declarative_component_schema import ( 308 HttpResponseFilter as HttpResponseFilterModel, 309) 310from airbyte_cdk.sources.declarative.models.declarative_component_schema import ( 311 IncrementingCountCursor as IncrementingCountCursorModel, 312) 313from airbyte_cdk.sources.declarative.models.declarative_component_schema import ( 314 InlineSchemaLoader as InlineSchemaLoaderModel, 315) 316from airbyte_cdk.sources.declarative.models.declarative_component_schema import ( 317 IterableDecoder as IterableDecoderModel, 318) 319from airbyte_cdk.sources.declarative.models.declarative_component_schema import ( 320 JsonDecoder as JsonDecoderModel, 321) 322from airbyte_cdk.sources.declarative.models.declarative_component_schema import ( 323 JsonFileSchemaLoader as JsonFileSchemaLoaderModel, 324) 325from airbyte_cdk.sources.declarative.models.declarative_component_schema import ( 326 JsonItemsDecoder as JsonItemsDecoderModel, 327) 328from airbyte_cdk.sources.declarative.models.declarative_component_schema import ( 329 JsonlDecoder as JsonlDecoderModel, 330) 331from airbyte_cdk.sources.declarative.models.declarative_component_schema import ( 332 JsonSchemaPropertySelector as JsonSchemaPropertySelectorModel, 333) 334from airbyte_cdk.sources.declarative.models.declarative_component_schema import ( 335 JwtAuthenticator as JwtAuthenticatorModel, 336) 337from airbyte_cdk.sources.declarative.models.declarative_component_schema import ( 338 JwtHeaders as JwtHeadersModel, 339) 340from airbyte_cdk.sources.declarative.models.declarative_component_schema import ( 341 JwtPayload as JwtPayloadModel, 342) 343from airbyte_cdk.sources.declarative.models.declarative_component_schema import ( 344 KeysReplace as KeysReplaceModel, 345) 346from airbyte_cdk.sources.declarative.models.declarative_component_schema import ( 347 KeysToLower as KeysToLowerModel, 348) 349from airbyte_cdk.sources.declarative.models.declarative_component_schema import ( 350 KeysToSnakeCase as KeysToSnakeCaseModel, 351) 352from airbyte_cdk.sources.declarative.models.declarative_component_schema import ( 353 LegacySessionTokenAuthenticator as LegacySessionTokenAuthenticatorModel, 354) 355from airbyte_cdk.sources.declarative.models.declarative_component_schema import ( 356 LegacyToPerPartitionStateMigration as LegacyToPerPartitionStateMigrationModel, 357) 358from airbyte_cdk.sources.declarative.models.declarative_component_schema import ( 359 ListPartitionRouter as ListPartitionRouterModel, 360) 361from airbyte_cdk.sources.declarative.models.declarative_component_schema import ( 362 MinMaxDatetime as MinMaxDatetimeModel, 363) 364from airbyte_cdk.sources.declarative.models.declarative_component_schema import ( 365 MovingWindowCallRatePolicy as MovingWindowCallRatePolicyModel, 366) 367from airbyte_cdk.sources.declarative.models.declarative_component_schema import ( 368 NoAuth as NoAuthModel, 369) 370from airbyte_cdk.sources.declarative.models.declarative_component_schema import ( 371 NoPagination as NoPaginationModel, 372) 373from airbyte_cdk.sources.declarative.models.declarative_component_schema import ( 374 OAuthAuthenticator as OAuthAuthenticatorModel, 375) 376from airbyte_cdk.sources.declarative.models.declarative_component_schema import ( 377 OffsetIncrement as OffsetIncrementModel, 378) 379from airbyte_cdk.sources.declarative.models.declarative_component_schema import ( 380 PageIncrement as PageIncrementModel, 381) 382from airbyte_cdk.sources.declarative.models.declarative_component_schema import ( 383 PaginationReset as PaginationResetModel, 384) 385from airbyte_cdk.sources.declarative.models.declarative_component_schema import ( 386 ParametrizedComponentsResolver as ParametrizedComponentsResolverModel, 387) 388from airbyte_cdk.sources.declarative.models.declarative_component_schema import ( 389 ParentStreamConfig as ParentStreamConfigModel, 390) 391from airbyte_cdk.sources.declarative.models.declarative_component_schema import ( 392 PredicateValidator as PredicateValidatorModel, 393) 394from airbyte_cdk.sources.declarative.models.declarative_component_schema import ( 395 PropertiesFromEndpoint as PropertiesFromEndpointModel, 396) 397from airbyte_cdk.sources.declarative.models.declarative_component_schema import ( 398 PropertyChunking as PropertyChunkingModel, 399) 400from airbyte_cdk.sources.declarative.models.declarative_component_schema import ( 401 PropertyLimitType as PropertyLimitTypeModel, 402) 403from airbyte_cdk.sources.declarative.models.declarative_component_schema import ( 404 QueryProperties as QueryPropertiesModel, 405) 406from airbyte_cdk.sources.declarative.models.declarative_component_schema import ( 407 Rate as RateModel, 408) 409from airbyte_cdk.sources.declarative.models.declarative_component_schema import ( 410 RecordExpander as RecordExpanderModel, 411) 412from airbyte_cdk.sources.declarative.models.declarative_component_schema import ( 413 RecordFilter as RecordFilterModel, 414) 415from airbyte_cdk.sources.declarative.models.declarative_component_schema import ( 416 RecordSelector as RecordSelectorModel, 417) 418from airbyte_cdk.sources.declarative.models.declarative_component_schema import ( 419 RefreshTokenUpdater as RefreshTokenUpdaterModel, 420) 421from airbyte_cdk.sources.declarative.models.declarative_component_schema import ( 422 RemoveFields as RemoveFieldsModel, 423) 424from airbyte_cdk.sources.declarative.models.declarative_component_schema import ( 425 RequestOption as RequestOptionModel, 426) 427from airbyte_cdk.sources.declarative.models.declarative_component_schema import ( 428 RequestPath as RequestPathModel, 429) 430from airbyte_cdk.sources.declarative.models.declarative_component_schema import ( 431 ResponseToFileExtractor as ResponseToFileExtractorModel, 432) 433from airbyte_cdk.sources.declarative.models.declarative_component_schema import ( 434 SchemaNormalization as SchemaNormalizationModel, 435) 436from airbyte_cdk.sources.declarative.models.declarative_component_schema import ( 437 SchemaTypeIdentifier as SchemaTypeIdentifierModel, 438) 439from airbyte_cdk.sources.declarative.models.declarative_component_schema import ( 440 SelectiveAuthenticator as SelectiveAuthenticatorModel, 441) 442from airbyte_cdk.sources.declarative.models.declarative_component_schema import ( 443 SessionTokenAuthenticator as SessionTokenAuthenticatorModel, 444) 445from airbyte_cdk.sources.declarative.models.declarative_component_schema import ( 446 SimpleRetriever as SimpleRetrieverModel, 447) 448from airbyte_cdk.sources.declarative.models.declarative_component_schema import Spec as SpecModel 449from airbyte_cdk.sources.declarative.models.declarative_component_schema import ( 450 StateDelegatingStream as StateDelegatingStreamModel, 451) 452from airbyte_cdk.sources.declarative.models.declarative_component_schema import ( 453 StreamConfig as StreamConfigModel, 454) 455from airbyte_cdk.sources.declarative.models.declarative_component_schema import ( 456 SubstreamPartitionRouter as SubstreamPartitionRouterModel, 457) 458from airbyte_cdk.sources.declarative.models.declarative_component_schema import ( 459 TypesMap as TypesMapModel, 460) 461from airbyte_cdk.sources.declarative.models.declarative_component_schema import ( 462 UnlimitedCallRatePolicy as UnlimitedCallRatePolicyModel, 463) 464from airbyte_cdk.sources.declarative.models.declarative_component_schema import ( 465 ValidateAdheresToSchema as ValidateAdheresToSchemaModel, 466) 467from airbyte_cdk.sources.declarative.models.declarative_component_schema import ValueType 468from airbyte_cdk.sources.declarative.models.declarative_component_schema import ( 469 WaitTimeFromHeader as WaitTimeFromHeaderModel, 470) 471from airbyte_cdk.sources.declarative.models.declarative_component_schema import ( 472 WaitUntilTimeFromHeader as WaitUntilTimeFromHeaderModel, 473) 474from airbyte_cdk.sources.declarative.models.declarative_component_schema import ( 475 XmlDecoder as XmlDecoderModel, 476) 477from airbyte_cdk.sources.declarative.models.declarative_component_schema import ( 478 ZipfileDecoder as ZipfileDecoderModel, 479) 480from airbyte_cdk.sources.declarative.partition_routers import ( 481 CartesianProductStreamSlicer, 482 GroupingPartitionRouter, 483 ListPartitionRouter, 484 PartitionRouter, 485 SinglePartitionRouter, 486 SubstreamPartitionRouter, 487) 488from airbyte_cdk.sources.declarative.partition_routers.async_job_partition_router import ( 489 AsyncJobPartitionRouter, 490) 491from airbyte_cdk.sources.declarative.partition_routers.substream_partition_router import ( 492 ParentStreamConfig, 493) 494from airbyte_cdk.sources.declarative.requesters import HttpRequester, RequestOption 495from airbyte_cdk.sources.declarative.requesters.error_handlers import ( 496 CompositeErrorHandler, 497 DefaultErrorHandler, 498 HttpResponseFilter, 499) 500from airbyte_cdk.sources.declarative.requesters.error_handlers.backoff_strategies import ( 501 ConstantBackoffStrategy, 502 ExponentialBackoffStrategy, 503 WaitTimeFromHeaderBackoffStrategy, 504 WaitUntilTimeFromHeaderBackoffStrategy, 505) 506from airbyte_cdk.sources.declarative.requesters.http_job_repository import AsyncHttpJobRepository 507from airbyte_cdk.sources.declarative.requesters.paginators import ( 508 DefaultPaginator, 509 NoPagination, 510 PaginatorTestReadDecorator, 511) 512from airbyte_cdk.sources.declarative.requesters.paginators.strategies import ( 513 CursorPaginationStrategy, 514 CursorStopCondition, 515 OffsetIncrement, 516 PageIncrement, 517 StopConditionPaginationStrategyDecorator, 518) 519from airbyte_cdk.sources.declarative.requesters.query_properties import ( 520 PropertiesFromEndpoint, 521 PropertyChunking, 522 QueryProperties, 523) 524from airbyte_cdk.sources.declarative.requesters.query_properties.property_chunking import ( 525 PropertyLimitType, 526) 527from airbyte_cdk.sources.declarative.requesters.query_properties.property_selector import ( 528 JsonSchemaPropertySelector, 529) 530from airbyte_cdk.sources.declarative.requesters.query_properties.strategies import ( 531 GroupByKey, 532) 533from airbyte_cdk.sources.declarative.requesters.request_option import RequestOptionType 534from airbyte_cdk.sources.declarative.requesters.request_options import ( 535 DatetimeBasedRequestOptionsProvider, 536 DefaultRequestOptionsProvider, 537 InterpolatedRequestOptionsProvider, 538 RequestOptionsProvider, 539) 540from airbyte_cdk.sources.declarative.requesters.request_options.per_partition_request_option_provider import ( 541 PerPartitionRequestOptionsProvider, 542) 543from airbyte_cdk.sources.declarative.requesters.request_path import RequestPath 544from airbyte_cdk.sources.declarative.requesters.requester import HttpMethod, Requester 545from airbyte_cdk.sources.declarative.resolvers import ( 546 ComponentMappingDefinition, 547 ConfigComponentsResolver, 548 HttpComponentsResolver, 549 ParametrizedComponentsResolver, 550 StreamConfig, 551 StreamParametersDefinition, 552) 553from airbyte_cdk.sources.declarative.retrievers import ( 554 AsyncRetriever, 555 LazySimpleRetriever, 556 SimpleRetriever, 557) 558from airbyte_cdk.sources.declarative.retrievers.file_uploader import ( 559 ConnectorBuilderFileUploader, 560 DefaultFileUploader, 561 FileUploader, 562 LocalFileSystemFileWriter, 563 NoopFileWriter, 564) 565from airbyte_cdk.sources.declarative.retrievers.pagination_tracker import PaginationTracker 566from airbyte_cdk.sources.declarative.schema import ( 567 ComplexFieldType, 568 DefaultSchemaLoader, 569 DynamicSchemaLoader, 570 InlineSchemaLoader, 571 JsonFileSchemaLoader, 572 SchemaLoader, 573 SchemaTypeIdentifier, 574 TypesMap, 575) 576from airbyte_cdk.sources.declarative.schema.caching_schema_loader_decorator import ( 577 CachingSchemaLoaderDecorator, 578) 579from airbyte_cdk.sources.declarative.schema.composite_schema_loader import CompositeSchemaLoader 580from airbyte_cdk.sources.declarative.spec import ConfigMigration, Spec 581from airbyte_cdk.sources.declarative.stream_slicers import ( 582 StreamSlicer, 583 StreamSlicerTestReadDecorator, 584) 585from airbyte_cdk.sources.declarative.stream_slicers.declarative_partition_generator import ( 586 DeclarativePartitionFactory, 587 StreamSlicerPartitionGenerator, 588) 589from airbyte_cdk.sources.declarative.transformations import ( 590 AddFields, 591 RecordTransformation, 592 RemoveFields, 593) 594from airbyte_cdk.sources.declarative.transformations.add_fields import AddedFieldDefinition 595from airbyte_cdk.sources.declarative.transformations.config_transformations import ( 596 ConfigAddFields, 597 ConfigRemapField, 598 ConfigRemoveFields, 599) 600from airbyte_cdk.sources.declarative.transformations.config_transformations.config_transformation import ( 601 ConfigTransformation, 602) 603from airbyte_cdk.sources.declarative.transformations.dpath_flatten_fields import ( 604 DpathFlattenFields, 605 KeyTransformation, 606) 607from airbyte_cdk.sources.declarative.transformations.flatten_fields import ( 608 FlattenFields, 609) 610from airbyte_cdk.sources.declarative.transformations.keys_replace_transformation import ( 611 KeysReplaceTransformation, 612) 613from airbyte_cdk.sources.declarative.transformations.keys_to_lower_transformation import ( 614 KeysToLowerTransformation, 615) 616from airbyte_cdk.sources.declarative.transformations.keys_to_snake_transformation import ( 617 KeysToSnakeCaseTransformation, 618) 619from airbyte_cdk.sources.declarative.validators import ( 620 DpathValidator, 621 PredicateValidator, 622 ValidateAdheresToSchema, 623) 624from airbyte_cdk.sources.http_logger import format_http_message 625from airbyte_cdk.sources.message import ( 626 InMemoryMessageRepository, 627 LogAppenderMessageRepositoryDecorator, 628 MessageRepository, 629 NoopMessageRepository, 630) 631from airbyte_cdk.sources.message.repository import StateFilteringMessageRepository 632from airbyte_cdk.sources.streams.call_rate import ( 633 APIBudget, 634 FixedWindowCallRatePolicy, 635 HttpAPIBudget, 636 HttpRequestRegexMatcher, 637 MovingWindowCallRatePolicy, 638 Rate, 639 UnlimitedCallRatePolicy, 640) 641from airbyte_cdk.sources.streams.concurrent.abstract_stream import AbstractStream 642from airbyte_cdk.sources.streams.concurrent.clamping import ( 643 ClampingEndProvider, 644 ClampingStrategy, 645 DayClampingStrategy, 646 MonthClampingStrategy, 647 NoClamping, 648 WeekClampingStrategy, 649 Weekday, 650) 651from airbyte_cdk.sources.streams.concurrent.cursor import ( 652 ConcurrentCursor, 653 Cursor, 654 CursorField, 655 FinalStateCursor, 656) 657from airbyte_cdk.sources.streams.concurrent.default_stream import DefaultStream 658from airbyte_cdk.sources.streams.concurrent.helpers import get_primary_key_from_stream 659from airbyte_cdk.sources.streams.concurrent.partitions.stream_slicer import ( 660 StreamSlicer as ConcurrentStreamSlicer, 661) 662from airbyte_cdk.sources.streams.concurrent.state_converters.datetime_stream_state_converter import ( 663 CustomFormatConcurrentStreamStateConverter, 664 DateTimeStreamStateConverter, 665) 666from airbyte_cdk.sources.streams.concurrent.state_converters.incrementing_count_stream_state_converter import ( 667 IncrementingCountStreamStateConverter, 668) 669from airbyte_cdk.sources.streams.http.error_handlers.response_models import ResponseAction 670from airbyte_cdk.sources.types import Config 671from airbyte_cdk.sources.utils.transform import TransformConfig, TypeTransformer 672 673ComponentDefinition = Mapping[str, Any] 674 675SCHEMA_TRANSFORMER_TYPE_MAPPING = { 676 SchemaNormalizationModel.None_: TransformConfig.NoTransform, 677 SchemaNormalizationModel.Default: TransformConfig.DefaultSchemaNormalization, 678} 679_NO_STREAM_SLICING = SinglePartitionRouter(parameters={}) 680 681# Ideally this should use the value defined in ConcurrentDeclarativeSource, but 682# this would be a circular import 683MAX_SLICES = 5 684 685LOGGER = logging.getLogger(f"airbyte.model_to_component_factory") 686 687 688class ModelToComponentFactory: 689 EPOCH_DATETIME_FORMAT = "%s" 690 691 def __init__( 692 self, 693 limit_pages_fetched_per_slice: Optional[int] = None, 694 limit_slices_fetched: Optional[int] = None, 695 emit_connector_builder_messages: bool = False, 696 disable_retries: bool = False, 697 disable_cache: bool = False, 698 message_repository: Optional[MessageRepository] = None, 699 connector_state_manager: Optional[ConnectorStateManager] = None, 700 max_concurrent_async_job_count: Optional[int] = None, 701 configured_catalog: Optional[ConfiguredAirbyteCatalog] = None, 702 api_budget: Optional[APIBudget] = None, 703 ): 704 self._init_mappings() 705 self._limit_pages_fetched_per_slice = limit_pages_fetched_per_slice 706 self._limit_slices_fetched = limit_slices_fetched 707 self._emit_connector_builder_messages = emit_connector_builder_messages 708 self._disable_retries = disable_retries 709 self._disable_cache = disable_cache 710 self._message_repository = message_repository or InMemoryMessageRepository( 711 self._evaluate_log_level(emit_connector_builder_messages) 712 ) 713 self._stream_name_to_configured_stream = self._create_stream_name_to_configured_stream( 714 configured_catalog 715 ) 716 self._connector_state_manager = connector_state_manager or ConnectorStateManager() 717 self._api_budget: Optional[Union[APIBudget]] = api_budget 718 self._job_tracker: JobTracker = JobTracker(max_concurrent_async_job_count or 1) 719 # placeholder for deprecation warnings 720 self._collected_deprecation_logs: List[ConnectorBuilderLogMessage] = [] 721 722 def _init_mappings(self) -> None: 723 self.PYDANTIC_MODEL_TO_CONSTRUCTOR: Mapping[Type[BaseModel], Callable[..., Any]] = { 724 AddedFieldDefinitionModel: self.create_added_field_definition, 725 AddFieldsModel: self.create_add_fields, 726 ApiKeyAuthenticatorModel: self.create_api_key_authenticator, 727 BasicHttpAuthenticatorModel: self.create_basic_http_authenticator, 728 BearerAuthenticatorModel: self.create_bearer_authenticator, 729 CheckStreamModel: self.create_check_stream, 730 DynamicStreamCheckConfigModel: self.create_dynamic_stream_check_config, 731 CheckDynamicStreamModel: self.create_check_dynamic_stream, 732 CompositeErrorHandlerModel: self.create_composite_error_handler, 733 ConcurrencyLevelModel: self.create_concurrency_level, 734 ConfigMigrationModel: self.create_config_migration, 735 ConfigAddFieldsModel: self.create_config_add_fields, 736 ConfigRemapFieldModel: self.create_config_remap_field, 737 ConfigRemoveFieldsModel: self.create_config_remove_fields, 738 ConstantBackoffStrategyModel: self.create_constant_backoff_strategy, 739 CsvDecoderModel: self.create_csv_decoder, 740 CursorPaginationModel: self.create_cursor_pagination, 741 CustomAuthenticatorModel: self.create_custom_component, 742 CustomBackoffStrategyModel: self.create_custom_component, 743 CustomDecoderModel: self.create_custom_component, 744 CustomErrorHandlerModel: self.create_custom_component, 745 CustomRecordExtractorModel: self.create_custom_component, 746 CustomRecordFilterModel: self.create_custom_component, 747 CustomRequesterModel: self.create_custom_component, 748 CustomRetrieverModel: self.create_custom_component, 749 CustomSchemaLoader: self.create_custom_component, 750 CustomSchemaNormalizationModel: self.create_custom_component, 751 CustomStateMigration: self.create_custom_component, 752 CustomPaginationStrategyModel: self.create_custom_component, 753 CustomPartitionRouterModel: self.create_custom_component, 754 CustomTransformationModel: self.create_custom_component, 755 CustomValidationStrategyModel: self.create_custom_component, 756 CustomConfigTransformationModel: self.create_custom_component, 757 DeclarativeStreamModel: self.create_default_stream, 758 DefaultErrorHandlerModel: self.create_default_error_handler, 759 DefaultPaginatorModel: self.create_default_paginator, 760 DpathExtractorModel: self.create_dpath_extractor, 761 DpathValidatorModel: self.create_dpath_validator, 762 ResponseToFileExtractorModel: self.create_response_to_file_extractor, 763 ExponentialBackoffStrategyModel: self.create_exponential_backoff_strategy, 764 SessionTokenAuthenticatorModel: self.create_session_token_authenticator, 765 GroupByKeyMergeStrategyModel: self.create_group_by_key, 766 HttpRequesterModel: self.create_http_requester, 767 HttpResponseFilterModel: self.create_http_response_filter, 768 InlineSchemaLoaderModel: self.create_inline_schema_loader, 769 JsonDecoderModel: self.create_json_decoder, 770 JsonItemsDecoderModel: self.create_json_items_decoder, 771 JsonlDecoderModel: self.create_jsonl_decoder, 772 JsonSchemaPropertySelectorModel: self.create_json_schema_property_selector, 773 GzipDecoderModel: self.create_gzip_decoder, 774 KeysToLowerModel: self.create_keys_to_lower_transformation, 775 KeysToSnakeCaseModel: self.create_keys_to_snake_transformation, 776 KeysReplaceModel: self.create_keys_replace_transformation, 777 FlattenFieldsModel: self.create_flatten_fields, 778 DpathFlattenFieldsModel: self.create_dpath_flatten_fields, 779 IterableDecoderModel: self.create_iterable_decoder, 780 XmlDecoderModel: self.create_xml_decoder, 781 JsonFileSchemaLoaderModel: self.create_json_file_schema_loader, 782 DynamicSchemaLoaderModel: self.create_dynamic_schema_loader, 783 SchemaTypeIdentifierModel: self.create_schema_type_identifier, 784 TypesMapModel: self.create_types_map, 785 ComplexFieldTypeModel: self.create_complex_field_type, 786 JwtAuthenticatorModel: self.create_jwt_authenticator, 787 LegacyToPerPartitionStateMigrationModel: self.create_legacy_to_per_partition_state_migration, 788 ListPartitionRouterModel: self.create_list_partition_router, 789 MinMaxDatetimeModel: self.create_min_max_datetime, 790 NoAuthModel: self.create_no_auth, 791 NoPaginationModel: self.create_no_pagination, 792 OAuthAuthenticatorModel: self.create_oauth_authenticator, 793 OffsetIncrementModel: self.create_offset_increment, 794 PageIncrementModel: self.create_page_increment, 795 ParentStreamConfigModel: self.create_parent_stream_config_with_substream_wrapper, 796 PredicateValidatorModel: self.create_predicate_validator, 797 PropertiesFromEndpointModel: self.create_properties_from_endpoint, 798 PropertyChunkingModel: self.create_property_chunking, 799 QueryPropertiesModel: self.create_query_properties, 800 RecordExpanderModel: self.create_record_expander, 801 RecordFilterModel: self.create_record_filter, 802 RecordSelectorModel: self.create_record_selector, 803 RemoveFieldsModel: self.create_remove_fields, 804 RequestPathModel: self.create_request_path, 805 RequestOptionModel: self.create_request_option, 806 LegacySessionTokenAuthenticatorModel: self.create_legacy_session_token_authenticator, 807 SelectiveAuthenticatorModel: self.create_selective_authenticator, 808 SimpleRetrieverModel: self.create_simple_retriever, 809 StateDelegatingStreamModel: self.create_state_delegating_stream, 810 SpecModel: self.create_spec, 811 SubstreamPartitionRouterModel: self.create_substream_partition_router, 812 ValidateAdheresToSchemaModel: self.create_validate_adheres_to_schema, 813 WaitTimeFromHeaderModel: self.create_wait_time_from_header, 814 WaitUntilTimeFromHeaderModel: self.create_wait_until_time_from_header, 815 AsyncRetrieverModel: self.create_async_retriever, 816 HttpComponentsResolverModel: self.create_http_components_resolver, 817 ConfigComponentsResolverModel: self.create_config_components_resolver, 818 ParametrizedComponentsResolverModel: self.create_parametrized_components_resolver, 819 StreamConfigModel: self.create_stream_config, 820 ComponentMappingDefinitionModel: self.create_components_mapping_definition, 821 ZipfileDecoderModel: self.create_zipfile_decoder, 822 HTTPAPIBudgetModel: self.create_http_api_budget, 823 FileUploaderModel: self.create_file_uploader, 824 FixedWindowCallRatePolicyModel: self.create_fixed_window_call_rate_policy, 825 MovingWindowCallRatePolicyModel: self.create_moving_window_call_rate_policy, 826 UnlimitedCallRatePolicyModel: self.create_unlimited_call_rate_policy, 827 RateModel: self.create_rate, 828 HttpRequestRegexMatcherModel: self.create_http_request_matcher, 829 GroupingPartitionRouterModel: self.create_grouping_partition_router, 830 } 831 832 # Needed for the case where we need to perform a second parse on the fields of a custom component 833 self.TYPE_NAME_TO_MODEL = {cls.__name__: cls for cls in self.PYDANTIC_MODEL_TO_CONSTRUCTOR} 834 835 @staticmethod 836 def _create_stream_name_to_configured_stream( 837 configured_catalog: Optional[ConfiguredAirbyteCatalog], 838 ) -> Mapping[str, ConfiguredAirbyteStream]: 839 return ( 840 {stream.stream.name: stream for stream in configured_catalog.streams} 841 if configured_catalog 842 else {} 843 ) 844 845 def create_component( 846 self, 847 model_type: Type[BaseModel], 848 component_definition: ComponentDefinition, 849 config: Config, 850 **kwargs: Any, 851 ) -> Any: 852 """ 853 Takes a given Pydantic model type and Mapping representing a component definition and creates a declarative component and 854 subcomponents which will be used at runtime. This is done by first parsing the mapping into a Pydantic model and then creating 855 creating declarative components from that model. 856 857 :param model_type: The type of declarative component that is being initialized 858 :param component_definition: The mapping that represents a declarative component 859 :param config: The connector config that is provided by the customer 860 :return: The declarative component to be used at runtime 861 """ 862 863 component_type = component_definition.get("type") 864 if component_definition.get("type") != model_type.__name__: 865 raise ValueError( 866 f"Expected manifest component of type {model_type.__name__}, but received {component_type} instead" 867 ) 868 869 declarative_component_model = model_type.parse_obj(component_definition) 870 871 if not isinstance(declarative_component_model, model_type): 872 raise ValueError( 873 f"Expected {model_type.__name__} component, but received {declarative_component_model.__class__.__name__}" 874 ) 875 876 return self._create_component_from_model( 877 model=declarative_component_model, config=config, **kwargs 878 ) 879 880 def _create_component_from_model(self, model: BaseModel, config: Config, **kwargs: Any) -> Any: 881 if model.__class__ not in self.PYDANTIC_MODEL_TO_CONSTRUCTOR: 882 raise ValueError( 883 f"{model.__class__} with attributes {model} is not a valid component type" 884 ) 885 component_constructor = self.PYDANTIC_MODEL_TO_CONSTRUCTOR.get(model.__class__) 886 if not component_constructor: 887 raise ValueError(f"Could not find constructor for {model.__class__}") 888 889 # collect deprecation warnings for supported models. 890 if isinstance(model, BaseModelWithDeprecations): 891 self._collect_model_deprecations(model) 892 893 return component_constructor(model=model, config=config, **kwargs) 894 895 def get_model_deprecations(self) -> List[ConnectorBuilderLogMessage]: 896 """ 897 Returns the deprecation warnings that were collected during the creation of components. 898 """ 899 return self._collected_deprecation_logs 900 901 def _collect_model_deprecations(self, model: BaseModelWithDeprecations) -> None: 902 """ 903 Collects deprecation logs from the given model and appends any new logs to the internal collection. 904 905 This method checks if the provided model has deprecation logs (identified by the presence of the DEPRECATION_LOGS_TAG attribute and a non-None `_deprecation_logs` property). It iterates through each deprecation log in the model and appends it to the `_collected_deprecation_logs` list if it has not already been collected, ensuring that duplicate logs are avoided. 906 907 Args: 908 model (BaseModelWithDeprecations): The model instance from which to collect deprecation logs. 909 """ 910 if hasattr(model, DEPRECATION_LOGS_TAG) and model._deprecation_logs is not None: 911 for log in model._deprecation_logs: 912 # avoid duplicates for deprecation logs observed. 913 if log not in self._collected_deprecation_logs: 914 self._collected_deprecation_logs.append(log) 915 916 def create_config_migration( 917 self, model: ConfigMigrationModel, config: Config 918 ) -> ConfigMigration: 919 transformations: List[ConfigTransformation] = [ 920 self._create_component_from_model(transformation, config) 921 for transformation in model.transformations 922 ] 923 924 return ConfigMigration( 925 description=model.description, 926 transformations=transformations, 927 ) 928 929 def create_config_add_fields( 930 self, model: ConfigAddFieldsModel, config: Config, **kwargs: Any 931 ) -> ConfigAddFields: 932 fields = [self._create_component_from_model(field, config) for field in model.fields] 933 return ConfigAddFields( 934 fields=fields, 935 condition=model.condition or "", 936 ) 937 938 @staticmethod 939 def create_config_remove_fields( 940 model: ConfigRemoveFieldsModel, config: Config, **kwargs: Any 941 ) -> ConfigRemoveFields: 942 return ConfigRemoveFields( 943 field_pointers=model.field_pointers, 944 condition=model.condition or "", 945 ) 946 947 @staticmethod 948 def create_config_remap_field( 949 model: ConfigRemapFieldModel, config: Config, **kwargs: Any 950 ) -> ConfigRemapField: 951 mapping = cast(Mapping[str, Any], model.map) 952 return ConfigRemapField( 953 map=mapping, 954 field_path=model.field_path, 955 config=config, 956 ) 957 958 def create_dpath_validator(self, model: DpathValidatorModel, config: Config) -> DpathValidator: 959 strategy = self._create_component_from_model(model.validation_strategy, config) 960 961 return DpathValidator( 962 field_path=model.field_path, 963 strategy=strategy, 964 ) 965 966 def create_predicate_validator( 967 self, model: PredicateValidatorModel, config: Config 968 ) -> PredicateValidator: 969 strategy = self._create_component_from_model(model.validation_strategy, config) 970 971 return PredicateValidator( 972 value=model.value, 973 strategy=strategy, 974 ) 975 976 @staticmethod 977 def create_validate_adheres_to_schema( 978 model: ValidateAdheresToSchemaModel, config: Config, **kwargs: Any 979 ) -> ValidateAdheresToSchema: 980 base_schema = cast(Mapping[str, Any], model.base_schema) 981 return ValidateAdheresToSchema( 982 schema=base_schema, 983 ) 984 985 @staticmethod 986 def create_added_field_definition( 987 model: AddedFieldDefinitionModel, config: Config, **kwargs: Any 988 ) -> AddedFieldDefinition: 989 interpolated_value = InterpolatedString.create( 990 model.value, parameters=model.parameters or {} 991 ) 992 return AddedFieldDefinition( 993 path=model.path, 994 value=interpolated_value, 995 value_type=ModelToComponentFactory._json_schema_type_name_to_type(model.value_type), 996 parameters=model.parameters or {}, 997 ) 998 999 def create_add_fields(self, model: AddFieldsModel, config: Config, **kwargs: Any) -> AddFields: 1000 added_field_definitions = [ 1001 self._create_component_from_model( 1002 model=added_field_definition_model, 1003 value_type=ModelToComponentFactory._json_schema_type_name_to_type( 1004 added_field_definition_model.value_type 1005 ), 1006 config=config, 1007 ) 1008 for added_field_definition_model in model.fields 1009 ] 1010 return AddFields( 1011 fields=added_field_definitions, 1012 condition=model.condition or "", 1013 parameters=model.parameters or {}, 1014 ) 1015 1016 def create_keys_to_lower_transformation( 1017 self, model: KeysToLowerModel, config: Config, **kwargs: Any 1018 ) -> KeysToLowerTransformation: 1019 return KeysToLowerTransformation() 1020 1021 def create_keys_to_snake_transformation( 1022 self, model: KeysToSnakeCaseModel, config: Config, **kwargs: Any 1023 ) -> KeysToSnakeCaseTransformation: 1024 return KeysToSnakeCaseTransformation() 1025 1026 def create_keys_replace_transformation( 1027 self, model: KeysReplaceModel, config: Config, **kwargs: Any 1028 ) -> KeysReplaceTransformation: 1029 return KeysReplaceTransformation( 1030 old=model.old, new=model.new, parameters=model.parameters or {} 1031 ) 1032 1033 def create_flatten_fields( 1034 self, model: FlattenFieldsModel, config: Config, **kwargs: Any 1035 ) -> FlattenFields: 1036 return FlattenFields( 1037 flatten_lists=model.flatten_lists if model.flatten_lists is not None else True 1038 ) 1039 1040 def create_dpath_flatten_fields( 1041 self, model: DpathFlattenFieldsModel, config: Config, **kwargs: Any 1042 ) -> DpathFlattenFields: 1043 model_field_path: List[Union[InterpolatedString, str]] = [x for x in model.field_path] 1044 key_transformation = ( 1045 KeyTransformation( 1046 config=config, 1047 prefix=model.key_transformation.prefix, 1048 suffix=model.key_transformation.suffix, 1049 parameters=model.parameters or {}, 1050 ) 1051 if model.key_transformation is not None 1052 else None 1053 ) 1054 return DpathFlattenFields( 1055 config=config, 1056 field_path=model_field_path, 1057 delete_origin_value=model.delete_origin_value 1058 if model.delete_origin_value is not None 1059 else False, 1060 replace_record=model.replace_record if model.replace_record is not None else False, 1061 key_transformation=key_transformation, 1062 parameters=model.parameters or {}, 1063 ) 1064 1065 @staticmethod 1066 def _json_schema_type_name_to_type(value_type: Optional[ValueType]) -> Optional[Type[Any]]: 1067 if not value_type: 1068 return None 1069 names_to_types = { 1070 ValueType.string: str, 1071 ValueType.number: float, 1072 ValueType.integer: int, 1073 ValueType.boolean: bool, 1074 } 1075 return names_to_types[value_type] 1076 1077 def create_api_key_authenticator( 1078 self, 1079 model: ApiKeyAuthenticatorModel, 1080 config: Config, 1081 token_provider: Optional[TokenProvider] = None, 1082 **kwargs: Any, 1083 ) -> ApiKeyAuthenticator: 1084 if model.inject_into is None and model.header is None: 1085 raise ValueError( 1086 "Expected either inject_into or header to be set for ApiKeyAuthenticator" 1087 ) 1088 1089 if model.inject_into is not None and model.header is not None: 1090 raise ValueError( 1091 "inject_into and header cannot be set both for ApiKeyAuthenticator - remove the deprecated header option" 1092 ) 1093 1094 if token_provider is not None and model.api_token != "": 1095 raise ValueError( 1096 "If token_provider is set, api_token is ignored and has to be set to empty string." 1097 ) 1098 1099 request_option = ( 1100 self._create_component_from_model( 1101 model.inject_into, config, parameters=model.parameters or {} 1102 ) 1103 if model.inject_into 1104 else RequestOption( 1105 inject_into=RequestOptionType.header, 1106 field_name=model.header or "", 1107 parameters=model.parameters or {}, 1108 ) 1109 ) 1110 1111 return ApiKeyAuthenticator( 1112 token_provider=( 1113 token_provider 1114 if token_provider is not None 1115 else InterpolatedStringTokenProvider( 1116 api_token=model.api_token or "", 1117 config=config, 1118 parameters=model.parameters or {}, 1119 ) 1120 ), 1121 request_option=request_option, 1122 config=config, 1123 parameters=model.parameters or {}, 1124 ) 1125 1126 def create_legacy_to_per_partition_state_migration( 1127 self, 1128 model: LegacyToPerPartitionStateMigrationModel, 1129 config: Mapping[str, Any], 1130 declarative_stream: DeclarativeStreamModel, 1131 ) -> LegacyToPerPartitionStateMigration: 1132 retriever = declarative_stream.retriever 1133 if not isinstance(retriever, (SimpleRetrieverModel, AsyncRetrieverModel)): 1134 raise ValueError( 1135 f"LegacyToPerPartitionStateMigrations can only be applied on a DeclarativeStream with a SimpleRetriever or AsyncRetriever. Got {type(retriever)}" 1136 ) 1137 partition_router = retriever.partition_router 1138 if not isinstance( 1139 partition_router, (SubstreamPartitionRouterModel, CustomPartitionRouterModel) 1140 ): 1141 raise ValueError( 1142 f"LegacyToPerPartitionStateMigrations can only be applied on a SimpleRetriever with a Substream partition router. Got {type(partition_router)}" 1143 ) 1144 if not hasattr(partition_router, "parent_stream_configs"): 1145 raise ValueError( 1146 "LegacyToPerPartitionStateMigrations can only be applied with a parent stream configuration." 1147 ) 1148 1149 if not hasattr(declarative_stream, "incremental_sync"): 1150 raise ValueError( 1151 "LegacyToPerPartitionStateMigrations can only be applied with an incremental_sync configuration." 1152 ) 1153 1154 return LegacyToPerPartitionStateMigration( 1155 partition_router, # type: ignore # was already checked above 1156 declarative_stream.incremental_sync, # type: ignore # was already checked. Migration can be applied only to incremental streams. 1157 config, 1158 declarative_stream.parameters, # type: ignore # different type is expected here Mapping[str, Any], got Dict[str, Any] 1159 ) 1160 1161 def create_session_token_authenticator( 1162 self, model: SessionTokenAuthenticatorModel, config: Config, name: str, **kwargs: Any 1163 ) -> Union[ApiKeyAuthenticator, BearerAuthenticator]: 1164 decoder = ( 1165 self._create_component_from_model(model=model.decoder, config=config) 1166 if model.decoder 1167 else JsonDecoder(parameters={}) 1168 ) 1169 login_requester = self._create_component_from_model( 1170 model=model.login_requester, 1171 config=config, 1172 name=f"{name}_login_requester", 1173 decoder=decoder, 1174 ) 1175 token_provider = SessionTokenProvider( 1176 login_requester=login_requester, 1177 session_token_path=model.session_token_path, 1178 expiration_duration=parse_duration(model.expiration_duration) 1179 if model.expiration_duration 1180 else None, 1181 parameters=model.parameters or {}, 1182 message_repository=self._message_repository, 1183 decoder=decoder, 1184 ) 1185 if model.request_authentication.type == "Bearer": 1186 return ModelToComponentFactory.create_bearer_authenticator( 1187 BearerAuthenticatorModel(type="BearerAuthenticator", api_token=""), # type: ignore # $parameters has a default value 1188 config, 1189 token_provider=token_provider, 1190 ) 1191 else: 1192 # Get the api_token template if specified, default to just the session token 1193 api_token_template = ( 1194 getattr(model.request_authentication, "api_token", None) or "{{ session_token }}" 1195 ) 1196 final_token_provider: TokenProvider = InterpolatedSessionTokenProvider( 1197 config=config, 1198 api_token=api_token_template, 1199 session_token_provider=token_provider, 1200 parameters=model.parameters or {}, 1201 ) 1202 return self.create_api_key_authenticator( 1203 ApiKeyAuthenticatorModel( 1204 type="ApiKeyAuthenticator", 1205 api_token="", 1206 inject_into=model.request_authentication.inject_into, 1207 ), # type: ignore # $parameters and headers default to None 1208 config=config, 1209 token_provider=final_token_provider, 1210 ) 1211 1212 @staticmethod 1213 def create_basic_http_authenticator( 1214 model: BasicHttpAuthenticatorModel, config: Config, **kwargs: Any 1215 ) -> BasicHttpAuthenticator: 1216 return BasicHttpAuthenticator( 1217 password=model.password or "", 1218 username=model.username, 1219 config=config, 1220 parameters=model.parameters or {}, 1221 ) 1222 1223 @staticmethod 1224 def create_bearer_authenticator( 1225 model: BearerAuthenticatorModel, 1226 config: Config, 1227 token_provider: Optional[TokenProvider] = None, 1228 **kwargs: Any, 1229 ) -> BearerAuthenticator: 1230 if token_provider is not None and model.api_token != "": 1231 raise ValueError( 1232 "If token_provider is set, api_token is ignored and has to be set to empty string." 1233 ) 1234 return BearerAuthenticator( 1235 token_provider=( 1236 token_provider 1237 if token_provider is not None 1238 else InterpolatedStringTokenProvider( 1239 api_token=model.api_token or "", 1240 config=config, 1241 parameters=model.parameters or {}, 1242 ) 1243 ), 1244 config=config, 1245 parameters=model.parameters or {}, 1246 ) 1247 1248 @staticmethod 1249 def create_dynamic_stream_check_config( 1250 model: DynamicStreamCheckConfigModel, config: Config, **kwargs: Any 1251 ) -> DynamicStreamCheckConfig: 1252 return DynamicStreamCheckConfig( 1253 dynamic_stream_name=model.dynamic_stream_name, 1254 stream_count=model.stream_count, 1255 ) 1256 1257 def create_check_stream( 1258 self, model: CheckStreamModel, config: Config, **kwargs: Any 1259 ) -> CheckStream: 1260 if model.dynamic_streams_check_configs is None and model.stream_names is None: 1261 raise ValueError( 1262 "Expected either stream_names or dynamic_streams_check_configs to be set for CheckStream" 1263 ) 1264 1265 dynamic_streams_check_configs = ( 1266 [ 1267 self._create_component_from_model(model=dynamic_stream_check_config, config=config) 1268 for dynamic_stream_check_config in model.dynamic_streams_check_configs 1269 ] 1270 if model.dynamic_streams_check_configs 1271 else [] 1272 ) 1273 1274 return CheckStream( 1275 stream_names=model.stream_names or [], 1276 dynamic_streams_check_configs=dynamic_streams_check_configs, 1277 parameters={}, 1278 ) 1279 1280 @staticmethod 1281 def create_check_dynamic_stream( 1282 model: CheckDynamicStreamModel, config: Config, **kwargs: Any 1283 ) -> CheckDynamicStream: 1284 assert model.use_check_availability is not None # for mypy 1285 1286 use_check_availability = model.use_check_availability 1287 1288 return CheckDynamicStream( 1289 stream_count=model.stream_count, 1290 use_check_availability=use_check_availability, 1291 parameters={}, 1292 ) 1293 1294 def create_composite_error_handler( 1295 self, model: CompositeErrorHandlerModel, config: Config, **kwargs: Any 1296 ) -> CompositeErrorHandler: 1297 error_handlers = [ 1298 self._create_component_from_model(model=error_handler_model, config=config) 1299 for error_handler_model in model.error_handlers 1300 ] 1301 return CompositeErrorHandler( 1302 error_handlers=error_handlers, parameters=model.parameters or {} 1303 ) 1304 1305 @staticmethod 1306 def create_concurrency_level( 1307 model: ConcurrencyLevelModel, config: Config, **kwargs: Any 1308 ) -> ConcurrencyLevel: 1309 return ConcurrencyLevel( 1310 default_concurrency=model.default_concurrency, 1311 max_concurrency=model.max_concurrency, 1312 config=config, 1313 parameters={}, 1314 ) 1315 1316 @staticmethod 1317 def apply_stream_state_migrations( 1318 stream_state_migrations: List[Any] | None, stream_state: MutableMapping[str, Any] 1319 ) -> MutableMapping[str, Any]: 1320 if stream_state_migrations: 1321 for state_migration in stream_state_migrations: 1322 if state_migration.should_migrate(stream_state): 1323 # The state variable is expected to be mutable but the migrate method returns an immutable mapping. 1324 stream_state = dict(state_migration.migrate(stream_state)) 1325 return stream_state 1326 1327 def create_concurrent_cursor_from_datetime_based_cursor( 1328 self, 1329 model_type: Type[BaseModel], 1330 component_definition: ComponentDefinition, 1331 stream_name: str, 1332 stream_namespace: Optional[str], 1333 stream_state: MutableMapping[str, Any], 1334 config: Config, 1335 message_repository: Optional[MessageRepository] = None, 1336 runtime_lookback_window: Optional[datetime.timedelta] = None, 1337 **kwargs: Any, 1338 ) -> ConcurrentCursor: 1339 component_type = component_definition.get("type") 1340 if component_definition.get("type") != model_type.__name__: 1341 raise ValueError( 1342 f"Expected manifest component of type {model_type.__name__}, but received {component_type} instead" 1343 ) 1344 1345 # FIXME the interfaces of the concurrent cursor are kind of annoying as they take a `ComponentDefinition` instead of the actual model. This was done because the ConcurrentDeclarativeSource didn't have access to the models [here for example](https://github.com/airbytehq/airbyte-python-cdk/blob/f525803b3fec9329e4cc8478996a92bf884bfde9/airbyte_cdk/sources/declarative/concurrent_declarative_source.py#L354C54-L354C91). So now we have two cases: 1346 # * The ComponentDefinition comes from model.__dict__ in which case we have `parameters` 1347 # * The ComponentDefinition comes from the manifest as a dict in which case we have `$parameters` 1348 # We should change those interfaces to use the model once we clean up the code in CDS at which point the parameter propagation should happen as part of the ModelToComponentFactory. 1349 if "$parameters" not in component_definition and "parameters" in component_definition: 1350 component_definition["$parameters"] = component_definition.get("parameters") # type: ignore # This is a dict 1351 datetime_based_cursor_model = model_type.parse_obj(component_definition) 1352 1353 if not isinstance(datetime_based_cursor_model, DatetimeBasedCursorModel): 1354 raise ValueError( 1355 f"Expected {model_type.__name__} component, but received {datetime_based_cursor_model.__class__.__name__}" 1356 ) 1357 1358 model_parameters = datetime_based_cursor_model.parameters or {} 1359 1360 cursor_field = self._get_catalog_defined_cursor_field( 1361 stream_name=stream_name, 1362 allow_catalog_defined_cursor_field=datetime_based_cursor_model.allow_catalog_defined_cursor_field 1363 or False, 1364 ) 1365 1366 if not cursor_field: 1367 interpolated_cursor_field = InterpolatedString.create( 1368 datetime_based_cursor_model.cursor_field, 1369 parameters=model_parameters, 1370 ) 1371 cursor_field = CursorField( 1372 cursor_field_key=interpolated_cursor_field.eval(config=config), 1373 supports_catalog_defined_cursor_field=datetime_based_cursor_model.allow_catalog_defined_cursor_field 1374 or False, 1375 ) 1376 1377 interpolated_partition_field_start = InterpolatedString.create( 1378 datetime_based_cursor_model.partition_field_start or "start_time", 1379 parameters=model_parameters, 1380 ) 1381 interpolated_partition_field_end = InterpolatedString.create( 1382 datetime_based_cursor_model.partition_field_end or "end_time", 1383 parameters=model_parameters, 1384 ) 1385 1386 slice_boundary_fields = ( 1387 interpolated_partition_field_start.eval(config=config), 1388 interpolated_partition_field_end.eval(config=config), 1389 ) 1390 1391 datetime_format = datetime_based_cursor_model.datetime_format 1392 1393 cursor_granularity = ( 1394 parse_duration(datetime_based_cursor_model.cursor_granularity) 1395 if datetime_based_cursor_model.cursor_granularity 1396 else None 1397 ) 1398 1399 lookback_window = None 1400 interpolated_lookback_window = ( 1401 InterpolatedString.create( 1402 datetime_based_cursor_model.lookback_window, 1403 parameters=model_parameters, 1404 ) 1405 if datetime_based_cursor_model.lookback_window 1406 else None 1407 ) 1408 if interpolated_lookback_window: 1409 evaluated_lookback_window = interpolated_lookback_window.eval(config=config) 1410 if evaluated_lookback_window: 1411 lookback_window = parse_duration(evaluated_lookback_window) 1412 1413 connector_state_converter: DateTimeStreamStateConverter 1414 connector_state_converter = CustomFormatConcurrentStreamStateConverter( 1415 datetime_format=datetime_format, 1416 input_datetime_formats=datetime_based_cursor_model.cursor_datetime_formats, 1417 is_sequential_state=True, # ConcurrentPerPartitionCursor only works with sequential state 1418 cursor_granularity=cursor_granularity, 1419 ) 1420 1421 # Adjusts the stream state by applying the runtime lookback window. 1422 # This is used to ensure correct state handling in case of failed partitions. 1423 stream_state_value = stream_state.get(cursor_field.cursor_field_key) 1424 if runtime_lookback_window and stream_state_value: 1425 new_stream_state = ( 1426 connector_state_converter.parse_timestamp(stream_state_value) 1427 - runtime_lookback_window 1428 ) 1429 stream_state[cursor_field.cursor_field_key] = connector_state_converter.output_format( 1430 new_stream_state 1431 ) 1432 1433 start_date_runtime_value: Union[InterpolatedString, str, MinMaxDatetime] 1434 if isinstance(datetime_based_cursor_model.start_datetime, MinMaxDatetimeModel): 1435 start_date_runtime_value = self.create_min_max_datetime( 1436 model=datetime_based_cursor_model.start_datetime, config=config 1437 ) 1438 else: 1439 start_date_runtime_value = datetime_based_cursor_model.start_datetime 1440 1441 end_date_runtime_value: Optional[Union[InterpolatedString, str, MinMaxDatetime]] 1442 if isinstance(datetime_based_cursor_model.end_datetime, MinMaxDatetimeModel): 1443 end_date_runtime_value = self.create_min_max_datetime( 1444 model=datetime_based_cursor_model.end_datetime, config=config 1445 ) 1446 else: 1447 end_date_runtime_value = datetime_based_cursor_model.end_datetime 1448 1449 interpolated_start_date = MinMaxDatetime.create( 1450 interpolated_string_or_min_max_datetime=start_date_runtime_value, 1451 parameters=datetime_based_cursor_model.parameters, 1452 ) 1453 interpolated_end_date = ( 1454 None 1455 if not end_date_runtime_value 1456 else MinMaxDatetime.create( 1457 end_date_runtime_value, datetime_based_cursor_model.parameters 1458 ) 1459 ) 1460 1461 # If datetime format is not specified then start/end datetime should inherit it from the stream slicer 1462 if not interpolated_start_date.datetime_format: 1463 interpolated_start_date.datetime_format = datetime_format 1464 if interpolated_end_date and not interpolated_end_date.datetime_format: 1465 interpolated_end_date.datetime_format = datetime_format 1466 1467 start_date = interpolated_start_date.get_datetime(config=config) 1468 end_date_provider = ( 1469 partial(interpolated_end_date.get_datetime, config) 1470 if interpolated_end_date 1471 else connector_state_converter.get_end_provider() 1472 ) 1473 1474 if ( 1475 datetime_based_cursor_model.step and not datetime_based_cursor_model.cursor_granularity 1476 ) or ( 1477 not datetime_based_cursor_model.step and datetime_based_cursor_model.cursor_granularity 1478 ): 1479 raise ValueError( 1480 f"If step is defined, cursor_granularity should be as well and vice-versa. " 1481 f"Right now, step is `{datetime_based_cursor_model.step}` and cursor_granularity is `{datetime_based_cursor_model.cursor_granularity}`" 1482 ) 1483 1484 # When step is not defined, default to a step size from the starting date to the present moment 1485 step_length = datetime.timedelta.max 1486 interpolated_step = ( 1487 InterpolatedString.create( 1488 datetime_based_cursor_model.step, 1489 parameters=model_parameters, 1490 ) 1491 if datetime_based_cursor_model.step 1492 else None 1493 ) 1494 if interpolated_step: 1495 evaluated_step = interpolated_step.eval(config) 1496 if evaluated_step: 1497 step_length = parse_duration(evaluated_step) 1498 1499 clamping_strategy: ClampingStrategy = NoClamping() 1500 if datetime_based_cursor_model.clamping: 1501 # While it is undesirable to interpolate within the model factory (as opposed to at runtime), 1502 # it is still better than shifting interpolation low-code concept into the ConcurrentCursor runtime 1503 # object which we want to keep agnostic of being low-code 1504 target = InterpolatedString( 1505 string=datetime_based_cursor_model.clamping.target, 1506 parameters=model_parameters, 1507 ) 1508 evaluated_target = target.eval(config=config) 1509 match evaluated_target: 1510 case "DAY": 1511 clamping_strategy = DayClampingStrategy() 1512 end_date_provider = ClampingEndProvider( 1513 DayClampingStrategy(is_ceiling=False), 1514 end_date_provider, # type: ignore # Having issues w/ inspection for GapType and CursorValueType as shown in existing tests. Confirmed functionality is working in practice 1515 granularity=cursor_granularity or datetime.timedelta(seconds=1), 1516 ) 1517 case "WEEK": 1518 if ( 1519 not datetime_based_cursor_model.clamping.target_details 1520 or "weekday" not in datetime_based_cursor_model.clamping.target_details 1521 ): 1522 raise ValueError( 1523 "Given WEEK clamping, weekday needs to be provided as target_details" 1524 ) 1525 weekday = self._assemble_weekday( 1526 datetime_based_cursor_model.clamping.target_details["weekday"] 1527 ) 1528 clamping_strategy = WeekClampingStrategy(weekday) 1529 end_date_provider = ClampingEndProvider( 1530 WeekClampingStrategy(weekday, is_ceiling=False), 1531 end_date_provider, # type: ignore # Having issues w/ inspection for GapType and CursorValueType as shown in existing tests. Confirmed functionality is working in practice 1532 granularity=cursor_granularity or datetime.timedelta(days=1), 1533 ) 1534 case "MONTH": 1535 clamping_strategy = MonthClampingStrategy() 1536 end_date_provider = ClampingEndProvider( 1537 MonthClampingStrategy(is_ceiling=False), 1538 end_date_provider, # type: ignore # Having issues w/ inspection for GapType and CursorValueType as shown in existing tests. Confirmed functionality is working in practice 1539 granularity=cursor_granularity or datetime.timedelta(days=1), 1540 ) 1541 case _: 1542 raise ValueError( 1543 f"Invalid clamping target {evaluated_target}, expected DAY, WEEK, MONTH" 1544 ) 1545 1546 return ConcurrentCursor( 1547 stream_name=stream_name, 1548 stream_namespace=stream_namespace, 1549 stream_state=stream_state, 1550 message_repository=message_repository or self._message_repository, 1551 connector_state_manager=self._connector_state_manager, 1552 connector_state_converter=connector_state_converter, 1553 cursor_field=cursor_field, 1554 slice_boundary_fields=slice_boundary_fields, 1555 start=start_date, # type: ignore # Having issues w/ inspection for GapType and CursorValueType as shown in existing tests. Confirmed functionality is working in practice 1556 end_provider=end_date_provider, # type: ignore # Having issues w/ inspection for GapType and CursorValueType as shown in existing tests. Confirmed functionality is working in practice 1557 lookback_window=lookback_window, 1558 slice_range=step_length, 1559 cursor_granularity=cursor_granularity, 1560 clamping_strategy=clamping_strategy, 1561 ) 1562 1563 def create_concurrent_cursor_from_incrementing_count_cursor( 1564 self, 1565 model_type: Type[BaseModel], 1566 component_definition: ComponentDefinition, 1567 stream_name: str, 1568 stream_namespace: Optional[str], 1569 stream_state: MutableMapping[str, Any], 1570 config: Config, 1571 message_repository: Optional[MessageRepository] = None, 1572 **kwargs: Any, 1573 ) -> ConcurrentCursor: 1574 component_type = component_definition.get("type") 1575 if component_definition.get("type") != model_type.__name__: 1576 raise ValueError( 1577 f"Expected manifest component of type {model_type.__name__}, but received {component_type} instead" 1578 ) 1579 1580 incrementing_count_cursor_model = model_type.parse_obj(component_definition) 1581 1582 if not isinstance(incrementing_count_cursor_model, IncrementingCountCursorModel): 1583 raise ValueError( 1584 f"Expected {model_type.__name__} component, but received {incrementing_count_cursor_model.__class__.__name__}" 1585 ) 1586 1587 start_value: Union[int, str, None] = incrementing_count_cursor_model.start_value 1588 # Pydantic Union type coercion can convert int 0 to string '0' depending on Union order. 1589 # We need to handle both int and str representations of numeric values. 1590 # Evaluate the InterpolatedString and convert to int for the ConcurrentCursor. 1591 if start_value is not None: 1592 interpolated_start_value = InterpolatedString.create( 1593 str(start_value), # Ensure we pass a string to InterpolatedString.create 1594 parameters=incrementing_count_cursor_model.parameters or {}, 1595 ) 1596 evaluated_start_value: int = int(interpolated_start_value.eval(config=config)) 1597 else: 1598 evaluated_start_value = 0 1599 1600 cursor_field = self._get_catalog_defined_cursor_field( 1601 stream_name=stream_name, 1602 allow_catalog_defined_cursor_field=incrementing_count_cursor_model.allow_catalog_defined_cursor_field 1603 or False, 1604 ) 1605 1606 if not cursor_field: 1607 interpolated_cursor_field = InterpolatedString.create( 1608 incrementing_count_cursor_model.cursor_field, 1609 parameters=incrementing_count_cursor_model.parameters or {}, 1610 ) 1611 cursor_field = CursorField( 1612 cursor_field_key=interpolated_cursor_field.eval(config=config), 1613 supports_catalog_defined_cursor_field=incrementing_count_cursor_model.allow_catalog_defined_cursor_field 1614 or False, 1615 ) 1616 1617 connector_state_converter = IncrementingCountStreamStateConverter( 1618 is_sequential_state=True, # ConcurrentPerPartitionCursor only works with sequential state 1619 ) 1620 1621 return ConcurrentCursor( 1622 stream_name=stream_name, 1623 stream_namespace=stream_namespace, 1624 stream_state=stream_state, 1625 message_repository=message_repository or self._message_repository, 1626 connector_state_manager=self._connector_state_manager, 1627 connector_state_converter=connector_state_converter, 1628 cursor_field=cursor_field, 1629 slice_boundary_fields=None, 1630 start=evaluated_start_value, # type: ignore # Having issues w/ inspection for GapType and CursorValueType as shown in existing tests. Confirmed functionality is working in practice 1631 end_provider=connector_state_converter.get_end_provider(), # type: ignore # Having issues w/ inspection for GapType and CursorValueType as shown in existing tests. Confirmed functionality is working in practice 1632 ) 1633 1634 def _assemble_weekday(self, weekday: str) -> Weekday: 1635 match weekday: 1636 case "MONDAY": 1637 return Weekday.MONDAY 1638 case "TUESDAY": 1639 return Weekday.TUESDAY 1640 case "WEDNESDAY": 1641 return Weekday.WEDNESDAY 1642 case "THURSDAY": 1643 return Weekday.THURSDAY 1644 case "FRIDAY": 1645 return Weekday.FRIDAY 1646 case "SATURDAY": 1647 return Weekday.SATURDAY 1648 case "SUNDAY": 1649 return Weekday.SUNDAY 1650 case _: 1651 raise ValueError(f"Unknown weekday {weekday}") 1652 1653 def create_concurrent_cursor_from_perpartition_cursor( 1654 self, 1655 state_manager: ConnectorStateManager, 1656 model_type: Type[BaseModel], 1657 component_definition: ComponentDefinition, 1658 stream_name: str, 1659 stream_namespace: Optional[str], 1660 config: Config, 1661 stream_state: MutableMapping[str, Any], 1662 partition_router: PartitionRouter, 1663 attempt_to_create_cursor_if_not_provided: bool = False, 1664 **kwargs: Any, 1665 ) -> ConcurrentPerPartitionCursor: 1666 component_type = component_definition.get("type") 1667 if component_definition.get("type") != model_type.__name__: 1668 raise ValueError( 1669 f"Expected manifest component of type {model_type.__name__}, but received {component_type} instead" 1670 ) 1671 1672 # FIXME the interfaces of the concurrent cursor are kind of annoying as they take a `ComponentDefinition` instead of the actual model. This was done because the ConcurrentDeclarativeSource didn't have access to the models [here for example](https://github.com/airbytehq/airbyte-python-cdk/blob/f525803b3fec9329e4cc8478996a92bf884bfde9/airbyte_cdk/sources/declarative/concurrent_declarative_source.py#L354C54-L354C91). So now we have two cases: 1673 # * The ComponentDefinition comes from model.__dict__ in which case we have `parameters` 1674 # * The ComponentDefinition comes from the manifest as a dict in which case we have `$parameters` 1675 # We should change those interfaces to use the model once we clean up the code in CDS at which point the parameter propagation should happen as part of the ModelToComponentFactory. 1676 if "$parameters" not in component_definition and "parameters" in component_definition: 1677 component_definition["$parameters"] = component_definition.get("parameters") # type: ignore # This is a dict 1678 datetime_based_cursor_model = model_type.parse_obj(component_definition) 1679 1680 if not isinstance(datetime_based_cursor_model, DatetimeBasedCursorModel): 1681 raise ValueError( 1682 f"Expected {model_type.__name__} component, but received {datetime_based_cursor_model.__class__.__name__}" 1683 ) 1684 1685 cursor_field = self._get_catalog_defined_cursor_field( 1686 stream_name=stream_name, 1687 allow_catalog_defined_cursor_field=datetime_based_cursor_model.allow_catalog_defined_cursor_field 1688 or False, 1689 ) 1690 1691 if not cursor_field: 1692 interpolated_cursor_field = InterpolatedString.create( 1693 datetime_based_cursor_model.cursor_field, 1694 # FIXME the interfaces of the concurrent cursor are kind of annoying as they take a `ComponentDefinition` instead of the actual model. This was done because the ConcurrentDeclarativeSource didn't have access to the models [here for example](https://github.com/airbytehq/airbyte-python-cdk/blob/f525803b3fec9329e4cc8478996a92bf884bfde9/airbyte_cdk/sources/declarative/concurrent_declarative_source.py#L354C54-L354C91). So now we have two cases: 1695 # * The ComponentDefinition comes from model.__dict__ in which case we have `parameters` 1696 # * The ComponentDefinition comes from the manifest as a dict in which case we have `$parameters` 1697 # We should change those interfaces to use the model once we clean up the code in CDS at which point the parameter propagation should happen as part of the ModelToComponentFactory. 1698 parameters=datetime_based_cursor_model.parameters or {}, 1699 ) 1700 cursor_field = CursorField( 1701 cursor_field_key=interpolated_cursor_field.eval(config=config), 1702 supports_catalog_defined_cursor_field=datetime_based_cursor_model.allow_catalog_defined_cursor_field 1703 or False, 1704 ) 1705 1706 datetime_format = datetime_based_cursor_model.datetime_format 1707 1708 cursor_granularity = ( 1709 parse_duration(datetime_based_cursor_model.cursor_granularity) 1710 if datetime_based_cursor_model.cursor_granularity 1711 else None 1712 ) 1713 1714 connector_state_converter: DateTimeStreamStateConverter 1715 connector_state_converter = CustomFormatConcurrentStreamStateConverter( 1716 datetime_format=datetime_format, 1717 input_datetime_formats=datetime_based_cursor_model.cursor_datetime_formats, 1718 is_sequential_state=True, # ConcurrentPerPartitionCursor only works with sequential state 1719 cursor_granularity=cursor_granularity, 1720 ) 1721 1722 # Create the cursor factory 1723 cursor_factory = ConcurrentCursorFactory( 1724 partial( 1725 self.create_concurrent_cursor_from_datetime_based_cursor, 1726 state_manager=state_manager, 1727 model_type=model_type, 1728 component_definition=component_definition, 1729 stream_name=stream_name, 1730 stream_namespace=stream_namespace, 1731 config=config, 1732 message_repository=NoopMessageRepository(), 1733 ) 1734 ) 1735 1736 # Per-partition state doesn't make sense for GroupingPartitionRouter, so force the global state 1737 use_global_cursor = isinstance( 1738 partition_router, GroupingPartitionRouter 1739 ) or component_definition.get("global_substream_cursor", False) 1740 1741 # Return the concurrent cursor and state converter 1742 return ConcurrentPerPartitionCursor( 1743 cursor_factory=cursor_factory, 1744 partition_router=partition_router, 1745 stream_name=stream_name, 1746 stream_namespace=stream_namespace, 1747 stream_state=stream_state, 1748 message_repository=self._message_repository, # type: ignore 1749 connector_state_manager=state_manager, 1750 connector_state_converter=connector_state_converter, 1751 cursor_field=cursor_field, 1752 use_global_cursor=use_global_cursor, 1753 attempt_to_create_cursor_if_not_provided=attempt_to_create_cursor_if_not_provided, 1754 ) 1755 1756 @staticmethod 1757 def create_constant_backoff_strategy( 1758 model: ConstantBackoffStrategyModel, config: Config, **kwargs: Any 1759 ) -> ConstantBackoffStrategy: 1760 ModelToComponentFactory._validate_jitter_range(model.jitter_range_in_seconds) 1761 return ConstantBackoffStrategy( 1762 backoff_time_in_seconds=model.backoff_time_in_seconds, 1763 jitter_range_in_seconds=model.jitter_range_in_seconds, 1764 config=config, 1765 parameters=model.parameters or {}, 1766 ) 1767 1768 @staticmethod 1769 def _validate_jitter_range(jitter_range_in_seconds: Optional[float]) -> None: 1770 if jitter_range_in_seconds is not None and jitter_range_in_seconds < 0: 1771 raise ValueError("jitter_range_in_seconds must be greater than or equal to 0") 1772 1773 def create_cursor_pagination( 1774 self, model: CursorPaginationModel, config: Config, decoder: Decoder, **kwargs: Any 1775 ) -> CursorPaginationStrategy: 1776 if isinstance(decoder, PaginationDecoderDecorator): 1777 inner_decoder = decoder.decoder 1778 else: 1779 inner_decoder = decoder 1780 decoder = PaginationDecoderDecorator(decoder=decoder) 1781 1782 if self._is_supported_decoder_for_pagination(inner_decoder): 1783 decoder_to_use = decoder 1784 else: 1785 raise ValueError( 1786 self._UNSUPPORTED_DECODER_ERROR.format(decoder_type=type(inner_decoder)) 1787 ) 1788 1789 # Pydantic v1 Union type coercion can convert int to string depending on Union order. 1790 # If page_size is a string that represents an integer (not an interpolation), convert it back. 1791 page_size = model.page_size 1792 if isinstance(page_size, str) and page_size.isdigit(): 1793 page_size = int(page_size) 1794 1795 return CursorPaginationStrategy( 1796 cursor_value=model.cursor_value, 1797 decoder=decoder_to_use, 1798 page_size=page_size, 1799 stop_condition=model.stop_condition, 1800 config=config, 1801 parameters=model.parameters or {}, 1802 ) 1803 1804 def create_custom_component(self, model: Any, config: Config, **kwargs: Any) -> Any: 1805 """ 1806 Generically creates a custom component based on the model type and a class_name reference to the custom Python class being 1807 instantiated. Only the model's additional properties that match the custom class definition are passed to the constructor 1808 :param model: The Pydantic model of the custom component being created 1809 :param config: The custom defined connector config 1810 :return: The declarative component built from the Pydantic model to be used at runtime 1811 """ 1812 custom_component_class = self._get_class_from_fully_qualified_class_name(model.class_name) 1813 component_fields = get_type_hints(custom_component_class) 1814 model_args = model.dict() 1815 model_args["config"] = config 1816 1817 # There are cases where a parent component will pass arguments to a child component via kwargs. When there are field collisions 1818 # we defer to these arguments over the component's definition 1819 for key, arg in kwargs.items(): 1820 model_args[key] = arg 1821 1822 # Pydantic is unable to parse a custom component's fields that are subcomponents into models because their fields and types are not 1823 # defined in the schema. The fields and types are defined within the Python class implementation. Pydantic can only parse down to 1824 # the custom component and this code performs a second parse to convert the sub-fields first into models, then declarative components 1825 for model_field, model_value in model_args.items(): 1826 # If a custom component field doesn't have a type set, we try to use the type hints to infer the type 1827 if ( 1828 isinstance(model_value, dict) 1829 and "type" not in model_value 1830 and model_field in component_fields 1831 ): 1832 derived_type = self._derive_component_type_from_type_hints( 1833 component_fields.get(model_field) 1834 ) 1835 if derived_type: 1836 model_value["type"] = derived_type 1837 1838 if self._is_component(model_value): 1839 model_args[model_field] = self._create_nested_component( 1840 model, 1841 model_field, 1842 model_value, 1843 config, 1844 **kwargs, 1845 ) 1846 elif isinstance(model_value, list): 1847 vals = [] 1848 for v in model_value: 1849 if isinstance(v, dict) and "type" not in v and model_field in component_fields: 1850 derived_type = self._derive_component_type_from_type_hints( 1851 component_fields.get(model_field) 1852 ) 1853 if derived_type: 1854 v["type"] = derived_type 1855 if self._is_component(v): 1856 vals.append( 1857 self._create_nested_component( 1858 model, 1859 model_field, 1860 v, 1861 config, 1862 **kwargs, 1863 ) 1864 ) 1865 else: 1866 vals.append(v) 1867 model_args[model_field] = vals 1868 1869 kwargs = { 1870 class_field: model_args[class_field] 1871 for class_field in component_fields.keys() 1872 if class_field in model_args 1873 } 1874 1875 if "api_budget" in component_fields and kwargs.get("api_budget") is None: 1876 kwargs["api_budget"] = self._api_budget 1877 1878 return custom_component_class(**kwargs) 1879 1880 @staticmethod 1881 def _get_class_from_fully_qualified_class_name( 1882 full_qualified_class_name: str, 1883 ) -> Any: 1884 """Get a class from its fully qualified name. 1885 1886 If a custom components module is needed, we assume it is already registered - probably 1887 as `source_declarative_manifest.components` or `components`. 1888 1889 Args: 1890 full_qualified_class_name (str): The fully qualified name of the class (e.g., "module.ClassName"). 1891 1892 Returns: 1893 Any: The class object. 1894 1895 Raises: 1896 ValueError: If the class cannot be loaded. 1897 """ 1898 split = full_qualified_class_name.split(".") 1899 module_name_full = ".".join(split[:-1]) 1900 class_name = split[-1] 1901 1902 try: 1903 module_ref = importlib.import_module(module_name_full) 1904 except ModuleNotFoundError as e: 1905 if split[0] == "source_declarative_manifest": 1906 # During testing, the modules containing the custom components are not moved to source_declarative_manifest. In order to run the test, add the source folder to your PYTHONPATH or add it runtime using sys.path.append 1907 try: 1908 import os 1909 1910 module_name_with_source_declarative_manifest = ".".join(split[1:-1]) 1911 module_ref = importlib.import_module( 1912 module_name_with_source_declarative_manifest 1913 ) 1914 except ModuleNotFoundError: 1915 raise ValueError(f"Could not load module `{module_name_full}`.") from e 1916 else: 1917 raise ValueError(f"Could not load module `{module_name_full}`.") from e 1918 1919 try: 1920 return getattr(module_ref, class_name) 1921 except AttributeError as e: 1922 raise ValueError( 1923 f"Could not load class `{class_name}` from module `{module_name_full}`.", 1924 ) from e 1925 1926 @staticmethod 1927 def _derive_component_type_from_type_hints(field_type: Any) -> Optional[str]: 1928 interface = field_type 1929 while True: 1930 origin = get_origin(interface) 1931 if origin: 1932 # Unnest types until we reach the raw type 1933 # List[T] -> T 1934 # Optional[List[T]] -> T 1935 args = get_args(interface) 1936 interface = args[0] 1937 else: 1938 break 1939 if isinstance(interface, type) and not ModelToComponentFactory.is_builtin_type(interface): 1940 return interface.__name__ 1941 return None 1942 1943 @staticmethod 1944 def is_builtin_type(cls: Optional[Type[Any]]) -> bool: 1945 if not cls: 1946 return False 1947 return cls.__module__ == "builtins" 1948 1949 @staticmethod 1950 def _extract_missing_parameters(error: TypeError) -> List[str]: 1951 parameter_search = re.search(r"keyword-only.*:\s(.*)", str(error)) 1952 if parameter_search: 1953 return re.findall(r"\'(.+?)\'", parameter_search.group(1)) 1954 else: 1955 return [] 1956 1957 def _create_nested_component( 1958 self, model: Any, model_field: str, model_value: Any, config: Config, **kwargs: Any 1959 ) -> Any: 1960 type_name = model_value.get("type", None) 1961 if not type_name: 1962 # If no type is specified, we can assume this is a dictionary object which can be returned instead of a subcomponent 1963 return model_value 1964 1965 model_type = self.TYPE_NAME_TO_MODEL.get(type_name, None) 1966 if model_type: 1967 parsed_model = model_type.parse_obj(model_value) 1968 try: 1969 # To improve usability of the language, certain fields are shared between components. This can come in the form of 1970 # a parent component passing some of its fields to a child component or the parent extracting fields from other child 1971 # components and passing it to others. One example is the DefaultPaginator referencing the HttpRequester url_base 1972 # while constructing a SimpleRetriever. However, custom components don't support this behavior because they are created 1973 # generically in create_custom_component(). This block allows developers to specify extra arguments in $parameters that 1974 # are needed by a component and could not be shared. 1975 model_constructor = self.PYDANTIC_MODEL_TO_CONSTRUCTOR.get(parsed_model.__class__) 1976 constructor_kwargs = inspect.getfullargspec(model_constructor).kwonlyargs 1977 model_parameters = model_value.get("$parameters", {}) 1978 matching_parameters = { 1979 kwarg: model_parameters[kwarg] 1980 for kwarg in constructor_kwargs 1981 if kwarg in model_parameters 1982 } 1983 matching_kwargs = { 1984 kwarg: kwargs[kwarg] for kwarg in constructor_kwargs if kwarg in kwargs 1985 } 1986 return self._create_component_from_model( 1987 model=parsed_model, config=config, **(matching_parameters | matching_kwargs) 1988 ) 1989 except TypeError as error: 1990 missing_parameters = self._extract_missing_parameters(error) 1991 if missing_parameters: 1992 raise ValueError( 1993 f"Error creating component '{type_name}' with parent custom component {model.class_name}: Please provide " 1994 + ", ".join( 1995 ( 1996 f"{type_name}.$parameters.{parameter}" 1997 for parameter in missing_parameters 1998 ) 1999 ) 2000 ) 2001 raise TypeError( 2002 f"Error creating component '{type_name}' with parent custom component {model.class_name}: {error}" 2003 ) 2004 else: 2005 raise ValueError( 2006 f"Error creating custom component {model.class_name}. Subcomponent creation has not been implemented for '{type_name}'" 2007 ) 2008 2009 @staticmethod 2010 def _is_component(model_value: Any) -> bool: 2011 return isinstance(model_value, dict) and model_value.get("type") is not None 2012 2013 def create_default_stream( 2014 self, model: DeclarativeStreamModel, config: Config, is_parent: bool = False, **kwargs: Any 2015 ) -> AbstractStream: 2016 primary_key = model.primary_key.__root__ if model.primary_key else None 2017 self._migrate_state(model, config) 2018 2019 partition_router = self._build_stream_slicer_from_partition_router( 2020 model.retriever, 2021 config, 2022 stream_name=model.name, 2023 **kwargs, 2024 ) 2025 concurrent_cursor = self._build_concurrent_cursor(model, partition_router, config) 2026 if model.incremental_sync and isinstance(model.incremental_sync, DatetimeBasedCursorModel): 2027 cursor_model: DatetimeBasedCursorModel = model.incremental_sync 2028 2029 end_time_option = ( 2030 self._create_component_from_model( 2031 cursor_model.end_time_option, config, parameters=cursor_model.parameters or {} 2032 ) 2033 if cursor_model.end_time_option 2034 else None 2035 ) 2036 start_time_option = ( 2037 self._create_component_from_model( 2038 cursor_model.start_time_option, config, parameters=cursor_model.parameters or {} 2039 ) 2040 if cursor_model.start_time_option 2041 else None 2042 ) 2043 2044 datetime_request_options_provider = DatetimeBasedRequestOptionsProvider( 2045 start_time_option=start_time_option, 2046 end_time_option=end_time_option, 2047 partition_field_start=cursor_model.partition_field_start, 2048 partition_field_end=cursor_model.partition_field_end, 2049 config=config, 2050 parameters=model.parameters or {}, 2051 ) 2052 request_options_provider = ( 2053 datetime_request_options_provider 2054 if not isinstance(concurrent_cursor, ConcurrentPerPartitionCursor) 2055 else PerPartitionRequestOptionsProvider( 2056 partition_router, datetime_request_options_provider 2057 ) 2058 ) 2059 elif model.incremental_sync and isinstance( 2060 model.incremental_sync, IncrementingCountCursorModel 2061 ): 2062 if isinstance(concurrent_cursor, ConcurrentPerPartitionCursor): 2063 raise ValueError( 2064 "PerPartition does not support per partition states because switching to global state is time based" 2065 ) 2066 2067 cursor_model: IncrementingCountCursorModel = model.incremental_sync # type: ignore 2068 2069 start_time_option = ( 2070 self._create_component_from_model( 2071 cursor_model.start_value_option, # type: ignore # mypy still thinks cursor_model of type DatetimeBasedCursor 2072 config, 2073 parameters=cursor_model.parameters or {}, 2074 ) 2075 if cursor_model.start_value_option # type: ignore # mypy still thinks cursor_model of type DatetimeBasedCursor 2076 else None 2077 ) 2078 2079 # The concurrent engine defaults the start/end fields on the slice to "start" and "end", but 2080 # the default DatetimeBasedRequestOptionsProvider() sets them to start_time/end_time 2081 partition_field_start = "start" 2082 2083 request_options_provider = DatetimeBasedRequestOptionsProvider( 2084 start_time_option=start_time_option, 2085 partition_field_start=partition_field_start, 2086 config=config, 2087 parameters=model.parameters or {}, 2088 ) 2089 else: 2090 request_options_provider = None 2091 2092 transformations = [] 2093 if model.transformations: 2094 for transformation_model in model.transformations: 2095 transformations.append( 2096 self._create_component_from_model(model=transformation_model, config=config) 2097 ) 2098 file_uploader = None 2099 if model.file_uploader: 2100 file_uploader = self._create_component_from_model( 2101 model=model.file_uploader, config=config 2102 ) 2103 2104 stream_slicer: ConcurrentStreamSlicer = ( 2105 partition_router 2106 if isinstance(concurrent_cursor, FinalStateCursor) 2107 else concurrent_cursor 2108 ) 2109 2110 retriever = self._create_component_from_model( 2111 model=model.retriever, 2112 config=config, 2113 name=model.name, 2114 primary_key=primary_key, 2115 request_options_provider=request_options_provider, 2116 stream_slicer=stream_slicer, 2117 partition_router=partition_router, 2118 has_stop_condition_cursor=self._is_stop_condition_on_cursor(model), 2119 is_client_side_incremental_sync=self._is_client_side_filtering_enabled(model), 2120 cursor=concurrent_cursor, 2121 transformations=transformations, 2122 file_uploader=file_uploader, 2123 incremental_sync=model.incremental_sync, 2124 ) 2125 if isinstance(retriever, AsyncRetriever): 2126 stream_slicer = retriever.stream_slicer 2127 2128 schema_loader: SchemaLoader 2129 if model.schema_loader and isinstance(model.schema_loader, list): 2130 nested_schema_loaders = [ 2131 self._create_component_from_model(model=nested_schema_loader, config=config) 2132 for nested_schema_loader in model.schema_loader 2133 ] 2134 schema_loader = CompositeSchemaLoader( 2135 schema_loaders=nested_schema_loaders, parameters={} 2136 ) 2137 elif model.schema_loader: 2138 schema_loader = self._create_component_from_model( 2139 model=model.schema_loader, # type: ignore # If defined, schema_loader is guaranteed not to be a list and will be one of the existing base models 2140 config=config, 2141 ) 2142 else: 2143 options = model.parameters or {} 2144 if "name" not in options: 2145 options["name"] = model.name 2146 schema_loader = DefaultSchemaLoader(config=config, parameters=options) 2147 schema_loader = CachingSchemaLoaderDecorator(schema_loader) 2148 2149 stream_name = model.name or "" 2150 return DefaultStream( 2151 partition_generator=StreamSlicerPartitionGenerator( 2152 DeclarativePartitionFactory( 2153 stream_name, 2154 schema_loader, 2155 retriever, 2156 self._message_repository, 2157 ), 2158 stream_slicer, 2159 slice_limit=self._limit_slices_fetched, 2160 ), 2161 name=stream_name, 2162 json_schema=schema_loader.get_json_schema, 2163 primary_key=get_primary_key_from_stream(primary_key), 2164 cursor_field=( 2165 concurrent_cursor.cursor_field 2166 if hasattr(concurrent_cursor, "cursor_field") 2167 else None 2168 ), 2169 logger=logging.getLogger(f"airbyte.{stream_name}"), 2170 cursor=concurrent_cursor, 2171 supports_file_transfer=hasattr(model, "file_uploader") and bool(model.file_uploader), 2172 ) 2173 2174 def _migrate_state(self, model: DeclarativeStreamModel, config: Config) -> None: 2175 stream_name = model.name or "" 2176 stream_state = self._connector_state_manager.get_stream_state( 2177 stream_name=stream_name, namespace=None 2178 ) 2179 if model.state_migrations: 2180 state_transformations = [ 2181 self._create_component_from_model(state_migration, config, declarative_stream=model) 2182 for state_migration in model.state_migrations 2183 ] 2184 else: 2185 state_transformations = [] 2186 stream_state = self.apply_stream_state_migrations(state_transformations, stream_state) 2187 self._connector_state_manager.update_state_for_stream( 2188 stream_name=stream_name, namespace=None, value=stream_state 2189 ) 2190 2191 def _is_stop_condition_on_cursor(self, model: DeclarativeStreamModel) -> bool: 2192 return bool( 2193 model.incremental_sync 2194 and hasattr(model.incremental_sync, "is_data_feed") 2195 and model.incremental_sync.is_data_feed 2196 ) 2197 2198 def _is_client_side_filtering_enabled(self, model: DeclarativeStreamModel) -> bool: 2199 return bool( 2200 model.incremental_sync 2201 and hasattr(model.incremental_sync, "is_client_side_incremental") 2202 and model.incremental_sync.is_client_side_incremental 2203 ) 2204 2205 def _build_stream_slicer_from_partition_router( 2206 self, 2207 model: Union[ 2208 AsyncRetrieverModel, 2209 CustomRetrieverModel, 2210 SimpleRetrieverModel, 2211 ], 2212 config: Config, 2213 stream_name: Optional[str] = None, 2214 **kwargs: Any, 2215 ) -> PartitionRouter: 2216 if ( 2217 hasattr(model, "partition_router") 2218 and isinstance(model, (SimpleRetrieverModel, AsyncRetrieverModel, CustomRetrieverModel)) 2219 and model.partition_router 2220 ): 2221 stream_slicer_model = model.partition_router 2222 if isinstance(stream_slicer_model, list): 2223 return CartesianProductStreamSlicer( 2224 [ 2225 self._create_component_from_model( 2226 model=slicer, config=config, stream_name=stream_name or "" 2227 ) 2228 for slicer in stream_slicer_model 2229 ], 2230 parameters={}, 2231 ) 2232 elif isinstance(stream_slicer_model, dict): 2233 # partition router comes from CustomRetrieverModel therefore has not been parsed as a model 2234 params = stream_slicer_model.get("$parameters") 2235 if not isinstance(params, dict): 2236 params = {} 2237 stream_slicer_model["$parameters"] = params 2238 2239 if stream_name is not None: 2240 params["stream_name"] = stream_name 2241 2242 return self._create_nested_component( # type: ignore[no-any-return] # There is no guarantee that this will return a stream slicer. If not, we expect an AttributeError during the call to `stream_slices` 2243 model, 2244 "partition_router", 2245 stream_slicer_model, 2246 config, 2247 **kwargs, 2248 ) 2249 else: 2250 return self._create_component_from_model( # type: ignore[no-any-return] # Will be created PartitionRouter as stream_slicer_model is model.partition_router 2251 model=stream_slicer_model, config=config, stream_name=stream_name or "" 2252 ) 2253 return SinglePartitionRouter(parameters={}) 2254 2255 def _build_concurrent_cursor( 2256 self, 2257 model: DeclarativeStreamModel, 2258 stream_slicer: Optional[PartitionRouter], 2259 config: Config, 2260 ) -> Cursor: 2261 stream_name = model.name or "" 2262 stream_state = self._connector_state_manager.get_stream_state(stream_name, None) 2263 2264 if ( 2265 model.incremental_sync 2266 and stream_slicer 2267 and not isinstance(stream_slicer, SinglePartitionRouter) 2268 ): 2269 if isinstance(model.incremental_sync, IncrementingCountCursorModel): 2270 # We don't currently support usage of partition routing and IncrementingCountCursor at the 2271 # same time because we didn't solve for design questions like what the lookback window would 2272 # be as well as global cursor fall backs. We have not seen customers that have needed both 2273 # at the same time yet and are currently punting on this until we need to solve it. 2274 raise ValueError( 2275 f"The low-code framework does not currently support usage of a PartitionRouter and an IncrementingCountCursor at the same time. Please specify only one of these options for stream {stream_name}." 2276 ) 2277 return self.create_concurrent_cursor_from_perpartition_cursor( # type: ignore # This is a known issue that we are creating and returning a ConcurrentCursor which does not technically implement the (low-code) StreamSlicer. However, (low-code) StreamSlicer and ConcurrentCursor both implement StreamSlicer.stream_slices() which is the primary method needed for checkpointing 2278 state_manager=self._connector_state_manager, 2279 model_type=DatetimeBasedCursorModel, 2280 component_definition=model.incremental_sync.__dict__, 2281 stream_name=stream_name, 2282 stream_state=stream_state, 2283 stream_namespace=None, 2284 config=config or {}, 2285 partition_router=stream_slicer, 2286 attempt_to_create_cursor_if_not_provided=True, # FIXME can we remove that now? 2287 ) 2288 elif model.incremental_sync: 2289 if type(model.incremental_sync) == IncrementingCountCursorModel: 2290 return self.create_concurrent_cursor_from_incrementing_count_cursor( # type: ignore # This is a known issue that we are creating and returning a ConcurrentCursor which does not technically implement the (low-code) StreamSlicer. However, (low-code) StreamSlicer and ConcurrentCursor both implement StreamSlicer.stream_slices() which is the primary method needed for checkpointing 2291 model_type=IncrementingCountCursorModel, 2292 component_definition=model.incremental_sync.__dict__, 2293 stream_name=stream_name, 2294 stream_namespace=None, 2295 stream_state=stream_state, 2296 config=config or {}, 2297 ) 2298 elif type(model.incremental_sync) == DatetimeBasedCursorModel: 2299 return self.create_concurrent_cursor_from_datetime_based_cursor( # type: ignore # This is a known issue that we are creating and returning a ConcurrentCursor which does not technically implement the (low-code) StreamSlicer. However, (low-code) StreamSlicer and ConcurrentCursor both implement StreamSlicer.stream_slices() which is the primary method needed for checkpointing 2300 model_type=type(model.incremental_sync), 2301 component_definition=model.incremental_sync.__dict__, 2302 stream_name=stream_name, 2303 stream_namespace=None, 2304 stream_state=stream_state, 2305 config=config or {}, 2306 attempt_to_create_cursor_if_not_provided=True, 2307 ) 2308 else: 2309 raise ValueError( 2310 f"Incremental sync of type {type(model.incremental_sync)} is not supported" 2311 ) 2312 return FinalStateCursor(stream_name, None, self._message_repository) 2313 2314 def create_default_error_handler( 2315 self, model: DefaultErrorHandlerModel, config: Config, **kwargs: Any 2316 ) -> DefaultErrorHandler: 2317 backoff_strategies = [] 2318 if model.backoff_strategies: 2319 for backoff_strategy_model in model.backoff_strategies: 2320 backoff_strategies.append( 2321 self._create_component_from_model(model=backoff_strategy_model, config=config) 2322 ) 2323 2324 response_filters = [] 2325 if model.response_filters: 2326 for response_filter_model in model.response_filters: 2327 response_filters.append( 2328 self._create_component_from_model(model=response_filter_model, config=config) 2329 ) 2330 response_filters.append( 2331 HttpResponseFilter(config=config, parameters=model.parameters or {}) 2332 ) 2333 2334 return DefaultErrorHandler( 2335 backoff_strategies=backoff_strategies, 2336 max_retries=model.max_retries, 2337 response_filters=response_filters, 2338 config=config, 2339 parameters=model.parameters or {}, 2340 ) 2341 2342 def create_default_paginator( 2343 self, 2344 model: DefaultPaginatorModel, 2345 config: Config, 2346 *, 2347 url_base: str, 2348 extractor_model: Optional[Union[CustomRecordExtractorModel, DpathExtractorModel]] = None, 2349 decoder: Optional[Decoder] = None, 2350 cursor_used_for_stop_condition: Optional[Cursor] = None, 2351 ) -> Union[DefaultPaginator, PaginatorTestReadDecorator]: 2352 if decoder: 2353 if self._is_supported_decoder_for_pagination(decoder): 2354 decoder_to_use = PaginationDecoderDecorator(decoder=decoder) 2355 else: 2356 raise ValueError(self._UNSUPPORTED_DECODER_ERROR.format(decoder_type=type(decoder))) 2357 else: 2358 decoder_to_use = PaginationDecoderDecorator(decoder=JsonDecoder(parameters={})) 2359 page_size_option = ( 2360 self._create_component_from_model(model=model.page_size_option, config=config) 2361 if model.page_size_option 2362 else None 2363 ) 2364 page_token_option = ( 2365 self._create_component_from_model(model=model.page_token_option, config=config) 2366 if model.page_token_option 2367 else None 2368 ) 2369 pagination_strategy = self._create_component_from_model( 2370 model=model.pagination_strategy, 2371 config=config, 2372 decoder=decoder_to_use, 2373 extractor_model=extractor_model, 2374 ) 2375 if cursor_used_for_stop_condition: 2376 pagination_strategy = StopConditionPaginationStrategyDecorator( 2377 pagination_strategy, CursorStopCondition(cursor_used_for_stop_condition) 2378 ) 2379 paginator = DefaultPaginator( 2380 decoder=decoder_to_use, 2381 page_size_option=page_size_option, 2382 page_token_option=page_token_option, 2383 pagination_strategy=pagination_strategy, 2384 url_base=url_base, 2385 config=config, 2386 parameters=model.parameters or {}, 2387 ) 2388 if self._limit_pages_fetched_per_slice: 2389 return PaginatorTestReadDecorator(paginator, self._limit_pages_fetched_per_slice) 2390 return paginator 2391 2392 def create_dpath_extractor( 2393 self, 2394 model: DpathExtractorModel, 2395 config: Config, 2396 decoder: Optional[Decoder] = None, 2397 **kwargs: Any, 2398 ) -> DpathExtractor: 2399 if decoder: 2400 decoder_to_use = decoder 2401 else: 2402 decoder_to_use = JsonDecoder(parameters={}) 2403 model_field_path: List[Union[InterpolatedString, str]] = [x for x in model.field_path] 2404 2405 record_expander = None 2406 if model.record_expander: 2407 record_expander = self._create_component_from_model( 2408 model=model.record_expander, 2409 config=config, 2410 ) 2411 2412 return DpathExtractor( 2413 decoder=decoder_to_use, 2414 field_path=model_field_path, 2415 config=config, 2416 parameters=model.parameters or {}, 2417 record_expander=record_expander, 2418 ) 2419 2420 def create_record_expander( 2421 self, 2422 model: RecordExpanderModel, 2423 config: Config, 2424 **kwargs: Any, 2425 ) -> RecordExpander: 2426 return RecordExpander( 2427 expand_records_from_field=model.expand_records_from_field, 2428 config=config, 2429 parameters=model.parameters or {}, 2430 remain_original_record=model.remain_original_record or False, 2431 on_no_records=OnNoRecords(model.on_no_records.value) 2432 if model.on_no_records 2433 else OnNoRecords.skip, 2434 ) 2435 2436 @staticmethod 2437 def create_response_to_file_extractor( 2438 model: ResponseToFileExtractorModel, 2439 **kwargs: Any, 2440 ) -> ResponseToFileExtractor: 2441 return ResponseToFileExtractor( 2442 parameters=model.parameters or {}, 2443 preserve_na_values=model.preserve_na_values or False, 2444 ) 2445 2446 @staticmethod 2447 def create_exponential_backoff_strategy( 2448 model: ExponentialBackoffStrategyModel, config: Config 2449 ) -> ExponentialBackoffStrategy: 2450 ModelToComponentFactory._validate_jitter_range(model.jitter_range_in_seconds) 2451 return ExponentialBackoffStrategy( 2452 factor=model.factor or 5, 2453 jitter_range_in_seconds=model.jitter_range_in_seconds, 2454 parameters=model.parameters or {}, 2455 config=config, 2456 ) 2457 2458 @staticmethod 2459 def create_group_by_key(model: GroupByKeyMergeStrategyModel, config: Config) -> GroupByKey: 2460 return GroupByKey(model.key, config=config, parameters=model.parameters or {}) 2461 2462 def create_http_requester( 2463 self, 2464 model: HttpRequesterModel, 2465 config: Config, 2466 decoder: Decoder = JsonDecoder(parameters={}), 2467 query_properties_key: Optional[str] = None, 2468 use_cache: Optional[bool] = None, 2469 *, 2470 name: str, 2471 ) -> HttpRequester: 2472 authenticator = ( 2473 self._create_component_from_model( 2474 model=model.authenticator, 2475 config=config, 2476 url_base=model.url or model.url_base, 2477 name=name, 2478 decoder=decoder, 2479 ) 2480 if model.authenticator 2481 else None 2482 ) 2483 error_handler = ( 2484 self._create_component_from_model(model=model.error_handler, config=config) 2485 if model.error_handler 2486 else DefaultErrorHandler( 2487 backoff_strategies=[], 2488 response_filters=[], 2489 config=config, 2490 parameters=model.parameters or {}, 2491 ) 2492 ) 2493 2494 api_budget = self._api_budget 2495 2496 request_options_provider = InterpolatedRequestOptionsProvider( 2497 request_body=model.request_body, 2498 request_body_data=model.request_body_data, 2499 request_body_json=model.request_body_json, 2500 request_headers=model.request_headers, 2501 request_parameters=model.request_parameters, # type: ignore # QueryProperties have been removed in `create_simple_retriever` 2502 query_properties_key=query_properties_key, 2503 config=config, 2504 parameters=model.parameters or {}, 2505 ) 2506 2507 assert model.use_cache is not None # for mypy 2508 assert model.http_method is not None # for mypy 2509 2510 should_use_cache = (model.use_cache or bool(use_cache)) and not self._disable_cache 2511 2512 return HttpRequester( 2513 name=name, 2514 url=model.url, 2515 url_base=model.url_base, 2516 path=model.path, 2517 authenticator=authenticator, 2518 error_handler=error_handler, 2519 api_budget=api_budget, 2520 http_method=HttpMethod[model.http_method.value], 2521 request_options_provider=request_options_provider, 2522 config=config, 2523 disable_retries=self._disable_retries, 2524 parameters=model.parameters or {}, 2525 message_repository=self._message_repository, 2526 use_cache=should_use_cache, 2527 decoder=decoder, 2528 stream_response=decoder.is_stream_response() if decoder else False, 2529 ) 2530 2531 @staticmethod 2532 def create_http_response_filter( 2533 model: HttpResponseFilterModel, config: Config, **kwargs: Any 2534 ) -> HttpResponseFilter: 2535 if model.action: 2536 action = ResponseAction(model.action.value) 2537 else: 2538 action = None 2539 2540 failure_type = FailureType(model.failure_type.value) if model.failure_type else None 2541 2542 http_codes = ( 2543 set(model.http_codes) if model.http_codes else set() 2544 ) # JSON schema notation has no set data type. The schema enforces an array of unique elements 2545 2546 return HttpResponseFilter( 2547 action=action, 2548 failure_type=failure_type, 2549 error_message=model.error_message or "", 2550 error_message_contains=model.error_message_contains or "", 2551 http_codes=http_codes, 2552 predicate=model.predicate or "", 2553 config=config, 2554 parameters=model.parameters or {}, 2555 ) 2556 2557 @staticmethod 2558 def create_inline_schema_loader( 2559 model: InlineSchemaLoaderModel, config: Config, **kwargs: Any 2560 ) -> InlineSchemaLoader: 2561 return InlineSchemaLoader(schema=model.schema_ or {}, parameters={}) 2562 2563 def create_complex_field_type( 2564 self, model: ComplexFieldTypeModel, config: Config, **kwargs: Any 2565 ) -> ComplexFieldType: 2566 items = ( 2567 self._create_component_from_model(model=model.items, config=config) 2568 if isinstance(model.items, ComplexFieldTypeModel) 2569 else model.items 2570 ) 2571 2572 return ComplexFieldType(field_type=model.field_type, items=items) 2573 2574 def create_types_map(self, model: TypesMapModel, config: Config, **kwargs: Any) -> TypesMap: 2575 target_type = ( 2576 self._create_component_from_model(model=model.target_type, config=config) 2577 if isinstance(model.target_type, ComplexFieldTypeModel) 2578 else model.target_type 2579 ) 2580 2581 return TypesMap( 2582 target_type=target_type, 2583 current_type=model.current_type, 2584 condition=model.condition if model.condition is not None else "True", 2585 ) 2586 2587 def create_schema_type_identifier( 2588 self, model: SchemaTypeIdentifierModel, config: Config, **kwargs: Any 2589 ) -> SchemaTypeIdentifier: 2590 types_mapping = [] 2591 if model.types_mapping: 2592 types_mapping.extend( 2593 [ 2594 self._create_component_from_model(types_map, config=config) 2595 for types_map in model.types_mapping 2596 ] 2597 ) 2598 model_schema_pointer: List[Union[InterpolatedString, str]] = ( 2599 [x for x in model.schema_pointer] if model.schema_pointer else [] 2600 ) 2601 model_key_pointer: List[Union[InterpolatedString, str]] = [x for x in model.key_pointer] 2602 model_type_pointer: Optional[List[Union[InterpolatedString, str]]] = ( 2603 [x for x in model.type_pointer] if model.type_pointer else None 2604 ) 2605 2606 return SchemaTypeIdentifier( 2607 schema_pointer=model_schema_pointer, 2608 key_pointer=model_key_pointer, 2609 type_pointer=model_type_pointer, 2610 types_mapping=types_mapping, 2611 parameters=model.parameters or {}, 2612 ) 2613 2614 def create_dynamic_schema_loader( 2615 self, model: DynamicSchemaLoaderModel, config: Config, **kwargs: Any 2616 ) -> DynamicSchemaLoader: 2617 schema_transformations = [] 2618 if model.schema_transformations: 2619 for transformation_model in model.schema_transformations: 2620 schema_transformations.append( 2621 self._create_component_from_model(model=transformation_model, config=config) 2622 ) 2623 name = "dynamic_properties" 2624 retriever = self._create_component_from_model( 2625 model=model.retriever, 2626 config=config, 2627 name=name, 2628 primary_key=None, 2629 partition_router=self._build_stream_slicer_from_partition_router( 2630 model.retriever, config 2631 ), 2632 transformations=[], 2633 use_cache=True, 2634 log_formatter=( 2635 lambda response: format_http_message( 2636 response, 2637 f"Schema loader '{name}' request", 2638 f"Request performed in order to extract schema.", 2639 name, 2640 is_auxiliary=True, 2641 ) 2642 ), 2643 ) 2644 schema_type_identifier = self._create_component_from_model( 2645 model.schema_type_identifier, config=config, parameters=model.parameters or {} 2646 ) 2647 schema_filter = ( 2648 self._create_component_from_model( 2649 model.schema_filter, config=config, parameters=model.parameters or {} 2650 ) 2651 if model.schema_filter is not None 2652 else None 2653 ) 2654 2655 return DynamicSchemaLoader( 2656 retriever=retriever, 2657 config=config, 2658 schema_transformations=schema_transformations, 2659 schema_filter=schema_filter, 2660 schema_type_identifier=schema_type_identifier, 2661 parameters=model.parameters or {}, 2662 ) 2663 2664 @staticmethod 2665 def create_json_decoder(model: JsonDecoderModel, config: Config, **kwargs: Any) -> Decoder: 2666 return JsonDecoder(parameters={}) 2667 2668 def create_csv_decoder(self, model: CsvDecoderModel, config: Config, **kwargs: Any) -> Decoder: 2669 return CompositeRawDecoder( 2670 parser=ModelToComponentFactory._get_parser(model, config), 2671 stream_response=False if self._emit_connector_builder_messages else True, 2672 ) 2673 2674 def create_jsonl_decoder( 2675 self, model: JsonlDecoderModel, config: Config, **kwargs: Any 2676 ) -> Decoder: 2677 return CompositeRawDecoder( 2678 parser=ModelToComponentFactory._get_parser(model, config), 2679 stream_response=False if self._emit_connector_builder_messages else True, 2680 ) 2681 2682 def create_json_items_decoder( 2683 self, model: JsonItemsDecoderModel, config: Config, **kwargs: Any 2684 ) -> Decoder: 2685 return CompositeRawDecoder( 2686 parser=ModelToComponentFactory._get_parser(model, config), 2687 stream_response=False if self._emit_connector_builder_messages else True, 2688 ) 2689 2690 def create_gzip_decoder( 2691 self, model: GzipDecoderModel, config: Config, **kwargs: Any 2692 ) -> Decoder: 2693 _compressed_response_types = { 2694 "gzip", 2695 "x-gzip", 2696 "gzip, deflate", 2697 "x-gzip, deflate", 2698 "application/zip", 2699 "application/gzip", 2700 "application/x-gzip", 2701 "application/x-zip-compressed", 2702 } 2703 2704 gzip_parser: GzipParser = ModelToComponentFactory._get_parser(model, config) # type: ignore # based on the model, we know this will be a GzipParser 2705 2706 if self._emit_connector_builder_messages: 2707 # This is very surprising but if the response is not streamed, 2708 # CompositeRawDecoder calls response.content and the requests library actually uncompress the data as opposed to response.raw, 2709 # which uses urllib3 directly and does not uncompress the data. 2710 return CompositeRawDecoder(gzip_parser.inner_parser, False) 2711 2712 return CompositeRawDecoder.by_headers( 2713 [({"Content-Encoding", "Content-Type"}, _compressed_response_types, gzip_parser)], 2714 stream_response=True, 2715 fallback_parser=gzip_parser.inner_parser, 2716 ) 2717 2718 @staticmethod 2719 def create_iterable_decoder( 2720 model: IterableDecoderModel, config: Config, **kwargs: Any 2721 ) -> IterableDecoder: 2722 return IterableDecoder(parameters={}) 2723 2724 @staticmethod 2725 def create_xml_decoder(model: XmlDecoderModel, config: Config, **kwargs: Any) -> XmlDecoder: 2726 return XmlDecoder(parameters={}) 2727 2728 def create_zipfile_decoder( 2729 self, model: ZipfileDecoderModel, config: Config, **kwargs: Any 2730 ) -> ZipfileDecoder: 2731 return ZipfileDecoder(parser=ModelToComponentFactory._get_parser(model.decoder, config)) 2732 2733 @staticmethod 2734 def _get_parser(model: BaseModel, config: Config) -> Parser: 2735 if isinstance(model, JsonDecoderModel): 2736 # Note that the logic is a bit different from the JsonDecoder as there is some legacy that is maintained to return {} on error cases 2737 return JsonParser() 2738 elif isinstance(model, JsonItemsDecoderModel): 2739 return JsonItemsParser( 2740 items_path=model.items_path, 2741 encoding=model.encoding, 2742 ) 2743 elif isinstance(model, JsonlDecoderModel): 2744 return JsonLineParser() 2745 elif isinstance(model, CsvDecoderModel): 2746 return CsvParser( 2747 encoding=model.encoding, 2748 delimiter=model.delimiter, 2749 set_values_to_none=model.set_values_to_none, 2750 ) 2751 elif isinstance(model, GzipDecoderModel): 2752 return GzipParser( 2753 inner_parser=ModelToComponentFactory._get_parser(model.decoder, config) 2754 ) 2755 elif isinstance( 2756 model, (CustomDecoderModel, IterableDecoderModel, XmlDecoderModel, ZipfileDecoderModel) 2757 ): 2758 raise ValueError(f"Decoder type {model} does not have parser associated to it") 2759 2760 raise ValueError(f"Unknown decoder type {model}") 2761 2762 @staticmethod 2763 def create_json_file_schema_loader( 2764 model: JsonFileSchemaLoaderModel, config: Config, **kwargs: Any 2765 ) -> JsonFileSchemaLoader: 2766 return JsonFileSchemaLoader( 2767 file_path=model.file_path or "", config=config, parameters=model.parameters or {} 2768 ) 2769 2770 def create_jwt_authenticator( 2771 self, model: JwtAuthenticatorModel, config: Config, **kwargs: Any 2772 ) -> JwtAuthenticator: 2773 jwt_headers = model.jwt_headers or JwtHeadersModel(kid=None, typ="JWT", cty=None) 2774 jwt_payload = model.jwt_payload or JwtPayloadModel(iss=None, sub=None, aud=None) 2775 request_option = ( 2776 self._create_component_from_model(model.request_option, config) 2777 if model.request_option 2778 else None 2779 ) 2780 return JwtAuthenticator( 2781 config=config, 2782 parameters=model.parameters or {}, 2783 algorithm=JwtAlgorithm(model.algorithm.value), 2784 secret_key=model.secret_key, 2785 base64_encode_secret_key=model.base64_encode_secret_key, 2786 token_duration=model.token_duration, 2787 header_prefix=model.header_prefix, 2788 kid=jwt_headers.kid, 2789 typ=jwt_headers.typ, 2790 cty=jwt_headers.cty, 2791 iss=jwt_payload.iss, 2792 sub=jwt_payload.sub, 2793 aud=jwt_payload.aud, 2794 additional_jwt_headers=model.additional_jwt_headers, 2795 additional_jwt_payload=model.additional_jwt_payload, 2796 passphrase=model.passphrase, 2797 request_option=request_option, 2798 ) 2799 2800 def create_list_partition_router( 2801 self, model: ListPartitionRouterModel, config: Config, **kwargs: Any 2802 ) -> ListPartitionRouter: 2803 request_option = ( 2804 self._create_component_from_model(model.request_option, config) 2805 if model.request_option 2806 else None 2807 ) 2808 return ListPartitionRouter( 2809 cursor_field=model.cursor_field, 2810 request_option=request_option, 2811 values=model.values, 2812 config=config, 2813 parameters=model.parameters or {}, 2814 ) 2815 2816 @staticmethod 2817 def create_min_max_datetime( 2818 model: MinMaxDatetimeModel, config: Config, **kwargs: Any 2819 ) -> MinMaxDatetime: 2820 return MinMaxDatetime( 2821 datetime=model.datetime, 2822 datetime_format=model.datetime_format or "", 2823 max_datetime=model.max_datetime or "", 2824 min_datetime=model.min_datetime or "", 2825 parameters=model.parameters or {}, 2826 ) 2827 2828 @staticmethod 2829 def create_no_auth(model: NoAuthModel, config: Config, **kwargs: Any) -> NoAuth: 2830 return NoAuth(parameters=model.parameters or {}) 2831 2832 @staticmethod 2833 def create_no_pagination( 2834 model: NoPaginationModel, config: Config, **kwargs: Any 2835 ) -> NoPagination: 2836 return NoPagination(parameters={}) 2837 2838 def create_oauth_authenticator( 2839 self, model: OAuthAuthenticatorModel, config: Config, **kwargs: Any 2840 ) -> DeclarativeOauth2Authenticator: 2841 profile_assertion = ( 2842 self._create_component_from_model(model.profile_assertion, config=config) 2843 if model.profile_assertion 2844 else None 2845 ) 2846 2847 refresh_token_error_status_codes, refresh_token_error_key, refresh_token_error_values = ( 2848 self._get_refresh_token_error_information(model) 2849 ) 2850 if model.refresh_token_updater: 2851 # ignore type error because fixing it would have a lot of dependencies, revisit later 2852 return DeclarativeSingleUseRefreshTokenOauth2Authenticator( # type: ignore 2853 config, 2854 InterpolatedString.create( 2855 model.token_refresh_endpoint, # type: ignore 2856 parameters=model.parameters or {}, 2857 ).eval(config), 2858 access_token_name=InterpolatedString.create( 2859 model.access_token_name or "access_token", parameters=model.parameters or {} 2860 ).eval(config), 2861 refresh_token_name=model.refresh_token_updater.refresh_token_name, 2862 expires_in_name=InterpolatedString.create( 2863 model.expires_in_name or "expires_in", parameters=model.parameters or {} 2864 ).eval(config), 2865 client_id_name=InterpolatedString.create( 2866 model.client_id_name or "client_id", parameters=model.parameters or {} 2867 ).eval(config), 2868 client_id=InterpolatedString.create( 2869 model.client_id, parameters=model.parameters or {} 2870 ).eval(config) 2871 if model.client_id 2872 else model.client_id, 2873 client_secret_name=InterpolatedString.create( 2874 model.client_secret_name or "client_secret", parameters=model.parameters or {} 2875 ).eval(config), 2876 client_secret=InterpolatedString.create( 2877 model.client_secret, parameters=model.parameters or {} 2878 ).eval(config) 2879 if model.client_secret 2880 else model.client_secret, 2881 access_token_config_path=model.refresh_token_updater.access_token_config_path, 2882 refresh_token_config_path=model.refresh_token_updater.refresh_token_config_path, 2883 token_expiry_date_config_path=model.refresh_token_updater.token_expiry_date_config_path, 2884 grant_type_name=InterpolatedString.create( 2885 model.grant_type_name or "grant_type", parameters=model.parameters or {} 2886 ).eval(config), 2887 grant_type=InterpolatedString.create( 2888 model.grant_type or "refresh_token", parameters=model.parameters or {} 2889 ).eval(config), 2890 refresh_request_body=InterpolatedMapping( 2891 model.refresh_request_body or {}, parameters=model.parameters or {} 2892 ).eval(config), 2893 refresh_request_headers=InterpolatedMapping( 2894 model.refresh_request_headers or {}, parameters=model.parameters or {} 2895 ).eval(config), 2896 send_refresh_request_as_query_params=bool( 2897 model.send_refresh_request_as_query_params 2898 ), 2899 scopes=model.scopes, 2900 token_expiry_date_format=model.token_expiry_date_format, 2901 token_expiry_is_time_of_expiration=bool(model.token_expiry_date_format), 2902 message_repository=self._message_repository, 2903 refresh_token_error_status_codes=refresh_token_error_status_codes, 2904 refresh_token_error_key=refresh_token_error_key, 2905 refresh_token_error_values=refresh_token_error_values, 2906 ) 2907 # ignore type error because fixing it would have a lot of dependencies, revisit later 2908 return DeclarativeOauth2Authenticator( # type: ignore 2909 access_token_name=model.access_token_name or "access_token", 2910 access_token_value=model.access_token_value, 2911 client_id_name=model.client_id_name or "client_id", 2912 client_id=model.client_id, 2913 client_secret_name=model.client_secret_name or "client_secret", 2914 client_secret=model.client_secret, 2915 expires_in_name=model.expires_in_name or "expires_in", 2916 grant_type_name=model.grant_type_name or "grant_type", 2917 grant_type=model.grant_type or "refresh_token", 2918 refresh_request_body=model.refresh_request_body, 2919 refresh_request_headers=model.refresh_request_headers, 2920 send_refresh_request_as_query_params=bool(model.send_refresh_request_as_query_params), 2921 refresh_token_name=model.refresh_token_name or "refresh_token", 2922 refresh_token=model.refresh_token, 2923 scopes=model.scopes, 2924 token_expiry_date=model.token_expiry_date, 2925 token_expiry_date_format=model.token_expiry_date_format, 2926 token_expiry_is_time_of_expiration=bool(model.token_expiry_date_format), 2927 token_refresh_endpoint=model.token_refresh_endpoint, 2928 config=config, 2929 parameters=model.parameters or {}, 2930 message_repository=self._message_repository, 2931 profile_assertion=profile_assertion, 2932 use_profile_assertion=model.use_profile_assertion, 2933 refresh_token_error_status_codes=refresh_token_error_status_codes, 2934 refresh_token_error_key=refresh_token_error_key, 2935 refresh_token_error_values=refresh_token_error_values, 2936 ) 2937 2938 @staticmethod 2939 def _get_refresh_token_error_information( 2940 model: OAuthAuthenticatorModel, 2941 ) -> Tuple[Tuple[int, ...], str, Tuple[str, ...]]: 2942 """ 2943 In a previous version of the CDK, the auth error as config_error was only done if a refresh token updater was 2944 defined. As a transition, we added those fields on the OAuthAuthenticatorModel. This method ensures that the 2945 information is defined only once and return the right fields. 2946 """ 2947 refresh_token_updater = model.refresh_token_updater 2948 is_defined_on_refresh_token_updated = refresh_token_updater and ( 2949 refresh_token_updater.refresh_token_error_status_codes 2950 or refresh_token_updater.refresh_token_error_key 2951 or refresh_token_updater.refresh_token_error_values 2952 ) 2953 is_defined_on_oauth_authenticator = ( 2954 model.refresh_token_error_status_codes 2955 or model.refresh_token_error_key 2956 or model.refresh_token_error_values 2957 ) 2958 if is_defined_on_refresh_token_updated and is_defined_on_oauth_authenticator: 2959 raise ValueError( 2960 "refresh_token_error should either be defined on the OAuthAuthenticatorModel or the RefreshTokenUpdaterModel, not both" 2961 ) 2962 2963 if is_defined_on_refresh_token_updated: 2964 not_optional_refresh_token_updater: RefreshTokenUpdaterModel = refresh_token_updater # type: ignore # we know from the condition that this is not None 2965 return ( 2966 tuple(not_optional_refresh_token_updater.refresh_token_error_status_codes) 2967 if not_optional_refresh_token_updater.refresh_token_error_status_codes 2968 else (), 2969 not_optional_refresh_token_updater.refresh_token_error_key or "", 2970 tuple(not_optional_refresh_token_updater.refresh_token_error_values) 2971 if not_optional_refresh_token_updater.refresh_token_error_values 2972 else (), 2973 ) 2974 elif is_defined_on_oauth_authenticator: 2975 return ( 2976 tuple(model.refresh_token_error_status_codes) 2977 if model.refresh_token_error_status_codes 2978 else (), 2979 model.refresh_token_error_key or "", 2980 tuple(model.refresh_token_error_values) if model.refresh_token_error_values else (), 2981 ) 2982 2983 # returning default values we think cover most cases 2984 return (400,), "error", ("invalid_grant", "invalid_permissions") 2985 2986 def create_offset_increment( 2987 self, 2988 model: OffsetIncrementModel, 2989 config: Config, 2990 decoder: Decoder, 2991 extractor_model: Optional[Union[CustomRecordExtractorModel, DpathExtractorModel]] = None, 2992 **kwargs: Any, 2993 ) -> OffsetIncrement: 2994 if isinstance(decoder, PaginationDecoderDecorator): 2995 inner_decoder = decoder.decoder 2996 else: 2997 inner_decoder = decoder 2998 decoder = PaginationDecoderDecorator(decoder=decoder) 2999 3000 if self._is_supported_decoder_for_pagination(inner_decoder): 3001 decoder_to_use = decoder 3002 else: 3003 raise ValueError( 3004 self._UNSUPPORTED_DECODER_ERROR.format(decoder_type=type(inner_decoder)) 3005 ) 3006 3007 # Ideally we would instantiate the runtime extractor from highest most level (in this case the SimpleRetriever) 3008 # so that it can be shared by OffSetIncrement and RecordSelector. However, due to how we instantiate the 3009 # decoder with various decorators here, but not in create_record_selector, it is simpler to retain existing 3010 # behavior by having two separate extractors with identical behavior since they use the same extractor model. 3011 # When we have more time to investigate we can look into reusing the same component. 3012 extractor = ( 3013 self._create_component_from_model( 3014 model=extractor_model, config=config, decoder=decoder_to_use 3015 ) 3016 if extractor_model 3017 else None 3018 ) 3019 3020 # Pydantic v1 Union type coercion can convert int to string depending on Union order. 3021 # If page_size is a string that represents an integer (not an interpolation), convert it back. 3022 page_size = model.page_size 3023 if isinstance(page_size, str) and page_size.isdigit(): 3024 page_size = int(page_size) 3025 3026 return OffsetIncrement( 3027 page_size=page_size, 3028 config=config, 3029 decoder=decoder_to_use, 3030 extractor=extractor, 3031 inject_on_first_request=model.inject_on_first_request or False, 3032 parameters=model.parameters or {}, 3033 ) 3034 3035 def create_page_increment( 3036 self, 3037 model: PageIncrementModel, 3038 config: Config, 3039 decoder: Optional[Decoder] = None, 3040 extractor_model: Optional[Union[CustomRecordExtractorModel, DpathExtractorModel]] = None, 3041 **kwargs: Any, 3042 ) -> PageIncrement: 3043 # Like OffsetIncrement, we instantiate a separate extractor with identical behavior to the 3044 # RecordSelector's so the strategy can count the raw records in the response. This ensures 3045 # pagination is driven by the API's page size, not the post-filter record count. 3046 extractor = ( 3047 self._create_component_from_model(model=extractor_model, config=config, decoder=decoder) 3048 if extractor_model 3049 else None 3050 ) 3051 3052 # Pydantic v1 Union type coercion can convert int to string depending on Union order. 3053 # If page_size is a string that represents an integer (not an interpolation), convert it back. 3054 page_size = model.page_size 3055 if isinstance(page_size, str) and page_size.isdigit(): 3056 page_size = int(page_size) 3057 3058 return PageIncrement( 3059 page_size=page_size, 3060 config=config, 3061 start_from_page=model.start_from_page or 0, 3062 inject_on_first_request=model.inject_on_first_request or False, 3063 extractor=extractor, 3064 parameters=model.parameters or {}, 3065 ) 3066 3067 def create_parent_stream_config( 3068 self, model: ParentStreamConfigModel, config: Config, *, stream_name: str, **kwargs: Any 3069 ) -> ParentStreamConfig: 3070 declarative_stream = self._create_component_from_model( 3071 model.stream, 3072 config=config, 3073 is_parent=True, 3074 **kwargs, 3075 ) 3076 request_option = ( 3077 self._create_component_from_model(model.request_option, config=config) 3078 if model.request_option 3079 else None 3080 ) 3081 3082 if model.lazy_read_pointer and any("*" in pointer for pointer in model.lazy_read_pointer): 3083 raise ValueError( 3084 "The '*' wildcard in 'lazy_read_pointer' is not supported — only direct paths are allowed." 3085 ) 3086 3087 model_lazy_read_pointer: List[Union[InterpolatedString, str]] = ( 3088 [x for x in model.lazy_read_pointer] if model.lazy_read_pointer else [] 3089 ) 3090 3091 return ParentStreamConfig( 3092 parent_key=model.parent_key, 3093 request_option=request_option, 3094 stream=declarative_stream, 3095 partition_field=model.partition_field, 3096 config=config, 3097 incremental_dependency=model.incremental_dependency or False, 3098 parameters=model.parameters or {}, 3099 extra_fields=model.extra_fields, 3100 lazy_read_pointer=model_lazy_read_pointer, 3101 ) 3102 3103 def create_properties_from_endpoint( 3104 self, model: PropertiesFromEndpointModel, config: Config, **kwargs: Any 3105 ) -> PropertiesFromEndpoint: 3106 retriever = self._create_component_from_model( 3107 model=model.retriever, 3108 config=config, 3109 name="dynamic_properties", 3110 primary_key=None, 3111 stream_slicer=None, 3112 transformations=[], 3113 use_cache=True, # Enable caching on the HttpRequester/HttpClient because the properties endpoint will be called for every slice being processed, and it is highly unlikely for the response to different 3114 ) 3115 return PropertiesFromEndpoint( 3116 property_field_path=model.property_field_path, 3117 retriever=retriever, 3118 config=config, 3119 parameters=model.parameters or {}, 3120 ) 3121 3122 def create_property_chunking( 3123 self, model: PropertyChunkingModel, config: Config, **kwargs: Any 3124 ) -> PropertyChunking: 3125 record_merge_strategy = ( 3126 self._create_component_from_model( 3127 model=model.record_merge_strategy, config=config, **kwargs 3128 ) 3129 if model.record_merge_strategy 3130 else None 3131 ) 3132 3133 property_limit_type: PropertyLimitType 3134 match model.property_limit_type: 3135 case PropertyLimitTypeModel.property_count: 3136 property_limit_type = PropertyLimitType.property_count 3137 case PropertyLimitTypeModel.characters: 3138 property_limit_type = PropertyLimitType.characters 3139 case _: 3140 raise ValueError(f"Invalid PropertyLimitType {property_limit_type}") 3141 3142 return PropertyChunking( 3143 property_limit_type=property_limit_type, 3144 property_limit=model.property_limit, 3145 record_merge_strategy=record_merge_strategy, 3146 config=config, 3147 parameters=model.parameters or {}, 3148 ) 3149 3150 def create_query_properties( 3151 self, model: QueryPropertiesModel, config: Config, *, stream_name: str, **kwargs: Any 3152 ) -> QueryProperties: 3153 if isinstance(model.property_list, list): 3154 property_list = model.property_list 3155 else: 3156 property_list = self._create_component_from_model( 3157 model=model.property_list, config=config, **kwargs 3158 ) 3159 3160 property_chunking = ( 3161 self._create_component_from_model( 3162 model=model.property_chunking, config=config, **kwargs 3163 ) 3164 if model.property_chunking 3165 else None 3166 ) 3167 3168 property_selector = ( 3169 self._create_component_from_model( 3170 model=model.property_selector, config=config, stream_name=stream_name, **kwargs 3171 ) 3172 if model.property_selector 3173 else None 3174 ) 3175 3176 return QueryProperties( 3177 property_list=property_list, 3178 always_include_properties=model.always_include_properties, 3179 property_chunking=property_chunking, 3180 property_selector=property_selector, 3181 config=config, 3182 parameters=model.parameters or {}, 3183 ) 3184 3185 def create_json_schema_property_selector( 3186 self, 3187 model: JsonSchemaPropertySelectorModel, 3188 config: Config, 3189 *, 3190 stream_name: str, 3191 **kwargs: Any, 3192 ) -> JsonSchemaPropertySelector: 3193 configured_stream = self._stream_name_to_configured_stream.get(stream_name) 3194 3195 transformations = [] 3196 if model.transformations: 3197 for transformation_model in model.transformations: 3198 transformations.append( 3199 self._create_component_from_model(model=transformation_model, config=config) 3200 ) 3201 3202 return JsonSchemaPropertySelector( 3203 configured_stream=configured_stream, 3204 properties_transformations=transformations, 3205 config=config, 3206 parameters=model.parameters or {}, 3207 ) 3208 3209 @staticmethod 3210 def create_record_filter( 3211 model: RecordFilterModel, config: Config, **kwargs: Any 3212 ) -> RecordFilter: 3213 return RecordFilter( 3214 condition=model.condition or "", config=config, parameters=model.parameters or {} 3215 ) 3216 3217 @staticmethod 3218 def create_request_path(model: RequestPathModel, config: Config, **kwargs: Any) -> RequestPath: 3219 return RequestPath(parameters={}) 3220 3221 @staticmethod 3222 def create_request_option( 3223 model: RequestOptionModel, config: Config, **kwargs: Any 3224 ) -> RequestOption: 3225 inject_into = RequestOptionType(model.inject_into.value) 3226 field_path: Optional[List[Union[InterpolatedString, str]]] = ( 3227 [ 3228 InterpolatedString.create(segment, parameters=kwargs.get("parameters", {})) 3229 for segment in model.field_path 3230 ] 3231 if model.field_path 3232 else None 3233 ) 3234 field_name = ( 3235 InterpolatedString.create(model.field_name, parameters=kwargs.get("parameters", {})) 3236 if model.field_name 3237 else None 3238 ) 3239 return RequestOption( 3240 field_name=field_name, 3241 field_path=field_path, 3242 inject_into=inject_into, 3243 parameters=kwargs.get("parameters", {}), 3244 ) 3245 3246 def create_record_selector( 3247 self, 3248 model: RecordSelectorModel, 3249 config: Config, 3250 *, 3251 name: str, 3252 transformations: List[RecordTransformation] | None = None, 3253 decoder: Decoder | None = None, 3254 client_side_incremental_sync_cursor: Optional[Cursor] = None, 3255 file_uploader: Optional[DefaultFileUploader] = None, 3256 **kwargs: Any, 3257 ) -> RecordSelector: 3258 extractor = self._create_component_from_model( 3259 model=model.extractor, decoder=decoder, config=config 3260 ) 3261 record_filter = ( 3262 self._create_component_from_model(model.record_filter, config=config) 3263 if model.record_filter 3264 else None 3265 ) 3266 3267 transform_before_filtering = ( 3268 False if model.transform_before_filtering is None else model.transform_before_filtering 3269 ) 3270 if client_side_incremental_sync_cursor: 3271 record_filter = ClientSideIncrementalRecordFilterDecorator( 3272 config=config, 3273 parameters=model.parameters, 3274 condition=model.record_filter.condition 3275 if (model.record_filter and hasattr(model.record_filter, "condition")) 3276 else None, 3277 cursor=client_side_incremental_sync_cursor, 3278 ) 3279 transform_before_filtering = ( 3280 True 3281 if model.transform_before_filtering is None 3282 else model.transform_before_filtering 3283 ) 3284 3285 if model.schema_normalization is None: 3286 # default to no schema normalization if not set 3287 model.schema_normalization = SchemaNormalizationModel.None_ 3288 3289 schema_normalization = ( 3290 TypeTransformer(SCHEMA_TRANSFORMER_TYPE_MAPPING[model.schema_normalization]) 3291 if isinstance(model.schema_normalization, SchemaNormalizationModel) 3292 else self._create_component_from_model(model.schema_normalization, config=config) # type: ignore[arg-type] # custom normalization model expected here 3293 ) 3294 3295 return RecordSelector( 3296 extractor=extractor, 3297 name=name, 3298 config=config, 3299 record_filter=record_filter, 3300 transformations=transformations or [], 3301 file_uploader=file_uploader, 3302 schema_normalization=schema_normalization, 3303 parameters=model.parameters or {}, 3304 transform_before_filtering=transform_before_filtering, 3305 ) 3306 3307 @staticmethod 3308 def create_remove_fields( 3309 model: RemoveFieldsModel, config: Config, **kwargs: Any 3310 ) -> RemoveFields: 3311 return RemoveFields( 3312 field_pointers=model.field_pointers, condition=model.condition or "", parameters={} 3313 ) 3314 3315 def create_selective_authenticator( 3316 self, model: SelectiveAuthenticatorModel, config: Config, **kwargs: Any 3317 ) -> DeclarativeAuthenticator: 3318 authenticators = { 3319 name: self._create_component_from_model(model=auth, config=config) 3320 for name, auth in model.authenticators.items() 3321 } 3322 # SelectiveAuthenticator will return instance of DeclarativeAuthenticator or raise ValueError error 3323 return SelectiveAuthenticator( # type: ignore[abstract] 3324 config=config, 3325 authenticators=authenticators, 3326 authenticator_selection_path=model.authenticator_selection_path, 3327 **kwargs, 3328 ) 3329 3330 @staticmethod 3331 def create_legacy_session_token_authenticator( 3332 model: LegacySessionTokenAuthenticatorModel, config: Config, *, url_base: str, **kwargs: Any 3333 ) -> LegacySessionTokenAuthenticator: 3334 return LegacySessionTokenAuthenticator( 3335 api_url=url_base, 3336 header=model.header, 3337 login_url=model.login_url, 3338 password=model.password or "", 3339 session_token=model.session_token or "", 3340 session_token_response_key=model.session_token_response_key or "", 3341 username=model.username or "", 3342 validate_session_url=model.validate_session_url, 3343 config=config, 3344 parameters=model.parameters or {}, 3345 ) 3346 3347 def create_simple_retriever( 3348 self, 3349 model: SimpleRetrieverModel, 3350 config: Config, 3351 *, 3352 name: str, 3353 primary_key: Optional[Union[str, List[str], List[List[str]]]], 3354 request_options_provider: Optional[RequestOptionsProvider] = None, 3355 cursor: Optional[Cursor] = None, 3356 has_stop_condition_cursor: bool = False, 3357 is_client_side_incremental_sync: bool = False, 3358 transformations: List[RecordTransformation], 3359 file_uploader: Optional[DefaultFileUploader] = None, 3360 incremental_sync: Optional[ 3361 Union[IncrementingCountCursorModel, DatetimeBasedCursorModel] 3362 ] = None, 3363 use_cache: Optional[bool] = None, 3364 log_formatter: Optional[Callable[[Response], Any]] = None, 3365 partition_router: Optional[PartitionRouter] = None, 3366 **kwargs: Any, 3367 ) -> SimpleRetriever: 3368 def _get_url(req: Requester) -> str: 3369 """ 3370 Closure to get the URL from the requester. This is used to get the URL in the case of a lazy retriever. 3371 This is needed because the URL is not set until the requester is created. 3372 """ 3373 3374 _url: str = ( 3375 model.requester.url 3376 if hasattr(model.requester, "url") and model.requester.url is not None 3377 else req.get_url(stream_state=None, stream_slice=None, next_page_token=None) 3378 ) 3379 _url_base: str = ( 3380 model.requester.url_base 3381 if hasattr(model.requester, "url_base") and model.requester.url_base is not None 3382 else req.get_url_base(stream_state=None, stream_slice=None, next_page_token=None) 3383 ) 3384 3385 return _url or _url_base 3386 3387 if cursor is None: 3388 cursor = FinalStateCursor(name, None, self._message_repository) 3389 3390 decoder = ( 3391 self._create_component_from_model(model=model.decoder, config=config) 3392 if model.decoder 3393 else JsonDecoder(parameters={}) 3394 ) 3395 record_selector = self._create_component_from_model( 3396 model=model.record_selector, 3397 name=name, 3398 config=config, 3399 decoder=decoder, 3400 transformations=transformations, 3401 client_side_incremental_sync_cursor=cursor if is_client_side_incremental_sync else None, 3402 file_uploader=file_uploader, 3403 ) 3404 3405 query_properties: Optional[QueryProperties] = None 3406 query_properties_key: Optional[str] = None 3407 self._ensure_query_properties_to_model(model.requester) 3408 if self._has_query_properties_in_request_parameters(model.requester): 3409 # It is better to be explicit about an error if PropertiesFromEndpoint is defined in multiple 3410 # places instead of default to request_parameters which isn't clearly documented 3411 if ( 3412 hasattr(model.requester, "fetch_properties_from_endpoint") 3413 and model.requester.fetch_properties_from_endpoint 3414 ): 3415 raise ValueError( 3416 f"PropertiesFromEndpoint should only be specified once per stream, but found in {model.requester.type}.fetch_properties_from_endpoint and {model.requester.type}.request_parameters" 3417 ) 3418 3419 query_properties_definitions = [] 3420 for key, request_parameter in model.requester.request_parameters.items(): # type: ignore # request_parameters is already validated to be a Mapping using _has_query_properties_in_request_parameters() 3421 if isinstance(request_parameter, QueryPropertiesModel): 3422 query_properties_key = key 3423 query_properties_definitions.append(request_parameter) 3424 3425 if len(query_properties_definitions) > 1: 3426 raise ValueError( 3427 f"request_parameters only supports defining one QueryProperties field, but found {len(query_properties_definitions)} usages" 3428 ) 3429 3430 if len(query_properties_definitions) == 1: 3431 query_properties = self._create_component_from_model( 3432 model=query_properties_definitions[0], stream_name=name, config=config 3433 ) 3434 3435 # Removes QueryProperties components from the interpolated mappings because it has been designed 3436 # to be used by the SimpleRetriever and will be resolved from the provider from the slice directly 3437 # instead of through jinja interpolation 3438 if hasattr(model.requester, "request_parameters") and isinstance( 3439 model.requester.request_parameters, Mapping 3440 ): 3441 model.requester.request_parameters = self._remove_query_properties( 3442 model.requester.request_parameters 3443 ) 3444 elif ( 3445 hasattr(model.requester, "fetch_properties_from_endpoint") 3446 and model.requester.fetch_properties_from_endpoint 3447 ): 3448 # todo: Deprecate this condition once dependent connectors migrate to query_properties 3449 query_properties_definition = QueryPropertiesModel( 3450 type="QueryProperties", 3451 property_list=model.requester.fetch_properties_from_endpoint, 3452 always_include_properties=None, 3453 property_chunking=None, 3454 ) # type: ignore # $parameters has a default value 3455 3456 query_properties = self.create_query_properties( 3457 model=query_properties_definition, 3458 stream_name=name, 3459 config=config, 3460 ) 3461 elif hasattr(model.requester, "query_properties") and model.requester.query_properties: 3462 query_properties = self.create_query_properties( 3463 model=model.requester.query_properties, 3464 stream_name=name, 3465 config=config, 3466 ) 3467 3468 requester = self._create_component_from_model( 3469 model=model.requester, 3470 decoder=decoder, 3471 name=name, 3472 query_properties_key=query_properties_key, 3473 use_cache=use_cache, 3474 config=config, 3475 ) 3476 3477 if not request_options_provider: 3478 request_options_provider = DefaultRequestOptionsProvider(parameters={}) 3479 if isinstance(request_options_provider, DefaultRequestOptionsProvider) and isinstance( 3480 partition_router, PartitionRouter 3481 ): 3482 request_options_provider = partition_router 3483 3484 paginator = ( 3485 self._create_component_from_model( 3486 model=model.paginator, 3487 config=config, 3488 url_base=_get_url(requester), 3489 extractor_model=model.record_selector.extractor, 3490 decoder=decoder, 3491 cursor_used_for_stop_condition=cursor if has_stop_condition_cursor else None, 3492 ) 3493 if model.paginator 3494 else NoPagination(parameters={}) 3495 ) 3496 3497 ignore_stream_slicer_parameters_on_paginated_requests = ( 3498 model.ignore_stream_slicer_parameters_on_paginated_requests or False 3499 ) 3500 3501 if ( 3502 model.partition_router 3503 and isinstance(model.partition_router, SubstreamPartitionRouterModel) 3504 and not bool(self._connector_state_manager.get_stream_state(name, None)) 3505 and any( 3506 parent_stream_config.lazy_read_pointer 3507 for parent_stream_config in model.partition_router.parent_stream_configs 3508 ) 3509 ): 3510 if incremental_sync: 3511 if incremental_sync.type != "DatetimeBasedCursor": 3512 raise ValueError( 3513 f"LazySimpleRetriever only supports DatetimeBasedCursor. Found: {incremental_sync.type}." 3514 ) 3515 3516 elif incremental_sync.step or incremental_sync.cursor_granularity: 3517 raise ValueError( 3518 f"Found more that one slice per parent. LazySimpleRetriever only supports single slice read for stream - {name}." 3519 ) 3520 3521 if model.decoder and model.decoder.type != "JsonDecoder": 3522 raise ValueError( 3523 f"LazySimpleRetriever only supports JsonDecoder. Found: {model.decoder.type}." 3524 ) 3525 3526 return LazySimpleRetriever( 3527 name=name, 3528 paginator=paginator, 3529 primary_key=primary_key, 3530 requester=requester, 3531 record_selector=record_selector, 3532 stream_slicer=_NO_STREAM_SLICING, 3533 request_option_provider=request_options_provider, 3534 config=config, 3535 ignore_stream_slicer_parameters_on_paginated_requests=ignore_stream_slicer_parameters_on_paginated_requests, 3536 parameters=model.parameters or {}, 3537 ) 3538 3539 if ( 3540 model.record_selector.record_filter 3541 and model.pagination_reset 3542 and model.pagination_reset.limits 3543 ): 3544 raise ValueError("PaginationResetLimits are not supported while having record filter.") 3545 3546 return SimpleRetriever( 3547 name=name, 3548 paginator=paginator, 3549 primary_key=primary_key, 3550 requester=requester, 3551 record_selector=record_selector, 3552 stream_slicer=_NO_STREAM_SLICING, 3553 request_option_provider=request_options_provider, 3554 config=config, 3555 ignore_stream_slicer_parameters_on_paginated_requests=ignore_stream_slicer_parameters_on_paginated_requests, 3556 additional_query_properties=query_properties, 3557 log_formatter=self._get_log_formatter(log_formatter, name), 3558 pagination_tracker_factory=self._create_pagination_tracker_factory( 3559 model.pagination_reset, cursor 3560 ), 3561 parameters=model.parameters or {}, 3562 ) 3563 3564 def _create_pagination_tracker_factory( 3565 self, model: Optional[PaginationResetModel], cursor: Cursor 3566 ) -> Callable[[], PaginationTracker]: 3567 if model is None: 3568 return lambda: PaginationTracker() 3569 3570 # Until we figure out a way to use any cursor for PaginationTracker, we will have to have this cursor selector logic 3571 cursor_factory: Callable[[], Optional[ConcurrentCursor]] = lambda: None 3572 if model.action == PaginationResetActionModel.RESET: 3573 # in that case, we will let cursor_factory to return None even if the stream has a cursor 3574 pass 3575 elif model.action == PaginationResetActionModel.SPLIT_USING_CURSOR: 3576 if isinstance(cursor, ConcurrentCursor): 3577 cursor_factory = lambda: cursor.copy_without_state() # type: ignore # the if condition validates that it is a ConcurrentCursor 3578 elif isinstance(cursor, ConcurrentPerPartitionCursor): 3579 cursor_factory = lambda: cursor._cursor_factory.create( # type: ignore # if this becomes a problem, we would need to extract the cursor_factory instantiation logic and make it accessible here 3580 {}, datetime.timedelta(0) 3581 ) 3582 elif not isinstance(cursor, FinalStateCursor): 3583 LOGGER.warning( 3584 "Unknown cursor for PaginationTracker. Pagination resets might not work properly" 3585 ) 3586 else: 3587 raise ValueError(f"Unknown PaginationReset action: {model.action}") 3588 3589 limit = model.limits.number_of_records if model and model.limits else None 3590 return lambda: PaginationTracker(cursor_factory(), limit) 3591 3592 def _get_log_formatter( 3593 self, log_formatter: Callable[[Response], Any] | None, name: str 3594 ) -> Callable[[Response], Any] | None: 3595 if self._should_limit_slices_fetched(): 3596 return ( 3597 ( 3598 lambda response: format_http_message( 3599 response, 3600 f"Stream '{name}' request", 3601 f"Request performed in order to extract records for stream '{name}'", 3602 name, 3603 ) 3604 ) 3605 if not log_formatter 3606 else log_formatter 3607 ) 3608 return None 3609 3610 def _should_limit_slices_fetched(self) -> bool: 3611 """ 3612 Returns True if the number of slices fetched should be limited, False otherwise. 3613 This is used to limit the number of slices fetched during tests. 3614 """ 3615 return bool(self._limit_slices_fetched or self._emit_connector_builder_messages) 3616 3617 @staticmethod 3618 def _has_query_properties_in_request_parameters( 3619 requester: Union[HttpRequesterModel, CustomRequesterModel], 3620 ) -> bool: 3621 if not hasattr(requester, "request_parameters"): 3622 return False 3623 request_parameters = requester.request_parameters 3624 if request_parameters and isinstance(request_parameters, Mapping): 3625 for request_parameter in request_parameters.values(): 3626 if isinstance(request_parameter, QueryPropertiesModel): 3627 return True 3628 return False 3629 3630 @staticmethod 3631 def _remove_query_properties( 3632 request_parameters: Mapping[str, Union[str, QueryPropertiesModel]], 3633 ) -> Mapping[str, str]: 3634 return { 3635 parameter_field: request_parameter 3636 for parameter_field, request_parameter in request_parameters.items() 3637 if not isinstance(request_parameter, QueryPropertiesModel) 3638 } 3639 3640 def create_state_delegating_stream( 3641 self, 3642 model: StateDelegatingStreamModel, 3643 config: Config, 3644 **kwargs: Any, 3645 ) -> DefaultStream: 3646 if ( 3647 model.full_refresh_stream.name != model.name 3648 or model.name != model.incremental_stream.name 3649 ): 3650 raise ValueError( 3651 f"state_delegating_stream, full_refresh_stream name and incremental_stream must have equal names. Instead has {model.name}, {model.full_refresh_stream.name} and {model.incremental_stream.name}." 3652 ) 3653 3654 # Resolve api_retention_period with config context (supports Jinja2 interpolation) 3655 resolved_retention_period: Optional[str] = None 3656 if model.api_retention_period: 3657 interpolated_retention = InterpolatedString.create( 3658 model.api_retention_period, parameters=model.parameters or {} 3659 ) 3660 resolved_value = interpolated_retention.eval(config=config) 3661 if resolved_value: 3662 resolved_retention_period = str(resolved_value) 3663 3664 if resolved_retention_period: 3665 for stream_model in (model.full_refresh_stream, model.incremental_stream): 3666 if isinstance(stream_model.incremental_sync, IncrementingCountCursorModel): 3667 raise ValueError( 3668 f"Stream '{model.name}' uses IncrementingCountCursor which is not supported " 3669 f"with api_retention_period. IncrementingCountCursor does not use datetime-based " 3670 f"cursors, so cursor age validation cannot be performed." 3671 ) 3672 3673 stream_state = self._connector_state_manager.get_stream_state(model.name, None) 3674 3675 if not stream_state: 3676 return self._create_component_from_model( # type: ignore[no-any-return] 3677 model.full_refresh_stream, config=config, **kwargs 3678 ) 3679 3680 incremental_stream: DefaultStream = self._create_component_from_model( 3681 model.incremental_stream, config=config, **kwargs 3682 ) # type: ignore[assignment] 3683 3684 # Only run cursor age validation for streams that are in the configured 3685 # catalog (or when no catalog was provided, e.g. during discover / connector 3686 # builder). Streams not selected by the user but instantiated as parent-stream 3687 # dependencies must not go through this path because it emits state messages 3688 # that the destination does not know about, causing "Stream not found" crashes. 3689 stream_is_in_catalog = ( 3690 not self._stream_name_to_configured_stream # no catalog → validate by default 3691 or model.name in self._stream_name_to_configured_stream 3692 ) 3693 if resolved_retention_period and stream_is_in_catalog: 3694 full_refresh_stream: DefaultStream = self._create_component_from_model( 3695 model.full_refresh_stream, config=config, **kwargs 3696 ) # type: ignore[assignment] 3697 if self._is_cursor_older_than_retention_period( 3698 stream_state, 3699 full_refresh_stream.cursor, 3700 incremental_stream.cursor, 3701 resolved_retention_period, 3702 model.name, 3703 ): 3704 # Clear state BEFORE constructing the full_refresh_stream so that 3705 # its cursor starts from start_date instead of the stale cursor. 3706 self._connector_state_manager.update_state_for_stream(model.name, None, {}) 3707 state_message = self._connector_state_manager.create_state_message(model.name, None) 3708 self._message_repository.emit_message(state_message) 3709 return self._create_component_from_model( # type: ignore[no-any-return] 3710 model.full_refresh_stream, config=config, **kwargs 3711 ) 3712 3713 return incremental_stream 3714 3715 @staticmethod 3716 def _is_cursor_older_than_retention_period( 3717 stream_state: Mapping[str, Any], 3718 full_refresh_cursor: Cursor, 3719 incremental_cursor: Cursor, 3720 api_retention_period: str, 3721 stream_name: str, 3722 ) -> bool: 3723 """Check if the cursor value in the state is older than the API's retention period. 3724 3725 Checks cursors in sequence: full refresh cursor first, then incremental cursor. 3726 FinalStateCursor returns now() for completed full refresh state (NO_CURSOR_STATE_KEY), 3727 which is always within retention, so we use incremental. For other states, it returns 3728 None and we fall back to checking the incremental cursor. 3729 3730 Returns True if the cursor is older than the retention period (should use full refresh). 3731 Returns False if the cursor is within the retention period (safe to use incremental). 3732 """ 3733 retention_duration = parse_duration(api_retention_period) 3734 retention_cutoff = datetime.datetime.now(datetime.timezone.utc) - retention_duration 3735 3736 # Check full refresh cursor first 3737 cursor_datetime = full_refresh_cursor.get_cursor_datetime_from_state(stream_state) 3738 3739 # If full refresh cursor returns None, check incremental cursor 3740 if cursor_datetime is None: 3741 cursor_datetime = incremental_cursor.get_cursor_datetime_from_state(stream_state) 3742 3743 if cursor_datetime is None: 3744 # Neither cursor could parse the state - fall back to full refresh to be safe 3745 return True 3746 3747 if cursor_datetime < retention_cutoff: 3748 logging.warning( 3749 f"Stream '{stream_name}' has a cursor value older than " 3750 f"the API's retention period of {api_retention_period} " 3751 f"(cutoff: {retention_cutoff.isoformat()}). " 3752 f"Falling back to full refresh to avoid data loss." 3753 ) 3754 return True 3755 3756 return False 3757 3758 def _get_state_delegating_stream_model( 3759 self, 3760 model: StateDelegatingStreamModel, 3761 parent_state: Optional[Mapping[str, Any]] = None, 3762 ) -> DeclarativeStreamModel: 3763 """Return the appropriate underlying stream model based on state.""" 3764 return ( 3765 model.incremental_stream 3766 if self._connector_state_manager.get_stream_state(model.name, None) or parent_state 3767 else model.full_refresh_stream 3768 ) 3769 3770 _OPTIONAL_ASYNC_STATUS_FIELDS = {"skipped"} 3771 3772 def _create_async_job_status_mapping( 3773 self, model: AsyncJobStatusMapModel, config: Config, **kwargs: Any 3774 ) -> Mapping[str, AsyncJobStatus]: 3775 api_status_to_cdk_status = {} 3776 for cdk_status, api_statuses in model.dict().items(): 3777 if cdk_status == "type": 3778 # This is an element of the dict because of the typing of the CDK but it is not a CDK status 3779 continue 3780 3781 if api_statuses is None: 3782 if cdk_status in self._OPTIONAL_ASYNC_STATUS_FIELDS: 3783 continue 3784 raise ValueError( 3785 f"Required CDK status '{cdk_status}' has no API statuses mapped. " 3786 f"Please provide at least an empty list for required status fields." 3787 ) 3788 3789 for status in api_statuses: 3790 if status in api_status_to_cdk_status: 3791 raise ValueError( 3792 f"API status {status} is already set for CDK status {cdk_status}. Please ensure API statuses are only provided once" 3793 ) 3794 api_status_to_cdk_status[status] = self._get_async_job_status(cdk_status) 3795 return api_status_to_cdk_status 3796 3797 def _get_async_job_status(self, status: str) -> AsyncJobStatus: 3798 match status: 3799 case "running": 3800 return AsyncJobStatus.RUNNING 3801 case "completed": 3802 return AsyncJobStatus.COMPLETED 3803 case "failed": 3804 return AsyncJobStatus.FAILED 3805 case "timeout": 3806 return AsyncJobStatus.TIMED_OUT 3807 case "skipped": 3808 return AsyncJobStatus.SKIPPED 3809 case _: 3810 raise ValueError(f"Unsupported CDK status {status}") 3811 3812 def create_async_retriever( 3813 self, 3814 model: AsyncRetrieverModel, 3815 config: Config, 3816 *, 3817 name: str, 3818 primary_key: Optional[ 3819 Union[str, List[str], List[List[str]]] 3820 ], # this seems to be needed to match create_simple_retriever 3821 stream_slicer: Optional[StreamSlicer], 3822 client_side_incremental_sync: Optional[Dict[str, Any]] = None, 3823 transformations: List[RecordTransformation], 3824 **kwargs: Any, 3825 ) -> AsyncRetriever: 3826 if model.download_target_requester and not model.download_target_extractor: 3827 raise ValueError( 3828 f"`download_target_extractor` required if using a `download_target_requester`" 3829 ) 3830 3831 def _get_download_retriever( 3832 requester: Requester, extractor: RecordExtractor, _decoder: Decoder 3833 ) -> SimpleRetriever: 3834 # We create a record selector for the download retriever 3835 # with no schema normalization and no transformations, neither record filter 3836 # as all this occurs in the record_selector of the AsyncRetriever 3837 record_selector = RecordSelector( 3838 extractor=extractor, 3839 name=name, 3840 record_filter=None, 3841 transformations=[], 3842 schema_normalization=TypeTransformer(TransformConfig.NoTransform), 3843 config=config, 3844 parameters={}, 3845 ) 3846 paginator = ( 3847 self._create_component_from_model( 3848 model=model.download_paginator, 3849 decoder=_decoder, 3850 config=config, 3851 url_base="", 3852 ) 3853 if model.download_paginator 3854 else NoPagination(parameters={}) 3855 ) 3856 3857 return SimpleRetriever( 3858 requester=requester, 3859 record_selector=record_selector, 3860 primary_key=None, 3861 name=name, 3862 paginator=paginator, 3863 config=config, 3864 parameters={}, 3865 log_formatter=self._get_log_formatter(None, name), 3866 ) 3867 3868 def _get_job_timeout() -> datetime.timedelta: 3869 user_defined_timeout: Optional[int] = ( 3870 int( 3871 InterpolatedString.create( 3872 str(model.polling_job_timeout), 3873 parameters={}, 3874 ).eval(config) 3875 ) 3876 if model.polling_job_timeout 3877 else None 3878 ) 3879 3880 # check for user defined timeout during the test read or 15 minutes 3881 test_read_timeout = datetime.timedelta(minutes=user_defined_timeout or 15) 3882 # default value for non-connector builder is 60 minutes. 3883 default_sync_timeout = datetime.timedelta(minutes=user_defined_timeout or 60) 3884 3885 return ( 3886 test_read_timeout if self._emit_connector_builder_messages else default_sync_timeout 3887 ) 3888 3889 decoder = ( 3890 self._create_component_from_model(model=model.decoder, config=config) 3891 if model.decoder 3892 else JsonDecoder(parameters={}) 3893 ) 3894 record_selector = self._create_component_from_model( 3895 model=model.record_selector, 3896 config=config, 3897 decoder=decoder, 3898 name=name, 3899 transformations=transformations, 3900 client_side_incremental_sync=client_side_incremental_sync, 3901 ) 3902 3903 stream_slicer = stream_slicer or SinglePartitionRouter(parameters={}) 3904 if self._should_limit_slices_fetched(): 3905 stream_slicer = cast( 3906 StreamSlicer, 3907 StreamSlicerTestReadDecorator( 3908 wrapped_slicer=stream_slicer, 3909 maximum_number_of_slices=self._limit_slices_fetched or 5, 3910 ), 3911 ) 3912 3913 creation_requester = self._create_component_from_model( 3914 model=model.creation_requester, 3915 decoder=decoder, 3916 config=config, 3917 name=f"job creation - {name}", 3918 ) 3919 polling_requester = self._create_component_from_model( 3920 model=model.polling_requester, 3921 decoder=decoder, 3922 config=config, 3923 name=f"job polling - {name}", 3924 ) 3925 job_download_components_name = f"job download - {name}" 3926 download_decoder = ( 3927 self._create_component_from_model(model=model.download_decoder, config=config) 3928 if model.download_decoder 3929 else JsonDecoder(parameters={}) 3930 ) 3931 download_extractor = ( 3932 self._create_component_from_model( 3933 model=model.download_extractor, 3934 config=config, 3935 decoder=download_decoder, 3936 parameters=model.parameters, 3937 ) 3938 if model.download_extractor 3939 else DpathExtractor( 3940 [], 3941 config=config, 3942 decoder=download_decoder, 3943 parameters=model.parameters or {}, 3944 ) 3945 ) 3946 download_requester = self._create_component_from_model( 3947 model=model.download_requester, 3948 decoder=download_decoder, 3949 config=config, 3950 name=job_download_components_name, 3951 ) 3952 download_retriever = _get_download_retriever( 3953 download_requester, download_extractor, download_decoder 3954 ) 3955 abort_requester = ( 3956 self._create_component_from_model( 3957 model=model.abort_requester, 3958 decoder=decoder, 3959 config=config, 3960 name=f"job abort - {name}", 3961 ) 3962 if model.abort_requester 3963 else None 3964 ) 3965 delete_requester = ( 3966 self._create_component_from_model( 3967 model=model.delete_requester, 3968 decoder=decoder, 3969 config=config, 3970 name=f"job delete - {name}", 3971 ) 3972 if model.delete_requester 3973 else None 3974 ) 3975 download_target_requester = ( 3976 self._create_component_from_model( 3977 model=model.download_target_requester, 3978 decoder=decoder, 3979 config=config, 3980 name=f"job extract_url - {name}", 3981 ) 3982 if model.download_target_requester 3983 else None 3984 ) 3985 status_extractor = self._create_component_from_model( 3986 model=model.status_extractor, decoder=decoder, config=config, name=name 3987 ) 3988 download_target_extractor = ( 3989 self._create_component_from_model( 3990 model=model.download_target_extractor, 3991 decoder=decoder, 3992 config=config, 3993 name=name, 3994 ) 3995 if model.download_target_extractor 3996 else None 3997 ) 3998 3999 job_repository: AsyncJobRepository = AsyncHttpJobRepository( 4000 creation_requester=creation_requester, 4001 polling_requester=polling_requester, 4002 download_retriever=download_retriever, 4003 download_target_requester=download_target_requester, 4004 abort_requester=abort_requester, 4005 delete_requester=delete_requester, 4006 status_extractor=status_extractor, 4007 status_mapping=self._create_async_job_status_mapping(model.status_mapping, config), 4008 download_target_extractor=download_target_extractor, 4009 job_timeout=_get_job_timeout(), 4010 ) 4011 4012 failed_retry_wait_time_in_seconds: Optional[int] = ( 4013 int( 4014 InterpolatedString.create( 4015 str(model.failed_retry_wait_time_in_seconds), 4016 parameters={}, 4017 ).eval(config) 4018 ) 4019 if model.failed_retry_wait_time_in_seconds 4020 else None 4021 ) 4022 4023 async_job_partition_router = AsyncJobPartitionRouter( 4024 job_orchestrator_factory=lambda stream_slices: AsyncJobOrchestrator( 4025 job_repository, 4026 stream_slices, 4027 self._job_tracker, 4028 self._message_repository, 4029 # FIXME work would need to be done here in order to detect if a stream as a parent stream that is bulk 4030 has_bulk_parent=False, 4031 # set the `job_max_retry` to 1 for the `Connector Builder`` use-case. 4032 # `None` == default retry is set to 3 attempts, under the hood. 4033 job_max_retry=1 if self._emit_connector_builder_messages else None, 4034 failed_retry_wait_time_in_seconds=failed_retry_wait_time_in_seconds, 4035 ), 4036 stream_slicer=stream_slicer, 4037 config=config, 4038 parameters=model.parameters or {}, 4039 ) 4040 4041 return AsyncRetriever( 4042 record_selector=record_selector, 4043 stream_slicer=async_job_partition_router, 4044 config=config, 4045 parameters=model.parameters or {}, 4046 ) 4047 4048 def create_spec(self, model: SpecModel, config: Config, **kwargs: Any) -> Spec: 4049 config_migrations = [ 4050 self._create_component_from_model(migration, config) 4051 for migration in ( 4052 model.config_normalization_rules.config_migrations 4053 if ( 4054 model.config_normalization_rules 4055 and model.config_normalization_rules.config_migrations 4056 ) 4057 else [] 4058 ) 4059 ] 4060 config_transformations = [ 4061 self._create_component_from_model(transformation, config) 4062 for transformation in ( 4063 model.config_normalization_rules.transformations 4064 if ( 4065 model.config_normalization_rules 4066 and model.config_normalization_rules.transformations 4067 ) 4068 else [] 4069 ) 4070 ] 4071 config_validations = [ 4072 self._create_component_from_model(validation, config) 4073 for validation in ( 4074 model.config_normalization_rules.validations 4075 if ( 4076 model.config_normalization_rules 4077 and model.config_normalization_rules.validations 4078 ) 4079 else [] 4080 ) 4081 ] 4082 4083 return Spec( 4084 connection_specification=model.connection_specification, 4085 documentation_url=model.documentation_url, 4086 advanced_auth=model.advanced_auth, 4087 parameters={}, 4088 config_migrations=config_migrations, 4089 config_transformations=config_transformations, 4090 config_validations=config_validations, 4091 ) 4092 4093 def create_substream_partition_router( 4094 self, 4095 model: SubstreamPartitionRouterModel, 4096 config: Config, 4097 *, 4098 stream_name: str, 4099 **kwargs: Any, 4100 ) -> SubstreamPartitionRouter: 4101 parent_stream_configs = [] 4102 if model.parent_stream_configs: 4103 parent_stream_configs.extend( 4104 [ 4105 self.create_parent_stream_config_with_substream_wrapper( 4106 model=parent_stream_config, config=config, stream_name=stream_name, **kwargs 4107 ) 4108 for parent_stream_config in model.parent_stream_configs 4109 ] 4110 ) 4111 4112 return SubstreamPartitionRouter( 4113 parent_stream_configs=parent_stream_configs, 4114 parameters=model.parameters or {}, 4115 config=config, 4116 ) 4117 4118 def create_parent_stream_config_with_substream_wrapper( 4119 self, model: ParentStreamConfigModel, config: Config, *, stream_name: str, **kwargs: Any 4120 ) -> Any: 4121 child_state = self._connector_state_manager.get_stream_state(stream_name, None) 4122 4123 parent_state: Optional[Mapping[str, Any]] = ( 4124 child_state if model.incremental_dependency and child_state else None 4125 ) 4126 connector_state_manager = self._instantiate_parent_stream_state_manager( 4127 child_state, config, model, parent_state 4128 ) 4129 4130 substream_factory = ModelToComponentFactory( 4131 connector_state_manager=connector_state_manager, 4132 limit_pages_fetched_per_slice=self._limit_pages_fetched_per_slice, 4133 limit_slices_fetched=self._limit_slices_fetched, 4134 emit_connector_builder_messages=self._emit_connector_builder_messages, 4135 disable_retries=self._disable_retries, 4136 disable_cache=self._disable_cache, 4137 message_repository=StateFilteringMessageRepository( 4138 LogAppenderMessageRepositoryDecorator( 4139 { 4140 "airbyte_cdk": {"stream": {"is_substream": True}}, 4141 "http": {"is_auxiliary": True}, 4142 }, 4143 self._message_repository, 4144 self._evaluate_log_level(self._emit_connector_builder_messages), 4145 ), 4146 ), 4147 api_budget=self._api_budget, 4148 ) 4149 4150 return substream_factory.create_parent_stream_config( 4151 model=model, config=config, stream_name=stream_name, **kwargs 4152 ) 4153 4154 def _instantiate_parent_stream_state_manager( 4155 self, 4156 child_state: MutableMapping[str, Any], 4157 config: Config, 4158 model: ParentStreamConfigModel, 4159 parent_state: Optional[Mapping[str, Any]] = None, 4160 ) -> ConnectorStateManager: 4161 """ 4162 With DefaultStream, the state needs to be provided during __init__ of the cursor as opposed to the 4163 `set_initial_state` flow that existed for the declarative cursors. This state is taken from 4164 self._connector_state_manager.get_stream_state (`self` being a newly created ModelToComponentFactory to account 4165 for the MessageRepository being different). So we need to pass a ConnectorStateManager to the 4166 ModelToComponentFactory that has the parent states. This method populates this if there is a child state and if 4167 incremental_dependency is set. 4168 """ 4169 if model.incremental_dependency and child_state: 4170 parent_stream_name = model.stream.name or "" 4171 extracted_parent_state = ConcurrentPerPartitionCursor.get_parent_state( 4172 child_state, parent_stream_name 4173 ) 4174 4175 if not extracted_parent_state: 4176 extracted_parent_state = ConcurrentPerPartitionCursor.get_global_state( 4177 child_state, parent_stream_name 4178 ) 4179 4180 if not extracted_parent_state and not isinstance(extracted_parent_state, dict): 4181 cursor_values = child_state.values() 4182 if cursor_values and len(cursor_values) == 1: 4183 incremental_sync_model: Union[ 4184 DatetimeBasedCursorModel, 4185 IncrementingCountCursorModel, 4186 ] = ( 4187 model.stream.incremental_sync # type: ignore # if we are there, it is because there is incremental_dependency and therefore there is an incremental_sync on the parent stream 4188 if isinstance(model.stream, DeclarativeStreamModel) 4189 else self._get_state_delegating_stream_model( 4190 model.stream, parent_state=parent_state 4191 ).incremental_sync 4192 ) 4193 cursor_field = InterpolatedString.create( 4194 incremental_sync_model.cursor_field, 4195 parameters=incremental_sync_model.parameters or {}, 4196 ).eval(config) 4197 extracted_parent_state = AirbyteStateMessage( 4198 type=AirbyteStateType.STREAM, 4199 stream=AirbyteStreamState( 4200 stream_descriptor=StreamDescriptor( 4201 name=parent_stream_name, namespace=None 4202 ), 4203 stream_state=AirbyteStateBlob( 4204 {cursor_field: list(cursor_values)[0]} 4205 ), 4206 ), 4207 ) 4208 return ConnectorStateManager([extracted_parent_state] if extracted_parent_state else []) 4209 4210 return ConnectorStateManager([]) 4211 4212 @staticmethod 4213 def create_wait_time_from_header( 4214 model: WaitTimeFromHeaderModel, config: Config, **kwargs: Any 4215 ) -> WaitTimeFromHeaderBackoffStrategy: 4216 return WaitTimeFromHeaderBackoffStrategy( 4217 header=model.header, 4218 parameters=model.parameters or {}, 4219 config=config, 4220 regex=model.regex, 4221 max_waiting_time_in_seconds=model.max_waiting_time_in_seconds 4222 if model.max_waiting_time_in_seconds is not None 4223 else None, 4224 ) 4225 4226 @staticmethod 4227 def create_wait_until_time_from_header( 4228 model: WaitUntilTimeFromHeaderModel, config: Config, **kwargs: Any 4229 ) -> WaitUntilTimeFromHeaderBackoffStrategy: 4230 return WaitUntilTimeFromHeaderBackoffStrategy( 4231 header=model.header, 4232 parameters=model.parameters or {}, 4233 config=config, 4234 min_wait=model.min_wait, 4235 regex=model.regex, 4236 ) 4237 4238 def get_message_repository(self) -> MessageRepository: 4239 return self._message_repository 4240 4241 def _evaluate_log_level(self, emit_connector_builder_messages: bool) -> Level: 4242 return Level.DEBUG if emit_connector_builder_messages else Level.INFO 4243 4244 @staticmethod 4245 def create_components_mapping_definition( 4246 model: ComponentMappingDefinitionModel, config: Config, **kwargs: Any 4247 ) -> ComponentMappingDefinition: 4248 interpolated_value = InterpolatedString.create( 4249 model.value, parameters=model.parameters or {} 4250 ) 4251 field_path = [ 4252 InterpolatedString.create(path, parameters=model.parameters or {}) 4253 for path in model.field_path 4254 ] 4255 return ComponentMappingDefinition( 4256 field_path=field_path, # type: ignore[arg-type] # field_path can be str and InterpolatedString 4257 value=interpolated_value, 4258 value_type=ModelToComponentFactory._json_schema_type_name_to_type(model.value_type), 4259 create_or_update=model.create_or_update, 4260 condition=model.condition, 4261 parameters=model.parameters or {}, 4262 ) 4263 4264 def create_http_components_resolver( 4265 self, model: HttpComponentsResolverModel, config: Config, stream_name: Optional[str] = None 4266 ) -> Any: 4267 retriever = self._create_component_from_model( 4268 model=model.retriever, 4269 config=config, 4270 name=f"{stream_name if stream_name else '__http_components_resolver'}", 4271 primary_key=None, 4272 stream_slicer=self._build_stream_slicer_from_partition_router(model.retriever, config), 4273 transformations=[], 4274 ) 4275 4276 components_mapping = [] 4277 for component_mapping_definition_model in model.components_mapping: 4278 if component_mapping_definition_model.condition: 4279 raise ValueError("`condition` is only supported for `ConfigComponentsResolver`") 4280 components_mapping.append( 4281 self._create_component_from_model( 4282 model=component_mapping_definition_model, 4283 value_type=ModelToComponentFactory._json_schema_type_name_to_type( 4284 component_mapping_definition_model.value_type 4285 ), 4286 config=config, 4287 ) 4288 ) 4289 4290 return HttpComponentsResolver( 4291 retriever=retriever, 4292 stream_slicer=self._build_stream_slicer_from_partition_router(model.retriever, config), 4293 config=config, 4294 components_mapping=components_mapping, 4295 parameters=model.parameters or {}, 4296 ) 4297 4298 @staticmethod 4299 def create_stream_config( 4300 model: StreamConfigModel, config: Config, **kwargs: Any 4301 ) -> StreamConfig: 4302 model_configs_pointer: List[Union[InterpolatedString, str]] = ( 4303 [x for x in model.configs_pointer] if model.configs_pointer else [] 4304 ) 4305 4306 return StreamConfig( 4307 configs_pointer=model_configs_pointer, 4308 default_values=model.default_values, 4309 parameters=model.parameters or {}, 4310 ) 4311 4312 def create_config_components_resolver( 4313 self, 4314 model: ConfigComponentsResolverModel, 4315 config: Config, 4316 ) -> Any: 4317 model_stream_configs = ( 4318 model.stream_config if isinstance(model.stream_config, list) else [model.stream_config] 4319 ) 4320 4321 stream_configs = [ 4322 self._create_component_from_model( 4323 stream_config, config=config, parameters=model.parameters or {} 4324 ) 4325 for stream_config in model_stream_configs 4326 ] 4327 4328 components_mapping = [ 4329 self._create_component_from_model( 4330 model=components_mapping_definition_model, 4331 value_type=ModelToComponentFactory._json_schema_type_name_to_type( 4332 components_mapping_definition_model.value_type 4333 ), 4334 config=config, 4335 parameters=model.parameters, 4336 ) 4337 for components_mapping_definition_model in model.components_mapping 4338 ] 4339 4340 return ConfigComponentsResolver( 4341 stream_configs=stream_configs, 4342 config=config, 4343 components_mapping=components_mapping, 4344 parameters=model.parameters or {}, 4345 ) 4346 4347 def create_parametrized_components_resolver( 4348 self, 4349 model: ParametrizedComponentsResolverModel, 4350 config: Config, 4351 ) -> ParametrizedComponentsResolver: 4352 stream_parameters = StreamParametersDefinition( 4353 list_of_parameters_for_stream=model.stream_parameters.list_of_parameters_for_stream 4354 ) 4355 4356 components_mapping = [] 4357 for components_mapping_definition_model in model.components_mapping: 4358 if components_mapping_definition_model.condition: 4359 raise ValueError("`condition` is only supported for `ConfigComponentsResolver`") 4360 components_mapping.append( 4361 self._create_component_from_model( 4362 model=components_mapping_definition_model, 4363 value_type=ModelToComponentFactory._json_schema_type_name_to_type( 4364 components_mapping_definition_model.value_type 4365 ), 4366 config=config, 4367 ) 4368 ) 4369 return ParametrizedComponentsResolver( 4370 stream_parameters=stream_parameters, 4371 config=config, 4372 components_mapping=components_mapping, 4373 parameters=model.parameters or {}, 4374 ) 4375 4376 _UNSUPPORTED_DECODER_ERROR = ( 4377 "Specified decoder of {decoder_type} is not supported for pagination." 4378 "Please set as `JsonDecoder`, `XmlDecoder`, or a `CompositeRawDecoder` with an inner_parser of `JsonParser` or `GzipParser` instead." 4379 "If using `GzipParser`, please ensure that the lowest level inner_parser is a `JsonParser`." 4380 ) 4381 4382 def _is_supported_decoder_for_pagination(self, decoder: Decoder) -> bool: 4383 if isinstance(decoder, (JsonDecoder, XmlDecoder)): 4384 return True 4385 elif isinstance(decoder, CompositeRawDecoder): 4386 return self._is_supported_parser_for_pagination(decoder.parser) 4387 else: 4388 return False 4389 4390 def _is_supported_parser_for_pagination(self, parser: Parser) -> bool: 4391 if isinstance(parser, JsonParser): 4392 return True 4393 elif isinstance(parser, GzipParser): 4394 return isinstance(parser.inner_parser, JsonParser) 4395 else: 4396 return False 4397 4398 def create_http_api_budget( 4399 self, model: HTTPAPIBudgetModel, config: Config, **kwargs: Any 4400 ) -> HttpAPIBudget: 4401 policies = [ 4402 self._create_component_from_model(model=policy, config=config) 4403 for policy in model.policies 4404 ] 4405 4406 return HttpAPIBudget( 4407 policies=policies, 4408 ratelimit_reset_header=model.ratelimit_reset_header or "ratelimit-reset", 4409 ratelimit_remaining_header=model.ratelimit_remaining_header or "ratelimit-remaining", 4410 status_codes_for_ratelimit_hit=model.status_codes_for_ratelimit_hit or [429], 4411 ) 4412 4413 def create_fixed_window_call_rate_policy( 4414 self, model: FixedWindowCallRatePolicyModel, config: Config, **kwargs: Any 4415 ) -> FixedWindowCallRatePolicy: 4416 matchers = [ 4417 self._create_component_from_model(model=matcher, config=config) 4418 for matcher in model.matchers 4419 ] 4420 4421 # Set the initial reset timestamp to 10 days from now. 4422 # This value will be updated by the first request. 4423 return FixedWindowCallRatePolicy( 4424 next_reset_ts=datetime.datetime.now() + datetime.timedelta(days=10), 4425 period=parse_duration(model.period), 4426 call_limit=model.call_limit, 4427 matchers=matchers, 4428 ) 4429 4430 def create_file_uploader( 4431 self, model: FileUploaderModel, config: Config, **kwargs: Any 4432 ) -> FileUploader: 4433 name = "File Uploader" 4434 requester = self._create_component_from_model( 4435 model=model.requester, 4436 config=config, 4437 name=name, 4438 **kwargs, 4439 ) 4440 download_target_extractor = self._create_component_from_model( 4441 model=model.download_target_extractor, 4442 config=config, 4443 name=name, 4444 **kwargs, 4445 ) 4446 emit_connector_builder_messages = self._emit_connector_builder_messages 4447 file_uploader = DefaultFileUploader( 4448 requester=requester, 4449 download_target_extractor=download_target_extractor, 4450 config=config, 4451 file_writer=NoopFileWriter() 4452 if emit_connector_builder_messages 4453 else LocalFileSystemFileWriter(), 4454 parameters=model.parameters or {}, 4455 filename_extractor=model.filename_extractor if model.filename_extractor else None, 4456 ) 4457 4458 return ( 4459 ConnectorBuilderFileUploader(file_uploader) 4460 if emit_connector_builder_messages 4461 else file_uploader 4462 ) 4463 4464 def create_moving_window_call_rate_policy( 4465 self, model: MovingWindowCallRatePolicyModel, config: Config, **kwargs: Any 4466 ) -> MovingWindowCallRatePolicy: 4467 rates = [ 4468 self._create_component_from_model(model=rate, config=config) for rate in model.rates 4469 ] 4470 matchers = [ 4471 self._create_component_from_model(model=matcher, config=config) 4472 for matcher in model.matchers 4473 ] 4474 return MovingWindowCallRatePolicy( 4475 rates=rates, 4476 matchers=matchers, 4477 ) 4478 4479 def create_unlimited_call_rate_policy( 4480 self, model: UnlimitedCallRatePolicyModel, config: Config, **kwargs: Any 4481 ) -> UnlimitedCallRatePolicy: 4482 matchers = [ 4483 self._create_component_from_model(model=matcher, config=config) 4484 for matcher in model.matchers 4485 ] 4486 4487 return UnlimitedCallRatePolicy( 4488 matchers=matchers, 4489 ) 4490 4491 def create_rate(self, model: RateModel, config: Config, **kwargs: Any) -> Rate: 4492 interpolated_limit = InterpolatedString.create(str(model.limit), parameters={}) 4493 return Rate( 4494 limit=int(interpolated_limit.eval(config=config)), 4495 interval=parse_duration(model.interval), 4496 ) 4497 4498 def create_http_request_matcher( 4499 self, model: HttpRequestRegexMatcherModel, config: Config, **kwargs: Any 4500 ) -> HttpRequestRegexMatcher: 4501 weight = model.weight 4502 if weight is not None: 4503 if isinstance(weight, str): 4504 weight = int(InterpolatedString.create(weight, parameters={}).eval(config)) 4505 else: 4506 weight = int(weight) 4507 if weight < 1: 4508 raise ValueError(f"weight must be >= 1, got {weight}") 4509 return HttpRequestRegexMatcher( 4510 method=model.method, 4511 url_base=model.url_base, 4512 url_path_pattern=model.url_path_pattern, 4513 params=model.params, 4514 headers=model.headers, 4515 weight=weight, 4516 ) 4517 4518 def set_api_budget(self, component_definition: ComponentDefinition, config: Config) -> None: 4519 self._api_budget = self.create_component( 4520 model_type=HTTPAPIBudgetModel, component_definition=component_definition, config=config 4521 ) 4522 4523 def create_grouping_partition_router( 4524 self, 4525 model: GroupingPartitionRouterModel, 4526 config: Config, 4527 *, 4528 stream_name: str, 4529 **kwargs: Any, 4530 ) -> GroupingPartitionRouter: 4531 underlying_router = self._create_component_from_model( 4532 model=model.underlying_partition_router, 4533 config=config, 4534 stream_name=stream_name, 4535 **kwargs, 4536 ) 4537 if model.group_size < 1: 4538 raise ValueError(f"Group size must be greater than 0, got {model.group_size}") 4539 4540 # Request options in underlying partition routers are not supported for GroupingPartitionRouter 4541 # because they are specific to individual partitions and cannot be aggregated or handled 4542 # when grouping, potentially leading to incorrect API calls. Any request customization 4543 # should be managed at the stream level through the requester's configuration. 4544 if isinstance(underlying_router, SubstreamPartitionRouter): 4545 if any( 4546 parent_config.request_option 4547 for parent_config in underlying_router.parent_stream_configs 4548 ): 4549 raise ValueError("Request options are not supported for GroupingPartitionRouter.") 4550 4551 if isinstance(underlying_router, ListPartitionRouter): 4552 if underlying_router.request_option: 4553 raise ValueError("Request options are not supported for GroupingPartitionRouter.") 4554 4555 return GroupingPartitionRouter( 4556 group_size=model.group_size, 4557 underlying_partition_router=underlying_router, 4558 deduplicate=model.deduplicate if model.deduplicate is not None else True, 4559 config=config, 4560 ) 4561 4562 def _ensure_query_properties_to_model( 4563 self, requester: Union[HttpRequesterModel, CustomRequesterModel] 4564 ) -> None: 4565 """ 4566 For some reason, it seems like CustomRequesterModel request_parameters stays as dictionaries which means that 4567 the other conditions relying on it being QueryPropertiesModel instead of a dict fail. Here, we migrate them to 4568 proper model. 4569 """ 4570 if not hasattr(requester, "request_parameters"): 4571 return 4572 4573 request_parameters = requester.request_parameters 4574 if request_parameters and isinstance(request_parameters, Dict): 4575 for request_parameter_key in request_parameters.keys(): 4576 request_parameter = request_parameters[request_parameter_key] 4577 if ( 4578 isinstance(request_parameter, Dict) 4579 and request_parameter.get("type") == "QueryProperties" 4580 ): 4581 request_parameters[request_parameter_key] = QueryPropertiesModel.parse_obj( 4582 request_parameter 4583 ) 4584 4585 def _get_catalog_defined_cursor_field( 4586 self, stream_name: str, allow_catalog_defined_cursor_field: bool 4587 ) -> Optional[CursorField]: 4588 if not allow_catalog_defined_cursor_field: 4589 return None 4590 4591 configured_stream = self._stream_name_to_configured_stream.get(stream_name) 4592 4593 # Depending on the operation is being performed, there may not be a configured stream yet. In this 4594 # case we return None which will then use the default cursor field defined on the cursor model. 4595 # We also treat cursor_field: [""] (list with empty string) as no cursor field, since this can 4596 # occur when the platform serializes "no cursor configured" streams incorrectly. 4597 if ( 4598 not configured_stream 4599 or not configured_stream.cursor_field 4600 or not configured_stream.cursor_field[0] 4601 ): 4602 return None 4603 elif len(configured_stream.cursor_field) > 1: 4604 raise ValueError( 4605 f"The `{stream_name}` stream does not support nested cursor_field. Please specify only a single cursor_field for the stream in the configured catalog." 4606 ) 4607 else: 4608 return CursorField( 4609 cursor_field_key=configured_stream.cursor_field[0], 4610 supports_catalog_defined_cursor_field=allow_catalog_defined_cursor_field, 4611 )
ComponentDefinition =
typing.Mapping[str, typing.Any]
SCHEMA_TRANSFORMER_TYPE_MAPPING =
{<SchemaNormalization.None_: 'None'>: <TransformConfig.NoTransform: 1>, <SchemaNormalization.Default: 'Default'>: <TransformConfig.DefaultSchemaNormalization: 2>}
MAX_SLICES =
5
LOGGER =
<Logger airbyte.model_to_component_factory (INFO)>
class
ModelToComponentFactory:
689class ModelToComponentFactory: 690 EPOCH_DATETIME_FORMAT = "%s" 691 692 def __init__( 693 self, 694 limit_pages_fetched_per_slice: Optional[int] = None, 695 limit_slices_fetched: Optional[int] = None, 696 emit_connector_builder_messages: bool = False, 697 disable_retries: bool = False, 698 disable_cache: bool = False, 699 message_repository: Optional[MessageRepository] = None, 700 connector_state_manager: Optional[ConnectorStateManager] = None, 701 max_concurrent_async_job_count: Optional[int] = None, 702 configured_catalog: Optional[ConfiguredAirbyteCatalog] = None, 703 api_budget: Optional[APIBudget] = None, 704 ): 705 self._init_mappings() 706 self._limit_pages_fetched_per_slice = limit_pages_fetched_per_slice 707 self._limit_slices_fetched = limit_slices_fetched 708 self._emit_connector_builder_messages = emit_connector_builder_messages 709 self._disable_retries = disable_retries 710 self._disable_cache = disable_cache 711 self._message_repository = message_repository or InMemoryMessageRepository( 712 self._evaluate_log_level(emit_connector_builder_messages) 713 ) 714 self._stream_name_to_configured_stream = self._create_stream_name_to_configured_stream( 715 configured_catalog 716 ) 717 self._connector_state_manager = connector_state_manager or ConnectorStateManager() 718 self._api_budget: Optional[Union[APIBudget]] = api_budget 719 self._job_tracker: JobTracker = JobTracker(max_concurrent_async_job_count or 1) 720 # placeholder for deprecation warnings 721 self._collected_deprecation_logs: List[ConnectorBuilderLogMessage] = [] 722 723 def _init_mappings(self) -> None: 724 self.PYDANTIC_MODEL_TO_CONSTRUCTOR: Mapping[Type[BaseModel], Callable[..., Any]] = { 725 AddedFieldDefinitionModel: self.create_added_field_definition, 726 AddFieldsModel: self.create_add_fields, 727 ApiKeyAuthenticatorModel: self.create_api_key_authenticator, 728 BasicHttpAuthenticatorModel: self.create_basic_http_authenticator, 729 BearerAuthenticatorModel: self.create_bearer_authenticator, 730 CheckStreamModel: self.create_check_stream, 731 DynamicStreamCheckConfigModel: self.create_dynamic_stream_check_config, 732 CheckDynamicStreamModel: self.create_check_dynamic_stream, 733 CompositeErrorHandlerModel: self.create_composite_error_handler, 734 ConcurrencyLevelModel: self.create_concurrency_level, 735 ConfigMigrationModel: self.create_config_migration, 736 ConfigAddFieldsModel: self.create_config_add_fields, 737 ConfigRemapFieldModel: self.create_config_remap_field, 738 ConfigRemoveFieldsModel: self.create_config_remove_fields, 739 ConstantBackoffStrategyModel: self.create_constant_backoff_strategy, 740 CsvDecoderModel: self.create_csv_decoder, 741 CursorPaginationModel: self.create_cursor_pagination, 742 CustomAuthenticatorModel: self.create_custom_component, 743 CustomBackoffStrategyModel: self.create_custom_component, 744 CustomDecoderModel: self.create_custom_component, 745 CustomErrorHandlerModel: self.create_custom_component, 746 CustomRecordExtractorModel: self.create_custom_component, 747 CustomRecordFilterModel: self.create_custom_component, 748 CustomRequesterModel: self.create_custom_component, 749 CustomRetrieverModel: self.create_custom_component, 750 CustomSchemaLoader: self.create_custom_component, 751 CustomSchemaNormalizationModel: self.create_custom_component, 752 CustomStateMigration: self.create_custom_component, 753 CustomPaginationStrategyModel: self.create_custom_component, 754 CustomPartitionRouterModel: self.create_custom_component, 755 CustomTransformationModel: self.create_custom_component, 756 CustomValidationStrategyModel: self.create_custom_component, 757 CustomConfigTransformationModel: self.create_custom_component, 758 DeclarativeStreamModel: self.create_default_stream, 759 DefaultErrorHandlerModel: self.create_default_error_handler, 760 DefaultPaginatorModel: self.create_default_paginator, 761 DpathExtractorModel: self.create_dpath_extractor, 762 DpathValidatorModel: self.create_dpath_validator, 763 ResponseToFileExtractorModel: self.create_response_to_file_extractor, 764 ExponentialBackoffStrategyModel: self.create_exponential_backoff_strategy, 765 SessionTokenAuthenticatorModel: self.create_session_token_authenticator, 766 GroupByKeyMergeStrategyModel: self.create_group_by_key, 767 HttpRequesterModel: self.create_http_requester, 768 HttpResponseFilterModel: self.create_http_response_filter, 769 InlineSchemaLoaderModel: self.create_inline_schema_loader, 770 JsonDecoderModel: self.create_json_decoder, 771 JsonItemsDecoderModel: self.create_json_items_decoder, 772 JsonlDecoderModel: self.create_jsonl_decoder, 773 JsonSchemaPropertySelectorModel: self.create_json_schema_property_selector, 774 GzipDecoderModel: self.create_gzip_decoder, 775 KeysToLowerModel: self.create_keys_to_lower_transformation, 776 KeysToSnakeCaseModel: self.create_keys_to_snake_transformation, 777 KeysReplaceModel: self.create_keys_replace_transformation, 778 FlattenFieldsModel: self.create_flatten_fields, 779 DpathFlattenFieldsModel: self.create_dpath_flatten_fields, 780 IterableDecoderModel: self.create_iterable_decoder, 781 XmlDecoderModel: self.create_xml_decoder, 782 JsonFileSchemaLoaderModel: self.create_json_file_schema_loader, 783 DynamicSchemaLoaderModel: self.create_dynamic_schema_loader, 784 SchemaTypeIdentifierModel: self.create_schema_type_identifier, 785 TypesMapModel: self.create_types_map, 786 ComplexFieldTypeModel: self.create_complex_field_type, 787 JwtAuthenticatorModel: self.create_jwt_authenticator, 788 LegacyToPerPartitionStateMigrationModel: self.create_legacy_to_per_partition_state_migration, 789 ListPartitionRouterModel: self.create_list_partition_router, 790 MinMaxDatetimeModel: self.create_min_max_datetime, 791 NoAuthModel: self.create_no_auth, 792 NoPaginationModel: self.create_no_pagination, 793 OAuthAuthenticatorModel: self.create_oauth_authenticator, 794 OffsetIncrementModel: self.create_offset_increment, 795 PageIncrementModel: self.create_page_increment, 796 ParentStreamConfigModel: self.create_parent_stream_config_with_substream_wrapper, 797 PredicateValidatorModel: self.create_predicate_validator, 798 PropertiesFromEndpointModel: self.create_properties_from_endpoint, 799 PropertyChunkingModel: self.create_property_chunking, 800 QueryPropertiesModel: self.create_query_properties, 801 RecordExpanderModel: self.create_record_expander, 802 RecordFilterModel: self.create_record_filter, 803 RecordSelectorModel: self.create_record_selector, 804 RemoveFieldsModel: self.create_remove_fields, 805 RequestPathModel: self.create_request_path, 806 RequestOptionModel: self.create_request_option, 807 LegacySessionTokenAuthenticatorModel: self.create_legacy_session_token_authenticator, 808 SelectiveAuthenticatorModel: self.create_selective_authenticator, 809 SimpleRetrieverModel: self.create_simple_retriever, 810 StateDelegatingStreamModel: self.create_state_delegating_stream, 811 SpecModel: self.create_spec, 812 SubstreamPartitionRouterModel: self.create_substream_partition_router, 813 ValidateAdheresToSchemaModel: self.create_validate_adheres_to_schema, 814 WaitTimeFromHeaderModel: self.create_wait_time_from_header, 815 WaitUntilTimeFromHeaderModel: self.create_wait_until_time_from_header, 816 AsyncRetrieverModel: self.create_async_retriever, 817 HttpComponentsResolverModel: self.create_http_components_resolver, 818 ConfigComponentsResolverModel: self.create_config_components_resolver, 819 ParametrizedComponentsResolverModel: self.create_parametrized_components_resolver, 820 StreamConfigModel: self.create_stream_config, 821 ComponentMappingDefinitionModel: self.create_components_mapping_definition, 822 ZipfileDecoderModel: self.create_zipfile_decoder, 823 HTTPAPIBudgetModel: self.create_http_api_budget, 824 FileUploaderModel: self.create_file_uploader, 825 FixedWindowCallRatePolicyModel: self.create_fixed_window_call_rate_policy, 826 MovingWindowCallRatePolicyModel: self.create_moving_window_call_rate_policy, 827 UnlimitedCallRatePolicyModel: self.create_unlimited_call_rate_policy, 828 RateModel: self.create_rate, 829 HttpRequestRegexMatcherModel: self.create_http_request_matcher, 830 GroupingPartitionRouterModel: self.create_grouping_partition_router, 831 } 832 833 # Needed for the case where we need to perform a second parse on the fields of a custom component 834 self.TYPE_NAME_TO_MODEL = {cls.__name__: cls for cls in self.PYDANTIC_MODEL_TO_CONSTRUCTOR} 835 836 @staticmethod 837 def _create_stream_name_to_configured_stream( 838 configured_catalog: Optional[ConfiguredAirbyteCatalog], 839 ) -> Mapping[str, ConfiguredAirbyteStream]: 840 return ( 841 {stream.stream.name: stream for stream in configured_catalog.streams} 842 if configured_catalog 843 else {} 844 ) 845 846 def create_component( 847 self, 848 model_type: Type[BaseModel], 849 component_definition: ComponentDefinition, 850 config: Config, 851 **kwargs: Any, 852 ) -> Any: 853 """ 854 Takes a given Pydantic model type and Mapping representing a component definition and creates a declarative component and 855 subcomponents which will be used at runtime. This is done by first parsing the mapping into a Pydantic model and then creating 856 creating declarative components from that model. 857 858 :param model_type: The type of declarative component that is being initialized 859 :param component_definition: The mapping that represents a declarative component 860 :param config: The connector config that is provided by the customer 861 :return: The declarative component to be used at runtime 862 """ 863 864 component_type = component_definition.get("type") 865 if component_definition.get("type") != model_type.__name__: 866 raise ValueError( 867 f"Expected manifest component of type {model_type.__name__}, but received {component_type} instead" 868 ) 869 870 declarative_component_model = model_type.parse_obj(component_definition) 871 872 if not isinstance(declarative_component_model, model_type): 873 raise ValueError( 874 f"Expected {model_type.__name__} component, but received {declarative_component_model.__class__.__name__}" 875 ) 876 877 return self._create_component_from_model( 878 model=declarative_component_model, config=config, **kwargs 879 ) 880 881 def _create_component_from_model(self, model: BaseModel, config: Config, **kwargs: Any) -> Any: 882 if model.__class__ not in self.PYDANTIC_MODEL_TO_CONSTRUCTOR: 883 raise ValueError( 884 f"{model.__class__} with attributes {model} is not a valid component type" 885 ) 886 component_constructor = self.PYDANTIC_MODEL_TO_CONSTRUCTOR.get(model.__class__) 887 if not component_constructor: 888 raise ValueError(f"Could not find constructor for {model.__class__}") 889 890 # collect deprecation warnings for supported models. 891 if isinstance(model, BaseModelWithDeprecations): 892 self._collect_model_deprecations(model) 893 894 return component_constructor(model=model, config=config, **kwargs) 895 896 def get_model_deprecations(self) -> List[ConnectorBuilderLogMessage]: 897 """ 898 Returns the deprecation warnings that were collected during the creation of components. 899 """ 900 return self._collected_deprecation_logs 901 902 def _collect_model_deprecations(self, model: BaseModelWithDeprecations) -> None: 903 """ 904 Collects deprecation logs from the given model and appends any new logs to the internal collection. 905 906 This method checks if the provided model has deprecation logs (identified by the presence of the DEPRECATION_LOGS_TAG attribute and a non-None `_deprecation_logs` property). It iterates through each deprecation log in the model and appends it to the `_collected_deprecation_logs` list if it has not already been collected, ensuring that duplicate logs are avoided. 907 908 Args: 909 model (BaseModelWithDeprecations): The model instance from which to collect deprecation logs. 910 """ 911 if hasattr(model, DEPRECATION_LOGS_TAG) and model._deprecation_logs is not None: 912 for log in model._deprecation_logs: 913 # avoid duplicates for deprecation logs observed. 914 if log not in self._collected_deprecation_logs: 915 self._collected_deprecation_logs.append(log) 916 917 def create_config_migration( 918 self, model: ConfigMigrationModel, config: Config 919 ) -> ConfigMigration: 920 transformations: List[ConfigTransformation] = [ 921 self._create_component_from_model(transformation, config) 922 for transformation in model.transformations 923 ] 924 925 return ConfigMigration( 926 description=model.description, 927 transformations=transformations, 928 ) 929 930 def create_config_add_fields( 931 self, model: ConfigAddFieldsModel, config: Config, **kwargs: Any 932 ) -> ConfigAddFields: 933 fields = [self._create_component_from_model(field, config) for field in model.fields] 934 return ConfigAddFields( 935 fields=fields, 936 condition=model.condition or "", 937 ) 938 939 @staticmethod 940 def create_config_remove_fields( 941 model: ConfigRemoveFieldsModel, config: Config, **kwargs: Any 942 ) -> ConfigRemoveFields: 943 return ConfigRemoveFields( 944 field_pointers=model.field_pointers, 945 condition=model.condition or "", 946 ) 947 948 @staticmethod 949 def create_config_remap_field( 950 model: ConfigRemapFieldModel, config: Config, **kwargs: Any 951 ) -> ConfigRemapField: 952 mapping = cast(Mapping[str, Any], model.map) 953 return ConfigRemapField( 954 map=mapping, 955 field_path=model.field_path, 956 config=config, 957 ) 958 959 def create_dpath_validator(self, model: DpathValidatorModel, config: Config) -> DpathValidator: 960 strategy = self._create_component_from_model(model.validation_strategy, config) 961 962 return DpathValidator( 963 field_path=model.field_path, 964 strategy=strategy, 965 ) 966 967 def create_predicate_validator( 968 self, model: PredicateValidatorModel, config: Config 969 ) -> PredicateValidator: 970 strategy = self._create_component_from_model(model.validation_strategy, config) 971 972 return PredicateValidator( 973 value=model.value, 974 strategy=strategy, 975 ) 976 977 @staticmethod 978 def create_validate_adheres_to_schema( 979 model: ValidateAdheresToSchemaModel, config: Config, **kwargs: Any 980 ) -> ValidateAdheresToSchema: 981 base_schema = cast(Mapping[str, Any], model.base_schema) 982 return ValidateAdheresToSchema( 983 schema=base_schema, 984 ) 985 986 @staticmethod 987 def create_added_field_definition( 988 model: AddedFieldDefinitionModel, config: Config, **kwargs: Any 989 ) -> AddedFieldDefinition: 990 interpolated_value = InterpolatedString.create( 991 model.value, parameters=model.parameters or {} 992 ) 993 return AddedFieldDefinition( 994 path=model.path, 995 value=interpolated_value, 996 value_type=ModelToComponentFactory._json_schema_type_name_to_type(model.value_type), 997 parameters=model.parameters or {}, 998 ) 999 1000 def create_add_fields(self, model: AddFieldsModel, config: Config, **kwargs: Any) -> AddFields: 1001 added_field_definitions = [ 1002 self._create_component_from_model( 1003 model=added_field_definition_model, 1004 value_type=ModelToComponentFactory._json_schema_type_name_to_type( 1005 added_field_definition_model.value_type 1006 ), 1007 config=config, 1008 ) 1009 for added_field_definition_model in model.fields 1010 ] 1011 return AddFields( 1012 fields=added_field_definitions, 1013 condition=model.condition or "", 1014 parameters=model.parameters or {}, 1015 ) 1016 1017 def create_keys_to_lower_transformation( 1018 self, model: KeysToLowerModel, config: Config, **kwargs: Any 1019 ) -> KeysToLowerTransformation: 1020 return KeysToLowerTransformation() 1021 1022 def create_keys_to_snake_transformation( 1023 self, model: KeysToSnakeCaseModel, config: Config, **kwargs: Any 1024 ) -> KeysToSnakeCaseTransformation: 1025 return KeysToSnakeCaseTransformation() 1026 1027 def create_keys_replace_transformation( 1028 self, model: KeysReplaceModel, config: Config, **kwargs: Any 1029 ) -> KeysReplaceTransformation: 1030 return KeysReplaceTransformation( 1031 old=model.old, new=model.new, parameters=model.parameters or {} 1032 ) 1033 1034 def create_flatten_fields( 1035 self, model: FlattenFieldsModel, config: Config, **kwargs: Any 1036 ) -> FlattenFields: 1037 return FlattenFields( 1038 flatten_lists=model.flatten_lists if model.flatten_lists is not None else True 1039 ) 1040 1041 def create_dpath_flatten_fields( 1042 self, model: DpathFlattenFieldsModel, config: Config, **kwargs: Any 1043 ) -> DpathFlattenFields: 1044 model_field_path: List[Union[InterpolatedString, str]] = [x for x in model.field_path] 1045 key_transformation = ( 1046 KeyTransformation( 1047 config=config, 1048 prefix=model.key_transformation.prefix, 1049 suffix=model.key_transformation.suffix, 1050 parameters=model.parameters or {}, 1051 ) 1052 if model.key_transformation is not None 1053 else None 1054 ) 1055 return DpathFlattenFields( 1056 config=config, 1057 field_path=model_field_path, 1058 delete_origin_value=model.delete_origin_value 1059 if model.delete_origin_value is not None 1060 else False, 1061 replace_record=model.replace_record if model.replace_record is not None else False, 1062 key_transformation=key_transformation, 1063 parameters=model.parameters or {}, 1064 ) 1065 1066 @staticmethod 1067 def _json_schema_type_name_to_type(value_type: Optional[ValueType]) -> Optional[Type[Any]]: 1068 if not value_type: 1069 return None 1070 names_to_types = { 1071 ValueType.string: str, 1072 ValueType.number: float, 1073 ValueType.integer: int, 1074 ValueType.boolean: bool, 1075 } 1076 return names_to_types[value_type] 1077 1078 def create_api_key_authenticator( 1079 self, 1080 model: ApiKeyAuthenticatorModel, 1081 config: Config, 1082 token_provider: Optional[TokenProvider] = None, 1083 **kwargs: Any, 1084 ) -> ApiKeyAuthenticator: 1085 if model.inject_into is None and model.header is None: 1086 raise ValueError( 1087 "Expected either inject_into or header to be set for ApiKeyAuthenticator" 1088 ) 1089 1090 if model.inject_into is not None and model.header is not None: 1091 raise ValueError( 1092 "inject_into and header cannot be set both for ApiKeyAuthenticator - remove the deprecated header option" 1093 ) 1094 1095 if token_provider is not None and model.api_token != "": 1096 raise ValueError( 1097 "If token_provider is set, api_token is ignored and has to be set to empty string." 1098 ) 1099 1100 request_option = ( 1101 self._create_component_from_model( 1102 model.inject_into, config, parameters=model.parameters or {} 1103 ) 1104 if model.inject_into 1105 else RequestOption( 1106 inject_into=RequestOptionType.header, 1107 field_name=model.header or "", 1108 parameters=model.parameters or {}, 1109 ) 1110 ) 1111 1112 return ApiKeyAuthenticator( 1113 token_provider=( 1114 token_provider 1115 if token_provider is not None 1116 else InterpolatedStringTokenProvider( 1117 api_token=model.api_token or "", 1118 config=config, 1119 parameters=model.parameters or {}, 1120 ) 1121 ), 1122 request_option=request_option, 1123 config=config, 1124 parameters=model.parameters or {}, 1125 ) 1126 1127 def create_legacy_to_per_partition_state_migration( 1128 self, 1129 model: LegacyToPerPartitionStateMigrationModel, 1130 config: Mapping[str, Any], 1131 declarative_stream: DeclarativeStreamModel, 1132 ) -> LegacyToPerPartitionStateMigration: 1133 retriever = declarative_stream.retriever 1134 if not isinstance(retriever, (SimpleRetrieverModel, AsyncRetrieverModel)): 1135 raise ValueError( 1136 f"LegacyToPerPartitionStateMigrations can only be applied on a DeclarativeStream with a SimpleRetriever or AsyncRetriever. Got {type(retriever)}" 1137 ) 1138 partition_router = retriever.partition_router 1139 if not isinstance( 1140 partition_router, (SubstreamPartitionRouterModel, CustomPartitionRouterModel) 1141 ): 1142 raise ValueError( 1143 f"LegacyToPerPartitionStateMigrations can only be applied on a SimpleRetriever with a Substream partition router. Got {type(partition_router)}" 1144 ) 1145 if not hasattr(partition_router, "parent_stream_configs"): 1146 raise ValueError( 1147 "LegacyToPerPartitionStateMigrations can only be applied with a parent stream configuration." 1148 ) 1149 1150 if not hasattr(declarative_stream, "incremental_sync"): 1151 raise ValueError( 1152 "LegacyToPerPartitionStateMigrations can only be applied with an incremental_sync configuration." 1153 ) 1154 1155 return LegacyToPerPartitionStateMigration( 1156 partition_router, # type: ignore # was already checked above 1157 declarative_stream.incremental_sync, # type: ignore # was already checked. Migration can be applied only to incremental streams. 1158 config, 1159 declarative_stream.parameters, # type: ignore # different type is expected here Mapping[str, Any], got Dict[str, Any] 1160 ) 1161 1162 def create_session_token_authenticator( 1163 self, model: SessionTokenAuthenticatorModel, config: Config, name: str, **kwargs: Any 1164 ) -> Union[ApiKeyAuthenticator, BearerAuthenticator]: 1165 decoder = ( 1166 self._create_component_from_model(model=model.decoder, config=config) 1167 if model.decoder 1168 else JsonDecoder(parameters={}) 1169 ) 1170 login_requester = self._create_component_from_model( 1171 model=model.login_requester, 1172 config=config, 1173 name=f"{name}_login_requester", 1174 decoder=decoder, 1175 ) 1176 token_provider = SessionTokenProvider( 1177 login_requester=login_requester, 1178 session_token_path=model.session_token_path, 1179 expiration_duration=parse_duration(model.expiration_duration) 1180 if model.expiration_duration 1181 else None, 1182 parameters=model.parameters or {}, 1183 message_repository=self._message_repository, 1184 decoder=decoder, 1185 ) 1186 if model.request_authentication.type == "Bearer": 1187 return ModelToComponentFactory.create_bearer_authenticator( 1188 BearerAuthenticatorModel(type="BearerAuthenticator", api_token=""), # type: ignore # $parameters has a default value 1189 config, 1190 token_provider=token_provider, 1191 ) 1192 else: 1193 # Get the api_token template if specified, default to just the session token 1194 api_token_template = ( 1195 getattr(model.request_authentication, "api_token", None) or "{{ session_token }}" 1196 ) 1197 final_token_provider: TokenProvider = InterpolatedSessionTokenProvider( 1198 config=config, 1199 api_token=api_token_template, 1200 session_token_provider=token_provider, 1201 parameters=model.parameters or {}, 1202 ) 1203 return self.create_api_key_authenticator( 1204 ApiKeyAuthenticatorModel( 1205 type="ApiKeyAuthenticator", 1206 api_token="", 1207 inject_into=model.request_authentication.inject_into, 1208 ), # type: ignore # $parameters and headers default to None 1209 config=config, 1210 token_provider=final_token_provider, 1211 ) 1212 1213 @staticmethod 1214 def create_basic_http_authenticator( 1215 model: BasicHttpAuthenticatorModel, config: Config, **kwargs: Any 1216 ) -> BasicHttpAuthenticator: 1217 return BasicHttpAuthenticator( 1218 password=model.password or "", 1219 username=model.username, 1220 config=config, 1221 parameters=model.parameters or {}, 1222 ) 1223 1224 @staticmethod 1225 def create_bearer_authenticator( 1226 model: BearerAuthenticatorModel, 1227 config: Config, 1228 token_provider: Optional[TokenProvider] = None, 1229 **kwargs: Any, 1230 ) -> BearerAuthenticator: 1231 if token_provider is not None and model.api_token != "": 1232 raise ValueError( 1233 "If token_provider is set, api_token is ignored and has to be set to empty string." 1234 ) 1235 return BearerAuthenticator( 1236 token_provider=( 1237 token_provider 1238 if token_provider is not None 1239 else InterpolatedStringTokenProvider( 1240 api_token=model.api_token or "", 1241 config=config, 1242 parameters=model.parameters or {}, 1243 ) 1244 ), 1245 config=config, 1246 parameters=model.parameters or {}, 1247 ) 1248 1249 @staticmethod 1250 def create_dynamic_stream_check_config( 1251 model: DynamicStreamCheckConfigModel, config: Config, **kwargs: Any 1252 ) -> DynamicStreamCheckConfig: 1253 return DynamicStreamCheckConfig( 1254 dynamic_stream_name=model.dynamic_stream_name, 1255 stream_count=model.stream_count, 1256 ) 1257 1258 def create_check_stream( 1259 self, model: CheckStreamModel, config: Config, **kwargs: Any 1260 ) -> CheckStream: 1261 if model.dynamic_streams_check_configs is None and model.stream_names is None: 1262 raise ValueError( 1263 "Expected either stream_names or dynamic_streams_check_configs to be set for CheckStream" 1264 ) 1265 1266 dynamic_streams_check_configs = ( 1267 [ 1268 self._create_component_from_model(model=dynamic_stream_check_config, config=config) 1269 for dynamic_stream_check_config in model.dynamic_streams_check_configs 1270 ] 1271 if model.dynamic_streams_check_configs 1272 else [] 1273 ) 1274 1275 return CheckStream( 1276 stream_names=model.stream_names or [], 1277 dynamic_streams_check_configs=dynamic_streams_check_configs, 1278 parameters={}, 1279 ) 1280 1281 @staticmethod 1282 def create_check_dynamic_stream( 1283 model: CheckDynamicStreamModel, config: Config, **kwargs: Any 1284 ) -> CheckDynamicStream: 1285 assert model.use_check_availability is not None # for mypy 1286 1287 use_check_availability = model.use_check_availability 1288 1289 return CheckDynamicStream( 1290 stream_count=model.stream_count, 1291 use_check_availability=use_check_availability, 1292 parameters={}, 1293 ) 1294 1295 def create_composite_error_handler( 1296 self, model: CompositeErrorHandlerModel, config: Config, **kwargs: Any 1297 ) -> CompositeErrorHandler: 1298 error_handlers = [ 1299 self._create_component_from_model(model=error_handler_model, config=config) 1300 for error_handler_model in model.error_handlers 1301 ] 1302 return CompositeErrorHandler( 1303 error_handlers=error_handlers, parameters=model.parameters or {} 1304 ) 1305 1306 @staticmethod 1307 def create_concurrency_level( 1308 model: ConcurrencyLevelModel, config: Config, **kwargs: Any 1309 ) -> ConcurrencyLevel: 1310 return ConcurrencyLevel( 1311 default_concurrency=model.default_concurrency, 1312 max_concurrency=model.max_concurrency, 1313 config=config, 1314 parameters={}, 1315 ) 1316 1317 @staticmethod 1318 def apply_stream_state_migrations( 1319 stream_state_migrations: List[Any] | None, stream_state: MutableMapping[str, Any] 1320 ) -> MutableMapping[str, Any]: 1321 if stream_state_migrations: 1322 for state_migration in stream_state_migrations: 1323 if state_migration.should_migrate(stream_state): 1324 # The state variable is expected to be mutable but the migrate method returns an immutable mapping. 1325 stream_state = dict(state_migration.migrate(stream_state)) 1326 return stream_state 1327 1328 def create_concurrent_cursor_from_datetime_based_cursor( 1329 self, 1330 model_type: Type[BaseModel], 1331 component_definition: ComponentDefinition, 1332 stream_name: str, 1333 stream_namespace: Optional[str], 1334 stream_state: MutableMapping[str, Any], 1335 config: Config, 1336 message_repository: Optional[MessageRepository] = None, 1337 runtime_lookback_window: Optional[datetime.timedelta] = None, 1338 **kwargs: Any, 1339 ) -> ConcurrentCursor: 1340 component_type = component_definition.get("type") 1341 if component_definition.get("type") != model_type.__name__: 1342 raise ValueError( 1343 f"Expected manifest component of type {model_type.__name__}, but received {component_type} instead" 1344 ) 1345 1346 # FIXME the interfaces of the concurrent cursor are kind of annoying as they take a `ComponentDefinition` instead of the actual model. This was done because the ConcurrentDeclarativeSource didn't have access to the models [here for example](https://github.com/airbytehq/airbyte-python-cdk/blob/f525803b3fec9329e4cc8478996a92bf884bfde9/airbyte_cdk/sources/declarative/concurrent_declarative_source.py#L354C54-L354C91). So now we have two cases: 1347 # * The ComponentDefinition comes from model.__dict__ in which case we have `parameters` 1348 # * The ComponentDefinition comes from the manifest as a dict in which case we have `$parameters` 1349 # We should change those interfaces to use the model once we clean up the code in CDS at which point the parameter propagation should happen as part of the ModelToComponentFactory. 1350 if "$parameters" not in component_definition and "parameters" in component_definition: 1351 component_definition["$parameters"] = component_definition.get("parameters") # type: ignore # This is a dict 1352 datetime_based_cursor_model = model_type.parse_obj(component_definition) 1353 1354 if not isinstance(datetime_based_cursor_model, DatetimeBasedCursorModel): 1355 raise ValueError( 1356 f"Expected {model_type.__name__} component, but received {datetime_based_cursor_model.__class__.__name__}" 1357 ) 1358 1359 model_parameters = datetime_based_cursor_model.parameters or {} 1360 1361 cursor_field = self._get_catalog_defined_cursor_field( 1362 stream_name=stream_name, 1363 allow_catalog_defined_cursor_field=datetime_based_cursor_model.allow_catalog_defined_cursor_field 1364 or False, 1365 ) 1366 1367 if not cursor_field: 1368 interpolated_cursor_field = InterpolatedString.create( 1369 datetime_based_cursor_model.cursor_field, 1370 parameters=model_parameters, 1371 ) 1372 cursor_field = CursorField( 1373 cursor_field_key=interpolated_cursor_field.eval(config=config), 1374 supports_catalog_defined_cursor_field=datetime_based_cursor_model.allow_catalog_defined_cursor_field 1375 or False, 1376 ) 1377 1378 interpolated_partition_field_start = InterpolatedString.create( 1379 datetime_based_cursor_model.partition_field_start or "start_time", 1380 parameters=model_parameters, 1381 ) 1382 interpolated_partition_field_end = InterpolatedString.create( 1383 datetime_based_cursor_model.partition_field_end or "end_time", 1384 parameters=model_parameters, 1385 ) 1386 1387 slice_boundary_fields = ( 1388 interpolated_partition_field_start.eval(config=config), 1389 interpolated_partition_field_end.eval(config=config), 1390 ) 1391 1392 datetime_format = datetime_based_cursor_model.datetime_format 1393 1394 cursor_granularity = ( 1395 parse_duration(datetime_based_cursor_model.cursor_granularity) 1396 if datetime_based_cursor_model.cursor_granularity 1397 else None 1398 ) 1399 1400 lookback_window = None 1401 interpolated_lookback_window = ( 1402 InterpolatedString.create( 1403 datetime_based_cursor_model.lookback_window, 1404 parameters=model_parameters, 1405 ) 1406 if datetime_based_cursor_model.lookback_window 1407 else None 1408 ) 1409 if interpolated_lookback_window: 1410 evaluated_lookback_window = interpolated_lookback_window.eval(config=config) 1411 if evaluated_lookback_window: 1412 lookback_window = parse_duration(evaluated_lookback_window) 1413 1414 connector_state_converter: DateTimeStreamStateConverter 1415 connector_state_converter = CustomFormatConcurrentStreamStateConverter( 1416 datetime_format=datetime_format, 1417 input_datetime_formats=datetime_based_cursor_model.cursor_datetime_formats, 1418 is_sequential_state=True, # ConcurrentPerPartitionCursor only works with sequential state 1419 cursor_granularity=cursor_granularity, 1420 ) 1421 1422 # Adjusts the stream state by applying the runtime lookback window. 1423 # This is used to ensure correct state handling in case of failed partitions. 1424 stream_state_value = stream_state.get(cursor_field.cursor_field_key) 1425 if runtime_lookback_window and stream_state_value: 1426 new_stream_state = ( 1427 connector_state_converter.parse_timestamp(stream_state_value) 1428 - runtime_lookback_window 1429 ) 1430 stream_state[cursor_field.cursor_field_key] = connector_state_converter.output_format( 1431 new_stream_state 1432 ) 1433 1434 start_date_runtime_value: Union[InterpolatedString, str, MinMaxDatetime] 1435 if isinstance(datetime_based_cursor_model.start_datetime, MinMaxDatetimeModel): 1436 start_date_runtime_value = self.create_min_max_datetime( 1437 model=datetime_based_cursor_model.start_datetime, config=config 1438 ) 1439 else: 1440 start_date_runtime_value = datetime_based_cursor_model.start_datetime 1441 1442 end_date_runtime_value: Optional[Union[InterpolatedString, str, MinMaxDatetime]] 1443 if isinstance(datetime_based_cursor_model.end_datetime, MinMaxDatetimeModel): 1444 end_date_runtime_value = self.create_min_max_datetime( 1445 model=datetime_based_cursor_model.end_datetime, config=config 1446 ) 1447 else: 1448 end_date_runtime_value = datetime_based_cursor_model.end_datetime 1449 1450 interpolated_start_date = MinMaxDatetime.create( 1451 interpolated_string_or_min_max_datetime=start_date_runtime_value, 1452 parameters=datetime_based_cursor_model.parameters, 1453 ) 1454 interpolated_end_date = ( 1455 None 1456 if not end_date_runtime_value 1457 else MinMaxDatetime.create( 1458 end_date_runtime_value, datetime_based_cursor_model.parameters 1459 ) 1460 ) 1461 1462 # If datetime format is not specified then start/end datetime should inherit it from the stream slicer 1463 if not interpolated_start_date.datetime_format: 1464 interpolated_start_date.datetime_format = datetime_format 1465 if interpolated_end_date and not interpolated_end_date.datetime_format: 1466 interpolated_end_date.datetime_format = datetime_format 1467 1468 start_date = interpolated_start_date.get_datetime(config=config) 1469 end_date_provider = ( 1470 partial(interpolated_end_date.get_datetime, config) 1471 if interpolated_end_date 1472 else connector_state_converter.get_end_provider() 1473 ) 1474 1475 if ( 1476 datetime_based_cursor_model.step and not datetime_based_cursor_model.cursor_granularity 1477 ) or ( 1478 not datetime_based_cursor_model.step and datetime_based_cursor_model.cursor_granularity 1479 ): 1480 raise ValueError( 1481 f"If step is defined, cursor_granularity should be as well and vice-versa. " 1482 f"Right now, step is `{datetime_based_cursor_model.step}` and cursor_granularity is `{datetime_based_cursor_model.cursor_granularity}`" 1483 ) 1484 1485 # When step is not defined, default to a step size from the starting date to the present moment 1486 step_length = datetime.timedelta.max 1487 interpolated_step = ( 1488 InterpolatedString.create( 1489 datetime_based_cursor_model.step, 1490 parameters=model_parameters, 1491 ) 1492 if datetime_based_cursor_model.step 1493 else None 1494 ) 1495 if interpolated_step: 1496 evaluated_step = interpolated_step.eval(config) 1497 if evaluated_step: 1498 step_length = parse_duration(evaluated_step) 1499 1500 clamping_strategy: ClampingStrategy = NoClamping() 1501 if datetime_based_cursor_model.clamping: 1502 # While it is undesirable to interpolate within the model factory (as opposed to at runtime), 1503 # it is still better than shifting interpolation low-code concept into the ConcurrentCursor runtime 1504 # object which we want to keep agnostic of being low-code 1505 target = InterpolatedString( 1506 string=datetime_based_cursor_model.clamping.target, 1507 parameters=model_parameters, 1508 ) 1509 evaluated_target = target.eval(config=config) 1510 match evaluated_target: 1511 case "DAY": 1512 clamping_strategy = DayClampingStrategy() 1513 end_date_provider = ClampingEndProvider( 1514 DayClampingStrategy(is_ceiling=False), 1515 end_date_provider, # type: ignore # Having issues w/ inspection for GapType and CursorValueType as shown in existing tests. Confirmed functionality is working in practice 1516 granularity=cursor_granularity or datetime.timedelta(seconds=1), 1517 ) 1518 case "WEEK": 1519 if ( 1520 not datetime_based_cursor_model.clamping.target_details 1521 or "weekday" not in datetime_based_cursor_model.clamping.target_details 1522 ): 1523 raise ValueError( 1524 "Given WEEK clamping, weekday needs to be provided as target_details" 1525 ) 1526 weekday = self._assemble_weekday( 1527 datetime_based_cursor_model.clamping.target_details["weekday"] 1528 ) 1529 clamping_strategy = WeekClampingStrategy(weekday) 1530 end_date_provider = ClampingEndProvider( 1531 WeekClampingStrategy(weekday, is_ceiling=False), 1532 end_date_provider, # type: ignore # Having issues w/ inspection for GapType and CursorValueType as shown in existing tests. Confirmed functionality is working in practice 1533 granularity=cursor_granularity or datetime.timedelta(days=1), 1534 ) 1535 case "MONTH": 1536 clamping_strategy = MonthClampingStrategy() 1537 end_date_provider = ClampingEndProvider( 1538 MonthClampingStrategy(is_ceiling=False), 1539 end_date_provider, # type: ignore # Having issues w/ inspection for GapType and CursorValueType as shown in existing tests. Confirmed functionality is working in practice 1540 granularity=cursor_granularity or datetime.timedelta(days=1), 1541 ) 1542 case _: 1543 raise ValueError( 1544 f"Invalid clamping target {evaluated_target}, expected DAY, WEEK, MONTH" 1545 ) 1546 1547 return ConcurrentCursor( 1548 stream_name=stream_name, 1549 stream_namespace=stream_namespace, 1550 stream_state=stream_state, 1551 message_repository=message_repository or self._message_repository, 1552 connector_state_manager=self._connector_state_manager, 1553 connector_state_converter=connector_state_converter, 1554 cursor_field=cursor_field, 1555 slice_boundary_fields=slice_boundary_fields, 1556 start=start_date, # type: ignore # Having issues w/ inspection for GapType and CursorValueType as shown in existing tests. Confirmed functionality is working in practice 1557 end_provider=end_date_provider, # type: ignore # Having issues w/ inspection for GapType and CursorValueType as shown in existing tests. Confirmed functionality is working in practice 1558 lookback_window=lookback_window, 1559 slice_range=step_length, 1560 cursor_granularity=cursor_granularity, 1561 clamping_strategy=clamping_strategy, 1562 ) 1563 1564 def create_concurrent_cursor_from_incrementing_count_cursor( 1565 self, 1566 model_type: Type[BaseModel], 1567 component_definition: ComponentDefinition, 1568 stream_name: str, 1569 stream_namespace: Optional[str], 1570 stream_state: MutableMapping[str, Any], 1571 config: Config, 1572 message_repository: Optional[MessageRepository] = None, 1573 **kwargs: Any, 1574 ) -> ConcurrentCursor: 1575 component_type = component_definition.get("type") 1576 if component_definition.get("type") != model_type.__name__: 1577 raise ValueError( 1578 f"Expected manifest component of type {model_type.__name__}, but received {component_type} instead" 1579 ) 1580 1581 incrementing_count_cursor_model = model_type.parse_obj(component_definition) 1582 1583 if not isinstance(incrementing_count_cursor_model, IncrementingCountCursorModel): 1584 raise ValueError( 1585 f"Expected {model_type.__name__} component, but received {incrementing_count_cursor_model.__class__.__name__}" 1586 ) 1587 1588 start_value: Union[int, str, None] = incrementing_count_cursor_model.start_value 1589 # Pydantic Union type coercion can convert int 0 to string '0' depending on Union order. 1590 # We need to handle both int and str representations of numeric values. 1591 # Evaluate the InterpolatedString and convert to int for the ConcurrentCursor. 1592 if start_value is not None: 1593 interpolated_start_value = InterpolatedString.create( 1594 str(start_value), # Ensure we pass a string to InterpolatedString.create 1595 parameters=incrementing_count_cursor_model.parameters or {}, 1596 ) 1597 evaluated_start_value: int = int(interpolated_start_value.eval(config=config)) 1598 else: 1599 evaluated_start_value = 0 1600 1601 cursor_field = self._get_catalog_defined_cursor_field( 1602 stream_name=stream_name, 1603 allow_catalog_defined_cursor_field=incrementing_count_cursor_model.allow_catalog_defined_cursor_field 1604 or False, 1605 ) 1606 1607 if not cursor_field: 1608 interpolated_cursor_field = InterpolatedString.create( 1609 incrementing_count_cursor_model.cursor_field, 1610 parameters=incrementing_count_cursor_model.parameters or {}, 1611 ) 1612 cursor_field = CursorField( 1613 cursor_field_key=interpolated_cursor_field.eval(config=config), 1614 supports_catalog_defined_cursor_field=incrementing_count_cursor_model.allow_catalog_defined_cursor_field 1615 or False, 1616 ) 1617 1618 connector_state_converter = IncrementingCountStreamStateConverter( 1619 is_sequential_state=True, # ConcurrentPerPartitionCursor only works with sequential state 1620 ) 1621 1622 return ConcurrentCursor( 1623 stream_name=stream_name, 1624 stream_namespace=stream_namespace, 1625 stream_state=stream_state, 1626 message_repository=message_repository or self._message_repository, 1627 connector_state_manager=self._connector_state_manager, 1628 connector_state_converter=connector_state_converter, 1629 cursor_field=cursor_field, 1630 slice_boundary_fields=None, 1631 start=evaluated_start_value, # type: ignore # Having issues w/ inspection for GapType and CursorValueType as shown in existing tests. Confirmed functionality is working in practice 1632 end_provider=connector_state_converter.get_end_provider(), # type: ignore # Having issues w/ inspection for GapType and CursorValueType as shown in existing tests. Confirmed functionality is working in practice 1633 ) 1634 1635 def _assemble_weekday(self, weekday: str) -> Weekday: 1636 match weekday: 1637 case "MONDAY": 1638 return Weekday.MONDAY 1639 case "TUESDAY": 1640 return Weekday.TUESDAY 1641 case "WEDNESDAY": 1642 return Weekday.WEDNESDAY 1643 case "THURSDAY": 1644 return Weekday.THURSDAY 1645 case "FRIDAY": 1646 return Weekday.FRIDAY 1647 case "SATURDAY": 1648 return Weekday.SATURDAY 1649 case "SUNDAY": 1650 return Weekday.SUNDAY 1651 case _: 1652 raise ValueError(f"Unknown weekday {weekday}") 1653 1654 def create_concurrent_cursor_from_perpartition_cursor( 1655 self, 1656 state_manager: ConnectorStateManager, 1657 model_type: Type[BaseModel], 1658 component_definition: ComponentDefinition, 1659 stream_name: str, 1660 stream_namespace: Optional[str], 1661 config: Config, 1662 stream_state: MutableMapping[str, Any], 1663 partition_router: PartitionRouter, 1664 attempt_to_create_cursor_if_not_provided: bool = False, 1665 **kwargs: Any, 1666 ) -> ConcurrentPerPartitionCursor: 1667 component_type = component_definition.get("type") 1668 if component_definition.get("type") != model_type.__name__: 1669 raise ValueError( 1670 f"Expected manifest component of type {model_type.__name__}, but received {component_type} instead" 1671 ) 1672 1673 # FIXME the interfaces of the concurrent cursor are kind of annoying as they take a `ComponentDefinition` instead of the actual model. This was done because the ConcurrentDeclarativeSource didn't have access to the models [here for example](https://github.com/airbytehq/airbyte-python-cdk/blob/f525803b3fec9329e4cc8478996a92bf884bfde9/airbyte_cdk/sources/declarative/concurrent_declarative_source.py#L354C54-L354C91). So now we have two cases: 1674 # * The ComponentDefinition comes from model.__dict__ in which case we have `parameters` 1675 # * The ComponentDefinition comes from the manifest as a dict in which case we have `$parameters` 1676 # We should change those interfaces to use the model once we clean up the code in CDS at which point the parameter propagation should happen as part of the ModelToComponentFactory. 1677 if "$parameters" not in component_definition and "parameters" in component_definition: 1678 component_definition["$parameters"] = component_definition.get("parameters") # type: ignore # This is a dict 1679 datetime_based_cursor_model = model_type.parse_obj(component_definition) 1680 1681 if not isinstance(datetime_based_cursor_model, DatetimeBasedCursorModel): 1682 raise ValueError( 1683 f"Expected {model_type.__name__} component, but received {datetime_based_cursor_model.__class__.__name__}" 1684 ) 1685 1686 cursor_field = self._get_catalog_defined_cursor_field( 1687 stream_name=stream_name, 1688 allow_catalog_defined_cursor_field=datetime_based_cursor_model.allow_catalog_defined_cursor_field 1689 or False, 1690 ) 1691 1692 if not cursor_field: 1693 interpolated_cursor_field = InterpolatedString.create( 1694 datetime_based_cursor_model.cursor_field, 1695 # FIXME the interfaces of the concurrent cursor are kind of annoying as they take a `ComponentDefinition` instead of the actual model. This was done because the ConcurrentDeclarativeSource didn't have access to the models [here for example](https://github.com/airbytehq/airbyte-python-cdk/blob/f525803b3fec9329e4cc8478996a92bf884bfde9/airbyte_cdk/sources/declarative/concurrent_declarative_source.py#L354C54-L354C91). So now we have two cases: 1696 # * The ComponentDefinition comes from model.__dict__ in which case we have `parameters` 1697 # * The ComponentDefinition comes from the manifest as a dict in which case we have `$parameters` 1698 # We should change those interfaces to use the model once we clean up the code in CDS at which point the parameter propagation should happen as part of the ModelToComponentFactory. 1699 parameters=datetime_based_cursor_model.parameters or {}, 1700 ) 1701 cursor_field = CursorField( 1702 cursor_field_key=interpolated_cursor_field.eval(config=config), 1703 supports_catalog_defined_cursor_field=datetime_based_cursor_model.allow_catalog_defined_cursor_field 1704 or False, 1705 ) 1706 1707 datetime_format = datetime_based_cursor_model.datetime_format 1708 1709 cursor_granularity = ( 1710 parse_duration(datetime_based_cursor_model.cursor_granularity) 1711 if datetime_based_cursor_model.cursor_granularity 1712 else None 1713 ) 1714 1715 connector_state_converter: DateTimeStreamStateConverter 1716 connector_state_converter = CustomFormatConcurrentStreamStateConverter( 1717 datetime_format=datetime_format, 1718 input_datetime_formats=datetime_based_cursor_model.cursor_datetime_formats, 1719 is_sequential_state=True, # ConcurrentPerPartitionCursor only works with sequential state 1720 cursor_granularity=cursor_granularity, 1721 ) 1722 1723 # Create the cursor factory 1724 cursor_factory = ConcurrentCursorFactory( 1725 partial( 1726 self.create_concurrent_cursor_from_datetime_based_cursor, 1727 state_manager=state_manager, 1728 model_type=model_type, 1729 component_definition=component_definition, 1730 stream_name=stream_name, 1731 stream_namespace=stream_namespace, 1732 config=config, 1733 message_repository=NoopMessageRepository(), 1734 ) 1735 ) 1736 1737 # Per-partition state doesn't make sense for GroupingPartitionRouter, so force the global state 1738 use_global_cursor = isinstance( 1739 partition_router, GroupingPartitionRouter 1740 ) or component_definition.get("global_substream_cursor", False) 1741 1742 # Return the concurrent cursor and state converter 1743 return ConcurrentPerPartitionCursor( 1744 cursor_factory=cursor_factory, 1745 partition_router=partition_router, 1746 stream_name=stream_name, 1747 stream_namespace=stream_namespace, 1748 stream_state=stream_state, 1749 message_repository=self._message_repository, # type: ignore 1750 connector_state_manager=state_manager, 1751 connector_state_converter=connector_state_converter, 1752 cursor_field=cursor_field, 1753 use_global_cursor=use_global_cursor, 1754 attempt_to_create_cursor_if_not_provided=attempt_to_create_cursor_if_not_provided, 1755 ) 1756 1757 @staticmethod 1758 def create_constant_backoff_strategy( 1759 model: ConstantBackoffStrategyModel, config: Config, **kwargs: Any 1760 ) -> ConstantBackoffStrategy: 1761 ModelToComponentFactory._validate_jitter_range(model.jitter_range_in_seconds) 1762 return ConstantBackoffStrategy( 1763 backoff_time_in_seconds=model.backoff_time_in_seconds, 1764 jitter_range_in_seconds=model.jitter_range_in_seconds, 1765 config=config, 1766 parameters=model.parameters or {}, 1767 ) 1768 1769 @staticmethod 1770 def _validate_jitter_range(jitter_range_in_seconds: Optional[float]) -> None: 1771 if jitter_range_in_seconds is not None and jitter_range_in_seconds < 0: 1772 raise ValueError("jitter_range_in_seconds must be greater than or equal to 0") 1773 1774 def create_cursor_pagination( 1775 self, model: CursorPaginationModel, config: Config, decoder: Decoder, **kwargs: Any 1776 ) -> CursorPaginationStrategy: 1777 if isinstance(decoder, PaginationDecoderDecorator): 1778 inner_decoder = decoder.decoder 1779 else: 1780 inner_decoder = decoder 1781 decoder = PaginationDecoderDecorator(decoder=decoder) 1782 1783 if self._is_supported_decoder_for_pagination(inner_decoder): 1784 decoder_to_use = decoder 1785 else: 1786 raise ValueError( 1787 self._UNSUPPORTED_DECODER_ERROR.format(decoder_type=type(inner_decoder)) 1788 ) 1789 1790 # Pydantic v1 Union type coercion can convert int to string depending on Union order. 1791 # If page_size is a string that represents an integer (not an interpolation), convert it back. 1792 page_size = model.page_size 1793 if isinstance(page_size, str) and page_size.isdigit(): 1794 page_size = int(page_size) 1795 1796 return CursorPaginationStrategy( 1797 cursor_value=model.cursor_value, 1798 decoder=decoder_to_use, 1799 page_size=page_size, 1800 stop_condition=model.stop_condition, 1801 config=config, 1802 parameters=model.parameters or {}, 1803 ) 1804 1805 def create_custom_component(self, model: Any, config: Config, **kwargs: Any) -> Any: 1806 """ 1807 Generically creates a custom component based on the model type and a class_name reference to the custom Python class being 1808 instantiated. Only the model's additional properties that match the custom class definition are passed to the constructor 1809 :param model: The Pydantic model of the custom component being created 1810 :param config: The custom defined connector config 1811 :return: The declarative component built from the Pydantic model to be used at runtime 1812 """ 1813 custom_component_class = self._get_class_from_fully_qualified_class_name(model.class_name) 1814 component_fields = get_type_hints(custom_component_class) 1815 model_args = model.dict() 1816 model_args["config"] = config 1817 1818 # There are cases where a parent component will pass arguments to a child component via kwargs. When there are field collisions 1819 # we defer to these arguments over the component's definition 1820 for key, arg in kwargs.items(): 1821 model_args[key] = arg 1822 1823 # Pydantic is unable to parse a custom component's fields that are subcomponents into models because their fields and types are not 1824 # defined in the schema. The fields and types are defined within the Python class implementation. Pydantic can only parse down to 1825 # the custom component and this code performs a second parse to convert the sub-fields first into models, then declarative components 1826 for model_field, model_value in model_args.items(): 1827 # If a custom component field doesn't have a type set, we try to use the type hints to infer the type 1828 if ( 1829 isinstance(model_value, dict) 1830 and "type" not in model_value 1831 and model_field in component_fields 1832 ): 1833 derived_type = self._derive_component_type_from_type_hints( 1834 component_fields.get(model_field) 1835 ) 1836 if derived_type: 1837 model_value["type"] = derived_type 1838 1839 if self._is_component(model_value): 1840 model_args[model_field] = self._create_nested_component( 1841 model, 1842 model_field, 1843 model_value, 1844 config, 1845 **kwargs, 1846 ) 1847 elif isinstance(model_value, list): 1848 vals = [] 1849 for v in model_value: 1850 if isinstance(v, dict) and "type" not in v and model_field in component_fields: 1851 derived_type = self._derive_component_type_from_type_hints( 1852 component_fields.get(model_field) 1853 ) 1854 if derived_type: 1855 v["type"] = derived_type 1856 if self._is_component(v): 1857 vals.append( 1858 self._create_nested_component( 1859 model, 1860 model_field, 1861 v, 1862 config, 1863 **kwargs, 1864 ) 1865 ) 1866 else: 1867 vals.append(v) 1868 model_args[model_field] = vals 1869 1870 kwargs = { 1871 class_field: model_args[class_field] 1872 for class_field in component_fields.keys() 1873 if class_field in model_args 1874 } 1875 1876 if "api_budget" in component_fields and kwargs.get("api_budget") is None: 1877 kwargs["api_budget"] = self._api_budget 1878 1879 return custom_component_class(**kwargs) 1880 1881 @staticmethod 1882 def _get_class_from_fully_qualified_class_name( 1883 full_qualified_class_name: str, 1884 ) -> Any: 1885 """Get a class from its fully qualified name. 1886 1887 If a custom components module is needed, we assume it is already registered - probably 1888 as `source_declarative_manifest.components` or `components`. 1889 1890 Args: 1891 full_qualified_class_name (str): The fully qualified name of the class (e.g., "module.ClassName"). 1892 1893 Returns: 1894 Any: The class object. 1895 1896 Raises: 1897 ValueError: If the class cannot be loaded. 1898 """ 1899 split = full_qualified_class_name.split(".") 1900 module_name_full = ".".join(split[:-1]) 1901 class_name = split[-1] 1902 1903 try: 1904 module_ref = importlib.import_module(module_name_full) 1905 except ModuleNotFoundError as e: 1906 if split[0] == "source_declarative_manifest": 1907 # During testing, the modules containing the custom components are not moved to source_declarative_manifest. In order to run the test, add the source folder to your PYTHONPATH or add it runtime using sys.path.append 1908 try: 1909 import os 1910 1911 module_name_with_source_declarative_manifest = ".".join(split[1:-1]) 1912 module_ref = importlib.import_module( 1913 module_name_with_source_declarative_manifest 1914 ) 1915 except ModuleNotFoundError: 1916 raise ValueError(f"Could not load module `{module_name_full}`.") from e 1917 else: 1918 raise ValueError(f"Could not load module `{module_name_full}`.") from e 1919 1920 try: 1921 return getattr(module_ref, class_name) 1922 except AttributeError as e: 1923 raise ValueError( 1924 f"Could not load class `{class_name}` from module `{module_name_full}`.", 1925 ) from e 1926 1927 @staticmethod 1928 def _derive_component_type_from_type_hints(field_type: Any) -> Optional[str]: 1929 interface = field_type 1930 while True: 1931 origin = get_origin(interface) 1932 if origin: 1933 # Unnest types until we reach the raw type 1934 # List[T] -> T 1935 # Optional[List[T]] -> T 1936 args = get_args(interface) 1937 interface = args[0] 1938 else: 1939 break 1940 if isinstance(interface, type) and not ModelToComponentFactory.is_builtin_type(interface): 1941 return interface.__name__ 1942 return None 1943 1944 @staticmethod 1945 def is_builtin_type(cls: Optional[Type[Any]]) -> bool: 1946 if not cls: 1947 return False 1948 return cls.__module__ == "builtins" 1949 1950 @staticmethod 1951 def _extract_missing_parameters(error: TypeError) -> List[str]: 1952 parameter_search = re.search(r"keyword-only.*:\s(.*)", str(error)) 1953 if parameter_search: 1954 return re.findall(r"\'(.+?)\'", parameter_search.group(1)) 1955 else: 1956 return [] 1957 1958 def _create_nested_component( 1959 self, model: Any, model_field: str, model_value: Any, config: Config, **kwargs: Any 1960 ) -> Any: 1961 type_name = model_value.get("type", None) 1962 if not type_name: 1963 # If no type is specified, we can assume this is a dictionary object which can be returned instead of a subcomponent 1964 return model_value 1965 1966 model_type = self.TYPE_NAME_TO_MODEL.get(type_name, None) 1967 if model_type: 1968 parsed_model = model_type.parse_obj(model_value) 1969 try: 1970 # To improve usability of the language, certain fields are shared between components. This can come in the form of 1971 # a parent component passing some of its fields to a child component or the parent extracting fields from other child 1972 # components and passing it to others. One example is the DefaultPaginator referencing the HttpRequester url_base 1973 # while constructing a SimpleRetriever. However, custom components don't support this behavior because they are created 1974 # generically in create_custom_component(). This block allows developers to specify extra arguments in $parameters that 1975 # are needed by a component and could not be shared. 1976 model_constructor = self.PYDANTIC_MODEL_TO_CONSTRUCTOR.get(parsed_model.__class__) 1977 constructor_kwargs = inspect.getfullargspec(model_constructor).kwonlyargs 1978 model_parameters = model_value.get("$parameters", {}) 1979 matching_parameters = { 1980 kwarg: model_parameters[kwarg] 1981 for kwarg in constructor_kwargs 1982 if kwarg in model_parameters 1983 } 1984 matching_kwargs = { 1985 kwarg: kwargs[kwarg] for kwarg in constructor_kwargs if kwarg in kwargs 1986 } 1987 return self._create_component_from_model( 1988 model=parsed_model, config=config, **(matching_parameters | matching_kwargs) 1989 ) 1990 except TypeError as error: 1991 missing_parameters = self._extract_missing_parameters(error) 1992 if missing_parameters: 1993 raise ValueError( 1994 f"Error creating component '{type_name}' with parent custom component {model.class_name}: Please provide " 1995 + ", ".join( 1996 ( 1997 f"{type_name}.$parameters.{parameter}" 1998 for parameter in missing_parameters 1999 ) 2000 ) 2001 ) 2002 raise TypeError( 2003 f"Error creating component '{type_name}' with parent custom component {model.class_name}: {error}" 2004 ) 2005 else: 2006 raise ValueError( 2007 f"Error creating custom component {model.class_name}. Subcomponent creation has not been implemented for '{type_name}'" 2008 ) 2009 2010 @staticmethod 2011 def _is_component(model_value: Any) -> bool: 2012 return isinstance(model_value, dict) and model_value.get("type") is not None 2013 2014 def create_default_stream( 2015 self, model: DeclarativeStreamModel, config: Config, is_parent: bool = False, **kwargs: Any 2016 ) -> AbstractStream: 2017 primary_key = model.primary_key.__root__ if model.primary_key else None 2018 self._migrate_state(model, config) 2019 2020 partition_router = self._build_stream_slicer_from_partition_router( 2021 model.retriever, 2022 config, 2023 stream_name=model.name, 2024 **kwargs, 2025 ) 2026 concurrent_cursor = self._build_concurrent_cursor(model, partition_router, config) 2027 if model.incremental_sync and isinstance(model.incremental_sync, DatetimeBasedCursorModel): 2028 cursor_model: DatetimeBasedCursorModel = model.incremental_sync 2029 2030 end_time_option = ( 2031 self._create_component_from_model( 2032 cursor_model.end_time_option, config, parameters=cursor_model.parameters or {} 2033 ) 2034 if cursor_model.end_time_option 2035 else None 2036 ) 2037 start_time_option = ( 2038 self._create_component_from_model( 2039 cursor_model.start_time_option, config, parameters=cursor_model.parameters or {} 2040 ) 2041 if cursor_model.start_time_option 2042 else None 2043 ) 2044 2045 datetime_request_options_provider = DatetimeBasedRequestOptionsProvider( 2046 start_time_option=start_time_option, 2047 end_time_option=end_time_option, 2048 partition_field_start=cursor_model.partition_field_start, 2049 partition_field_end=cursor_model.partition_field_end, 2050 config=config, 2051 parameters=model.parameters or {}, 2052 ) 2053 request_options_provider = ( 2054 datetime_request_options_provider 2055 if not isinstance(concurrent_cursor, ConcurrentPerPartitionCursor) 2056 else PerPartitionRequestOptionsProvider( 2057 partition_router, datetime_request_options_provider 2058 ) 2059 ) 2060 elif model.incremental_sync and isinstance( 2061 model.incremental_sync, IncrementingCountCursorModel 2062 ): 2063 if isinstance(concurrent_cursor, ConcurrentPerPartitionCursor): 2064 raise ValueError( 2065 "PerPartition does not support per partition states because switching to global state is time based" 2066 ) 2067 2068 cursor_model: IncrementingCountCursorModel = model.incremental_sync # type: ignore 2069 2070 start_time_option = ( 2071 self._create_component_from_model( 2072 cursor_model.start_value_option, # type: ignore # mypy still thinks cursor_model of type DatetimeBasedCursor 2073 config, 2074 parameters=cursor_model.parameters or {}, 2075 ) 2076 if cursor_model.start_value_option # type: ignore # mypy still thinks cursor_model of type DatetimeBasedCursor 2077 else None 2078 ) 2079 2080 # The concurrent engine defaults the start/end fields on the slice to "start" and "end", but 2081 # the default DatetimeBasedRequestOptionsProvider() sets them to start_time/end_time 2082 partition_field_start = "start" 2083 2084 request_options_provider = DatetimeBasedRequestOptionsProvider( 2085 start_time_option=start_time_option, 2086 partition_field_start=partition_field_start, 2087 config=config, 2088 parameters=model.parameters or {}, 2089 ) 2090 else: 2091 request_options_provider = None 2092 2093 transformations = [] 2094 if model.transformations: 2095 for transformation_model in model.transformations: 2096 transformations.append( 2097 self._create_component_from_model(model=transformation_model, config=config) 2098 ) 2099 file_uploader = None 2100 if model.file_uploader: 2101 file_uploader = self._create_component_from_model( 2102 model=model.file_uploader, config=config 2103 ) 2104 2105 stream_slicer: ConcurrentStreamSlicer = ( 2106 partition_router 2107 if isinstance(concurrent_cursor, FinalStateCursor) 2108 else concurrent_cursor 2109 ) 2110 2111 retriever = self._create_component_from_model( 2112 model=model.retriever, 2113 config=config, 2114 name=model.name, 2115 primary_key=primary_key, 2116 request_options_provider=request_options_provider, 2117 stream_slicer=stream_slicer, 2118 partition_router=partition_router, 2119 has_stop_condition_cursor=self._is_stop_condition_on_cursor(model), 2120 is_client_side_incremental_sync=self._is_client_side_filtering_enabled(model), 2121 cursor=concurrent_cursor, 2122 transformations=transformations, 2123 file_uploader=file_uploader, 2124 incremental_sync=model.incremental_sync, 2125 ) 2126 if isinstance(retriever, AsyncRetriever): 2127 stream_slicer = retriever.stream_slicer 2128 2129 schema_loader: SchemaLoader 2130 if model.schema_loader and isinstance(model.schema_loader, list): 2131 nested_schema_loaders = [ 2132 self._create_component_from_model(model=nested_schema_loader, config=config) 2133 for nested_schema_loader in model.schema_loader 2134 ] 2135 schema_loader = CompositeSchemaLoader( 2136 schema_loaders=nested_schema_loaders, parameters={} 2137 ) 2138 elif model.schema_loader: 2139 schema_loader = self._create_component_from_model( 2140 model=model.schema_loader, # type: ignore # If defined, schema_loader is guaranteed not to be a list and will be one of the existing base models 2141 config=config, 2142 ) 2143 else: 2144 options = model.parameters or {} 2145 if "name" not in options: 2146 options["name"] = model.name 2147 schema_loader = DefaultSchemaLoader(config=config, parameters=options) 2148 schema_loader = CachingSchemaLoaderDecorator(schema_loader) 2149 2150 stream_name = model.name or "" 2151 return DefaultStream( 2152 partition_generator=StreamSlicerPartitionGenerator( 2153 DeclarativePartitionFactory( 2154 stream_name, 2155 schema_loader, 2156 retriever, 2157 self._message_repository, 2158 ), 2159 stream_slicer, 2160 slice_limit=self._limit_slices_fetched, 2161 ), 2162 name=stream_name, 2163 json_schema=schema_loader.get_json_schema, 2164 primary_key=get_primary_key_from_stream(primary_key), 2165 cursor_field=( 2166 concurrent_cursor.cursor_field 2167 if hasattr(concurrent_cursor, "cursor_field") 2168 else None 2169 ), 2170 logger=logging.getLogger(f"airbyte.{stream_name}"), 2171 cursor=concurrent_cursor, 2172 supports_file_transfer=hasattr(model, "file_uploader") and bool(model.file_uploader), 2173 ) 2174 2175 def _migrate_state(self, model: DeclarativeStreamModel, config: Config) -> None: 2176 stream_name = model.name or "" 2177 stream_state = self._connector_state_manager.get_stream_state( 2178 stream_name=stream_name, namespace=None 2179 ) 2180 if model.state_migrations: 2181 state_transformations = [ 2182 self._create_component_from_model(state_migration, config, declarative_stream=model) 2183 for state_migration in model.state_migrations 2184 ] 2185 else: 2186 state_transformations = [] 2187 stream_state = self.apply_stream_state_migrations(state_transformations, stream_state) 2188 self._connector_state_manager.update_state_for_stream( 2189 stream_name=stream_name, namespace=None, value=stream_state 2190 ) 2191 2192 def _is_stop_condition_on_cursor(self, model: DeclarativeStreamModel) -> bool: 2193 return bool( 2194 model.incremental_sync 2195 and hasattr(model.incremental_sync, "is_data_feed") 2196 and model.incremental_sync.is_data_feed 2197 ) 2198 2199 def _is_client_side_filtering_enabled(self, model: DeclarativeStreamModel) -> bool: 2200 return bool( 2201 model.incremental_sync 2202 and hasattr(model.incremental_sync, "is_client_side_incremental") 2203 and model.incremental_sync.is_client_side_incremental 2204 ) 2205 2206 def _build_stream_slicer_from_partition_router( 2207 self, 2208 model: Union[ 2209 AsyncRetrieverModel, 2210 CustomRetrieverModel, 2211 SimpleRetrieverModel, 2212 ], 2213 config: Config, 2214 stream_name: Optional[str] = None, 2215 **kwargs: Any, 2216 ) -> PartitionRouter: 2217 if ( 2218 hasattr(model, "partition_router") 2219 and isinstance(model, (SimpleRetrieverModel, AsyncRetrieverModel, CustomRetrieverModel)) 2220 and model.partition_router 2221 ): 2222 stream_slicer_model = model.partition_router 2223 if isinstance(stream_slicer_model, list): 2224 return CartesianProductStreamSlicer( 2225 [ 2226 self._create_component_from_model( 2227 model=slicer, config=config, stream_name=stream_name or "" 2228 ) 2229 for slicer in stream_slicer_model 2230 ], 2231 parameters={}, 2232 ) 2233 elif isinstance(stream_slicer_model, dict): 2234 # partition router comes from CustomRetrieverModel therefore has not been parsed as a model 2235 params = stream_slicer_model.get("$parameters") 2236 if not isinstance(params, dict): 2237 params = {} 2238 stream_slicer_model["$parameters"] = params 2239 2240 if stream_name is not None: 2241 params["stream_name"] = stream_name 2242 2243 return self._create_nested_component( # type: ignore[no-any-return] # There is no guarantee that this will return a stream slicer. If not, we expect an AttributeError during the call to `stream_slices` 2244 model, 2245 "partition_router", 2246 stream_slicer_model, 2247 config, 2248 **kwargs, 2249 ) 2250 else: 2251 return self._create_component_from_model( # type: ignore[no-any-return] # Will be created PartitionRouter as stream_slicer_model is model.partition_router 2252 model=stream_slicer_model, config=config, stream_name=stream_name or "" 2253 ) 2254 return SinglePartitionRouter(parameters={}) 2255 2256 def _build_concurrent_cursor( 2257 self, 2258 model: DeclarativeStreamModel, 2259 stream_slicer: Optional[PartitionRouter], 2260 config: Config, 2261 ) -> Cursor: 2262 stream_name = model.name or "" 2263 stream_state = self._connector_state_manager.get_stream_state(stream_name, None) 2264 2265 if ( 2266 model.incremental_sync 2267 and stream_slicer 2268 and not isinstance(stream_slicer, SinglePartitionRouter) 2269 ): 2270 if isinstance(model.incremental_sync, IncrementingCountCursorModel): 2271 # We don't currently support usage of partition routing and IncrementingCountCursor at the 2272 # same time because we didn't solve for design questions like what the lookback window would 2273 # be as well as global cursor fall backs. We have not seen customers that have needed both 2274 # at the same time yet and are currently punting on this until we need to solve it. 2275 raise ValueError( 2276 f"The low-code framework does not currently support usage of a PartitionRouter and an IncrementingCountCursor at the same time. Please specify only one of these options for stream {stream_name}." 2277 ) 2278 return self.create_concurrent_cursor_from_perpartition_cursor( # type: ignore # This is a known issue that we are creating and returning a ConcurrentCursor which does not technically implement the (low-code) StreamSlicer. However, (low-code) StreamSlicer and ConcurrentCursor both implement StreamSlicer.stream_slices() which is the primary method needed for checkpointing 2279 state_manager=self._connector_state_manager, 2280 model_type=DatetimeBasedCursorModel, 2281 component_definition=model.incremental_sync.__dict__, 2282 stream_name=stream_name, 2283 stream_state=stream_state, 2284 stream_namespace=None, 2285 config=config or {}, 2286 partition_router=stream_slicer, 2287 attempt_to_create_cursor_if_not_provided=True, # FIXME can we remove that now? 2288 ) 2289 elif model.incremental_sync: 2290 if type(model.incremental_sync) == IncrementingCountCursorModel: 2291 return self.create_concurrent_cursor_from_incrementing_count_cursor( # type: ignore # This is a known issue that we are creating and returning a ConcurrentCursor which does not technically implement the (low-code) StreamSlicer. However, (low-code) StreamSlicer and ConcurrentCursor both implement StreamSlicer.stream_slices() which is the primary method needed for checkpointing 2292 model_type=IncrementingCountCursorModel, 2293 component_definition=model.incremental_sync.__dict__, 2294 stream_name=stream_name, 2295 stream_namespace=None, 2296 stream_state=stream_state, 2297 config=config or {}, 2298 ) 2299 elif type(model.incremental_sync) == DatetimeBasedCursorModel: 2300 return self.create_concurrent_cursor_from_datetime_based_cursor( # type: ignore # This is a known issue that we are creating and returning a ConcurrentCursor which does not technically implement the (low-code) StreamSlicer. However, (low-code) StreamSlicer and ConcurrentCursor both implement StreamSlicer.stream_slices() which is the primary method needed for checkpointing 2301 model_type=type(model.incremental_sync), 2302 component_definition=model.incremental_sync.__dict__, 2303 stream_name=stream_name, 2304 stream_namespace=None, 2305 stream_state=stream_state, 2306 config=config or {}, 2307 attempt_to_create_cursor_if_not_provided=True, 2308 ) 2309 else: 2310 raise ValueError( 2311 f"Incremental sync of type {type(model.incremental_sync)} is not supported" 2312 ) 2313 return FinalStateCursor(stream_name, None, self._message_repository) 2314 2315 def create_default_error_handler( 2316 self, model: DefaultErrorHandlerModel, config: Config, **kwargs: Any 2317 ) -> DefaultErrorHandler: 2318 backoff_strategies = [] 2319 if model.backoff_strategies: 2320 for backoff_strategy_model in model.backoff_strategies: 2321 backoff_strategies.append( 2322 self._create_component_from_model(model=backoff_strategy_model, config=config) 2323 ) 2324 2325 response_filters = [] 2326 if model.response_filters: 2327 for response_filter_model in model.response_filters: 2328 response_filters.append( 2329 self._create_component_from_model(model=response_filter_model, config=config) 2330 ) 2331 response_filters.append( 2332 HttpResponseFilter(config=config, parameters=model.parameters or {}) 2333 ) 2334 2335 return DefaultErrorHandler( 2336 backoff_strategies=backoff_strategies, 2337 max_retries=model.max_retries, 2338 response_filters=response_filters, 2339 config=config, 2340 parameters=model.parameters or {}, 2341 ) 2342 2343 def create_default_paginator( 2344 self, 2345 model: DefaultPaginatorModel, 2346 config: Config, 2347 *, 2348 url_base: str, 2349 extractor_model: Optional[Union[CustomRecordExtractorModel, DpathExtractorModel]] = None, 2350 decoder: Optional[Decoder] = None, 2351 cursor_used_for_stop_condition: Optional[Cursor] = None, 2352 ) -> Union[DefaultPaginator, PaginatorTestReadDecorator]: 2353 if decoder: 2354 if self._is_supported_decoder_for_pagination(decoder): 2355 decoder_to_use = PaginationDecoderDecorator(decoder=decoder) 2356 else: 2357 raise ValueError(self._UNSUPPORTED_DECODER_ERROR.format(decoder_type=type(decoder))) 2358 else: 2359 decoder_to_use = PaginationDecoderDecorator(decoder=JsonDecoder(parameters={})) 2360 page_size_option = ( 2361 self._create_component_from_model(model=model.page_size_option, config=config) 2362 if model.page_size_option 2363 else None 2364 ) 2365 page_token_option = ( 2366 self._create_component_from_model(model=model.page_token_option, config=config) 2367 if model.page_token_option 2368 else None 2369 ) 2370 pagination_strategy = self._create_component_from_model( 2371 model=model.pagination_strategy, 2372 config=config, 2373 decoder=decoder_to_use, 2374 extractor_model=extractor_model, 2375 ) 2376 if cursor_used_for_stop_condition: 2377 pagination_strategy = StopConditionPaginationStrategyDecorator( 2378 pagination_strategy, CursorStopCondition(cursor_used_for_stop_condition) 2379 ) 2380 paginator = DefaultPaginator( 2381 decoder=decoder_to_use, 2382 page_size_option=page_size_option, 2383 page_token_option=page_token_option, 2384 pagination_strategy=pagination_strategy, 2385 url_base=url_base, 2386 config=config, 2387 parameters=model.parameters or {}, 2388 ) 2389 if self._limit_pages_fetched_per_slice: 2390 return PaginatorTestReadDecorator(paginator, self._limit_pages_fetched_per_slice) 2391 return paginator 2392 2393 def create_dpath_extractor( 2394 self, 2395 model: DpathExtractorModel, 2396 config: Config, 2397 decoder: Optional[Decoder] = None, 2398 **kwargs: Any, 2399 ) -> DpathExtractor: 2400 if decoder: 2401 decoder_to_use = decoder 2402 else: 2403 decoder_to_use = JsonDecoder(parameters={}) 2404 model_field_path: List[Union[InterpolatedString, str]] = [x for x in model.field_path] 2405 2406 record_expander = None 2407 if model.record_expander: 2408 record_expander = self._create_component_from_model( 2409 model=model.record_expander, 2410 config=config, 2411 ) 2412 2413 return DpathExtractor( 2414 decoder=decoder_to_use, 2415 field_path=model_field_path, 2416 config=config, 2417 parameters=model.parameters or {}, 2418 record_expander=record_expander, 2419 ) 2420 2421 def create_record_expander( 2422 self, 2423 model: RecordExpanderModel, 2424 config: Config, 2425 **kwargs: Any, 2426 ) -> RecordExpander: 2427 return RecordExpander( 2428 expand_records_from_field=model.expand_records_from_field, 2429 config=config, 2430 parameters=model.parameters or {}, 2431 remain_original_record=model.remain_original_record or False, 2432 on_no_records=OnNoRecords(model.on_no_records.value) 2433 if model.on_no_records 2434 else OnNoRecords.skip, 2435 ) 2436 2437 @staticmethod 2438 def create_response_to_file_extractor( 2439 model: ResponseToFileExtractorModel, 2440 **kwargs: Any, 2441 ) -> ResponseToFileExtractor: 2442 return ResponseToFileExtractor( 2443 parameters=model.parameters or {}, 2444 preserve_na_values=model.preserve_na_values or False, 2445 ) 2446 2447 @staticmethod 2448 def create_exponential_backoff_strategy( 2449 model: ExponentialBackoffStrategyModel, config: Config 2450 ) -> ExponentialBackoffStrategy: 2451 ModelToComponentFactory._validate_jitter_range(model.jitter_range_in_seconds) 2452 return ExponentialBackoffStrategy( 2453 factor=model.factor or 5, 2454 jitter_range_in_seconds=model.jitter_range_in_seconds, 2455 parameters=model.parameters or {}, 2456 config=config, 2457 ) 2458 2459 @staticmethod 2460 def create_group_by_key(model: GroupByKeyMergeStrategyModel, config: Config) -> GroupByKey: 2461 return GroupByKey(model.key, config=config, parameters=model.parameters or {}) 2462 2463 def create_http_requester( 2464 self, 2465 model: HttpRequesterModel, 2466 config: Config, 2467 decoder: Decoder = JsonDecoder(parameters={}), 2468 query_properties_key: Optional[str] = None, 2469 use_cache: Optional[bool] = None, 2470 *, 2471 name: str, 2472 ) -> HttpRequester: 2473 authenticator = ( 2474 self._create_component_from_model( 2475 model=model.authenticator, 2476 config=config, 2477 url_base=model.url or model.url_base, 2478 name=name, 2479 decoder=decoder, 2480 ) 2481 if model.authenticator 2482 else None 2483 ) 2484 error_handler = ( 2485 self._create_component_from_model(model=model.error_handler, config=config) 2486 if model.error_handler 2487 else DefaultErrorHandler( 2488 backoff_strategies=[], 2489 response_filters=[], 2490 config=config, 2491 parameters=model.parameters or {}, 2492 ) 2493 ) 2494 2495 api_budget = self._api_budget 2496 2497 request_options_provider = InterpolatedRequestOptionsProvider( 2498 request_body=model.request_body, 2499 request_body_data=model.request_body_data, 2500 request_body_json=model.request_body_json, 2501 request_headers=model.request_headers, 2502 request_parameters=model.request_parameters, # type: ignore # QueryProperties have been removed in `create_simple_retriever` 2503 query_properties_key=query_properties_key, 2504 config=config, 2505 parameters=model.parameters or {}, 2506 ) 2507 2508 assert model.use_cache is not None # for mypy 2509 assert model.http_method is not None # for mypy 2510 2511 should_use_cache = (model.use_cache or bool(use_cache)) and not self._disable_cache 2512 2513 return HttpRequester( 2514 name=name, 2515 url=model.url, 2516 url_base=model.url_base, 2517 path=model.path, 2518 authenticator=authenticator, 2519 error_handler=error_handler, 2520 api_budget=api_budget, 2521 http_method=HttpMethod[model.http_method.value], 2522 request_options_provider=request_options_provider, 2523 config=config, 2524 disable_retries=self._disable_retries, 2525 parameters=model.parameters or {}, 2526 message_repository=self._message_repository, 2527 use_cache=should_use_cache, 2528 decoder=decoder, 2529 stream_response=decoder.is_stream_response() if decoder else False, 2530 ) 2531 2532 @staticmethod 2533 def create_http_response_filter( 2534 model: HttpResponseFilterModel, config: Config, **kwargs: Any 2535 ) -> HttpResponseFilter: 2536 if model.action: 2537 action = ResponseAction(model.action.value) 2538 else: 2539 action = None 2540 2541 failure_type = FailureType(model.failure_type.value) if model.failure_type else None 2542 2543 http_codes = ( 2544 set(model.http_codes) if model.http_codes else set() 2545 ) # JSON schema notation has no set data type. The schema enforces an array of unique elements 2546 2547 return HttpResponseFilter( 2548 action=action, 2549 failure_type=failure_type, 2550 error_message=model.error_message or "", 2551 error_message_contains=model.error_message_contains or "", 2552 http_codes=http_codes, 2553 predicate=model.predicate or "", 2554 config=config, 2555 parameters=model.parameters or {}, 2556 ) 2557 2558 @staticmethod 2559 def create_inline_schema_loader( 2560 model: InlineSchemaLoaderModel, config: Config, **kwargs: Any 2561 ) -> InlineSchemaLoader: 2562 return InlineSchemaLoader(schema=model.schema_ or {}, parameters={}) 2563 2564 def create_complex_field_type( 2565 self, model: ComplexFieldTypeModel, config: Config, **kwargs: Any 2566 ) -> ComplexFieldType: 2567 items = ( 2568 self._create_component_from_model(model=model.items, config=config) 2569 if isinstance(model.items, ComplexFieldTypeModel) 2570 else model.items 2571 ) 2572 2573 return ComplexFieldType(field_type=model.field_type, items=items) 2574 2575 def create_types_map(self, model: TypesMapModel, config: Config, **kwargs: Any) -> TypesMap: 2576 target_type = ( 2577 self._create_component_from_model(model=model.target_type, config=config) 2578 if isinstance(model.target_type, ComplexFieldTypeModel) 2579 else model.target_type 2580 ) 2581 2582 return TypesMap( 2583 target_type=target_type, 2584 current_type=model.current_type, 2585 condition=model.condition if model.condition is not None else "True", 2586 ) 2587 2588 def create_schema_type_identifier( 2589 self, model: SchemaTypeIdentifierModel, config: Config, **kwargs: Any 2590 ) -> SchemaTypeIdentifier: 2591 types_mapping = [] 2592 if model.types_mapping: 2593 types_mapping.extend( 2594 [ 2595 self._create_component_from_model(types_map, config=config) 2596 for types_map in model.types_mapping 2597 ] 2598 ) 2599 model_schema_pointer: List[Union[InterpolatedString, str]] = ( 2600 [x for x in model.schema_pointer] if model.schema_pointer else [] 2601 ) 2602 model_key_pointer: List[Union[InterpolatedString, str]] = [x for x in model.key_pointer] 2603 model_type_pointer: Optional[List[Union[InterpolatedString, str]]] = ( 2604 [x for x in model.type_pointer] if model.type_pointer else None 2605 ) 2606 2607 return SchemaTypeIdentifier( 2608 schema_pointer=model_schema_pointer, 2609 key_pointer=model_key_pointer, 2610 type_pointer=model_type_pointer, 2611 types_mapping=types_mapping, 2612 parameters=model.parameters or {}, 2613 ) 2614 2615 def create_dynamic_schema_loader( 2616 self, model: DynamicSchemaLoaderModel, config: Config, **kwargs: Any 2617 ) -> DynamicSchemaLoader: 2618 schema_transformations = [] 2619 if model.schema_transformations: 2620 for transformation_model in model.schema_transformations: 2621 schema_transformations.append( 2622 self._create_component_from_model(model=transformation_model, config=config) 2623 ) 2624 name = "dynamic_properties" 2625 retriever = self._create_component_from_model( 2626 model=model.retriever, 2627 config=config, 2628 name=name, 2629 primary_key=None, 2630 partition_router=self._build_stream_slicer_from_partition_router( 2631 model.retriever, config 2632 ), 2633 transformations=[], 2634 use_cache=True, 2635 log_formatter=( 2636 lambda response: format_http_message( 2637 response, 2638 f"Schema loader '{name}' request", 2639 f"Request performed in order to extract schema.", 2640 name, 2641 is_auxiliary=True, 2642 ) 2643 ), 2644 ) 2645 schema_type_identifier = self._create_component_from_model( 2646 model.schema_type_identifier, config=config, parameters=model.parameters or {} 2647 ) 2648 schema_filter = ( 2649 self._create_component_from_model( 2650 model.schema_filter, config=config, parameters=model.parameters or {} 2651 ) 2652 if model.schema_filter is not None 2653 else None 2654 ) 2655 2656 return DynamicSchemaLoader( 2657 retriever=retriever, 2658 config=config, 2659 schema_transformations=schema_transformations, 2660 schema_filter=schema_filter, 2661 schema_type_identifier=schema_type_identifier, 2662 parameters=model.parameters or {}, 2663 ) 2664 2665 @staticmethod 2666 def create_json_decoder(model: JsonDecoderModel, config: Config, **kwargs: Any) -> Decoder: 2667 return JsonDecoder(parameters={}) 2668 2669 def create_csv_decoder(self, model: CsvDecoderModel, config: Config, **kwargs: Any) -> Decoder: 2670 return CompositeRawDecoder( 2671 parser=ModelToComponentFactory._get_parser(model, config), 2672 stream_response=False if self._emit_connector_builder_messages else True, 2673 ) 2674 2675 def create_jsonl_decoder( 2676 self, model: JsonlDecoderModel, config: Config, **kwargs: Any 2677 ) -> Decoder: 2678 return CompositeRawDecoder( 2679 parser=ModelToComponentFactory._get_parser(model, config), 2680 stream_response=False if self._emit_connector_builder_messages else True, 2681 ) 2682 2683 def create_json_items_decoder( 2684 self, model: JsonItemsDecoderModel, config: Config, **kwargs: Any 2685 ) -> Decoder: 2686 return CompositeRawDecoder( 2687 parser=ModelToComponentFactory._get_parser(model, config), 2688 stream_response=False if self._emit_connector_builder_messages else True, 2689 ) 2690 2691 def create_gzip_decoder( 2692 self, model: GzipDecoderModel, config: Config, **kwargs: Any 2693 ) -> Decoder: 2694 _compressed_response_types = { 2695 "gzip", 2696 "x-gzip", 2697 "gzip, deflate", 2698 "x-gzip, deflate", 2699 "application/zip", 2700 "application/gzip", 2701 "application/x-gzip", 2702 "application/x-zip-compressed", 2703 } 2704 2705 gzip_parser: GzipParser = ModelToComponentFactory._get_parser(model, config) # type: ignore # based on the model, we know this will be a GzipParser 2706 2707 if self._emit_connector_builder_messages: 2708 # This is very surprising but if the response is not streamed, 2709 # CompositeRawDecoder calls response.content and the requests library actually uncompress the data as opposed to response.raw, 2710 # which uses urllib3 directly and does not uncompress the data. 2711 return CompositeRawDecoder(gzip_parser.inner_parser, False) 2712 2713 return CompositeRawDecoder.by_headers( 2714 [({"Content-Encoding", "Content-Type"}, _compressed_response_types, gzip_parser)], 2715 stream_response=True, 2716 fallback_parser=gzip_parser.inner_parser, 2717 ) 2718 2719 @staticmethod 2720 def create_iterable_decoder( 2721 model: IterableDecoderModel, config: Config, **kwargs: Any 2722 ) -> IterableDecoder: 2723 return IterableDecoder(parameters={}) 2724 2725 @staticmethod 2726 def create_xml_decoder(model: XmlDecoderModel, config: Config, **kwargs: Any) -> XmlDecoder: 2727 return XmlDecoder(parameters={}) 2728 2729 def create_zipfile_decoder( 2730 self, model: ZipfileDecoderModel, config: Config, **kwargs: Any 2731 ) -> ZipfileDecoder: 2732 return ZipfileDecoder(parser=ModelToComponentFactory._get_parser(model.decoder, config)) 2733 2734 @staticmethod 2735 def _get_parser(model: BaseModel, config: Config) -> Parser: 2736 if isinstance(model, JsonDecoderModel): 2737 # Note that the logic is a bit different from the JsonDecoder as there is some legacy that is maintained to return {} on error cases 2738 return JsonParser() 2739 elif isinstance(model, JsonItemsDecoderModel): 2740 return JsonItemsParser( 2741 items_path=model.items_path, 2742 encoding=model.encoding, 2743 ) 2744 elif isinstance(model, JsonlDecoderModel): 2745 return JsonLineParser() 2746 elif isinstance(model, CsvDecoderModel): 2747 return CsvParser( 2748 encoding=model.encoding, 2749 delimiter=model.delimiter, 2750 set_values_to_none=model.set_values_to_none, 2751 ) 2752 elif isinstance(model, GzipDecoderModel): 2753 return GzipParser( 2754 inner_parser=ModelToComponentFactory._get_parser(model.decoder, config) 2755 ) 2756 elif isinstance( 2757 model, (CustomDecoderModel, IterableDecoderModel, XmlDecoderModel, ZipfileDecoderModel) 2758 ): 2759 raise ValueError(f"Decoder type {model} does not have parser associated to it") 2760 2761 raise ValueError(f"Unknown decoder type {model}") 2762 2763 @staticmethod 2764 def create_json_file_schema_loader( 2765 model: JsonFileSchemaLoaderModel, config: Config, **kwargs: Any 2766 ) -> JsonFileSchemaLoader: 2767 return JsonFileSchemaLoader( 2768 file_path=model.file_path or "", config=config, parameters=model.parameters or {} 2769 ) 2770 2771 def create_jwt_authenticator( 2772 self, model: JwtAuthenticatorModel, config: Config, **kwargs: Any 2773 ) -> JwtAuthenticator: 2774 jwt_headers = model.jwt_headers or JwtHeadersModel(kid=None, typ="JWT", cty=None) 2775 jwt_payload = model.jwt_payload or JwtPayloadModel(iss=None, sub=None, aud=None) 2776 request_option = ( 2777 self._create_component_from_model(model.request_option, config) 2778 if model.request_option 2779 else None 2780 ) 2781 return JwtAuthenticator( 2782 config=config, 2783 parameters=model.parameters or {}, 2784 algorithm=JwtAlgorithm(model.algorithm.value), 2785 secret_key=model.secret_key, 2786 base64_encode_secret_key=model.base64_encode_secret_key, 2787 token_duration=model.token_duration, 2788 header_prefix=model.header_prefix, 2789 kid=jwt_headers.kid, 2790 typ=jwt_headers.typ, 2791 cty=jwt_headers.cty, 2792 iss=jwt_payload.iss, 2793 sub=jwt_payload.sub, 2794 aud=jwt_payload.aud, 2795 additional_jwt_headers=model.additional_jwt_headers, 2796 additional_jwt_payload=model.additional_jwt_payload, 2797 passphrase=model.passphrase, 2798 request_option=request_option, 2799 ) 2800 2801 def create_list_partition_router( 2802 self, model: ListPartitionRouterModel, config: Config, **kwargs: Any 2803 ) -> ListPartitionRouter: 2804 request_option = ( 2805 self._create_component_from_model(model.request_option, config) 2806 if model.request_option 2807 else None 2808 ) 2809 return ListPartitionRouter( 2810 cursor_field=model.cursor_field, 2811 request_option=request_option, 2812 values=model.values, 2813 config=config, 2814 parameters=model.parameters or {}, 2815 ) 2816 2817 @staticmethod 2818 def create_min_max_datetime( 2819 model: MinMaxDatetimeModel, config: Config, **kwargs: Any 2820 ) -> MinMaxDatetime: 2821 return MinMaxDatetime( 2822 datetime=model.datetime, 2823 datetime_format=model.datetime_format or "", 2824 max_datetime=model.max_datetime or "", 2825 min_datetime=model.min_datetime or "", 2826 parameters=model.parameters or {}, 2827 ) 2828 2829 @staticmethod 2830 def create_no_auth(model: NoAuthModel, config: Config, **kwargs: Any) -> NoAuth: 2831 return NoAuth(parameters=model.parameters or {}) 2832 2833 @staticmethod 2834 def create_no_pagination( 2835 model: NoPaginationModel, config: Config, **kwargs: Any 2836 ) -> NoPagination: 2837 return NoPagination(parameters={}) 2838 2839 def create_oauth_authenticator( 2840 self, model: OAuthAuthenticatorModel, config: Config, **kwargs: Any 2841 ) -> DeclarativeOauth2Authenticator: 2842 profile_assertion = ( 2843 self._create_component_from_model(model.profile_assertion, config=config) 2844 if model.profile_assertion 2845 else None 2846 ) 2847 2848 refresh_token_error_status_codes, refresh_token_error_key, refresh_token_error_values = ( 2849 self._get_refresh_token_error_information(model) 2850 ) 2851 if model.refresh_token_updater: 2852 # ignore type error because fixing it would have a lot of dependencies, revisit later 2853 return DeclarativeSingleUseRefreshTokenOauth2Authenticator( # type: ignore 2854 config, 2855 InterpolatedString.create( 2856 model.token_refresh_endpoint, # type: ignore 2857 parameters=model.parameters or {}, 2858 ).eval(config), 2859 access_token_name=InterpolatedString.create( 2860 model.access_token_name or "access_token", parameters=model.parameters or {} 2861 ).eval(config), 2862 refresh_token_name=model.refresh_token_updater.refresh_token_name, 2863 expires_in_name=InterpolatedString.create( 2864 model.expires_in_name or "expires_in", parameters=model.parameters or {} 2865 ).eval(config), 2866 client_id_name=InterpolatedString.create( 2867 model.client_id_name or "client_id", parameters=model.parameters or {} 2868 ).eval(config), 2869 client_id=InterpolatedString.create( 2870 model.client_id, parameters=model.parameters or {} 2871 ).eval(config) 2872 if model.client_id 2873 else model.client_id, 2874 client_secret_name=InterpolatedString.create( 2875 model.client_secret_name or "client_secret", parameters=model.parameters or {} 2876 ).eval(config), 2877 client_secret=InterpolatedString.create( 2878 model.client_secret, parameters=model.parameters or {} 2879 ).eval(config) 2880 if model.client_secret 2881 else model.client_secret, 2882 access_token_config_path=model.refresh_token_updater.access_token_config_path, 2883 refresh_token_config_path=model.refresh_token_updater.refresh_token_config_path, 2884 token_expiry_date_config_path=model.refresh_token_updater.token_expiry_date_config_path, 2885 grant_type_name=InterpolatedString.create( 2886 model.grant_type_name or "grant_type", parameters=model.parameters or {} 2887 ).eval(config), 2888 grant_type=InterpolatedString.create( 2889 model.grant_type or "refresh_token", parameters=model.parameters or {} 2890 ).eval(config), 2891 refresh_request_body=InterpolatedMapping( 2892 model.refresh_request_body or {}, parameters=model.parameters or {} 2893 ).eval(config), 2894 refresh_request_headers=InterpolatedMapping( 2895 model.refresh_request_headers or {}, parameters=model.parameters or {} 2896 ).eval(config), 2897 send_refresh_request_as_query_params=bool( 2898 model.send_refresh_request_as_query_params 2899 ), 2900 scopes=model.scopes, 2901 token_expiry_date_format=model.token_expiry_date_format, 2902 token_expiry_is_time_of_expiration=bool(model.token_expiry_date_format), 2903 message_repository=self._message_repository, 2904 refresh_token_error_status_codes=refresh_token_error_status_codes, 2905 refresh_token_error_key=refresh_token_error_key, 2906 refresh_token_error_values=refresh_token_error_values, 2907 ) 2908 # ignore type error because fixing it would have a lot of dependencies, revisit later 2909 return DeclarativeOauth2Authenticator( # type: ignore 2910 access_token_name=model.access_token_name or "access_token", 2911 access_token_value=model.access_token_value, 2912 client_id_name=model.client_id_name or "client_id", 2913 client_id=model.client_id, 2914 client_secret_name=model.client_secret_name or "client_secret", 2915 client_secret=model.client_secret, 2916 expires_in_name=model.expires_in_name or "expires_in", 2917 grant_type_name=model.grant_type_name or "grant_type", 2918 grant_type=model.grant_type or "refresh_token", 2919 refresh_request_body=model.refresh_request_body, 2920 refresh_request_headers=model.refresh_request_headers, 2921 send_refresh_request_as_query_params=bool(model.send_refresh_request_as_query_params), 2922 refresh_token_name=model.refresh_token_name or "refresh_token", 2923 refresh_token=model.refresh_token, 2924 scopes=model.scopes, 2925 token_expiry_date=model.token_expiry_date, 2926 token_expiry_date_format=model.token_expiry_date_format, 2927 token_expiry_is_time_of_expiration=bool(model.token_expiry_date_format), 2928 token_refresh_endpoint=model.token_refresh_endpoint, 2929 config=config, 2930 parameters=model.parameters or {}, 2931 message_repository=self._message_repository, 2932 profile_assertion=profile_assertion, 2933 use_profile_assertion=model.use_profile_assertion, 2934 refresh_token_error_status_codes=refresh_token_error_status_codes, 2935 refresh_token_error_key=refresh_token_error_key, 2936 refresh_token_error_values=refresh_token_error_values, 2937 ) 2938 2939 @staticmethod 2940 def _get_refresh_token_error_information( 2941 model: OAuthAuthenticatorModel, 2942 ) -> Tuple[Tuple[int, ...], str, Tuple[str, ...]]: 2943 """ 2944 In a previous version of the CDK, the auth error as config_error was only done if a refresh token updater was 2945 defined. As a transition, we added those fields on the OAuthAuthenticatorModel. This method ensures that the 2946 information is defined only once and return the right fields. 2947 """ 2948 refresh_token_updater = model.refresh_token_updater 2949 is_defined_on_refresh_token_updated = refresh_token_updater and ( 2950 refresh_token_updater.refresh_token_error_status_codes 2951 or refresh_token_updater.refresh_token_error_key 2952 or refresh_token_updater.refresh_token_error_values 2953 ) 2954 is_defined_on_oauth_authenticator = ( 2955 model.refresh_token_error_status_codes 2956 or model.refresh_token_error_key 2957 or model.refresh_token_error_values 2958 ) 2959 if is_defined_on_refresh_token_updated and is_defined_on_oauth_authenticator: 2960 raise ValueError( 2961 "refresh_token_error should either be defined on the OAuthAuthenticatorModel or the RefreshTokenUpdaterModel, not both" 2962 ) 2963 2964 if is_defined_on_refresh_token_updated: 2965 not_optional_refresh_token_updater: RefreshTokenUpdaterModel = refresh_token_updater # type: ignore # we know from the condition that this is not None 2966 return ( 2967 tuple(not_optional_refresh_token_updater.refresh_token_error_status_codes) 2968 if not_optional_refresh_token_updater.refresh_token_error_status_codes 2969 else (), 2970 not_optional_refresh_token_updater.refresh_token_error_key or "", 2971 tuple(not_optional_refresh_token_updater.refresh_token_error_values) 2972 if not_optional_refresh_token_updater.refresh_token_error_values 2973 else (), 2974 ) 2975 elif is_defined_on_oauth_authenticator: 2976 return ( 2977 tuple(model.refresh_token_error_status_codes) 2978 if model.refresh_token_error_status_codes 2979 else (), 2980 model.refresh_token_error_key or "", 2981 tuple(model.refresh_token_error_values) if model.refresh_token_error_values else (), 2982 ) 2983 2984 # returning default values we think cover most cases 2985 return (400,), "error", ("invalid_grant", "invalid_permissions") 2986 2987 def create_offset_increment( 2988 self, 2989 model: OffsetIncrementModel, 2990 config: Config, 2991 decoder: Decoder, 2992 extractor_model: Optional[Union[CustomRecordExtractorModel, DpathExtractorModel]] = None, 2993 **kwargs: Any, 2994 ) -> OffsetIncrement: 2995 if isinstance(decoder, PaginationDecoderDecorator): 2996 inner_decoder = decoder.decoder 2997 else: 2998 inner_decoder = decoder 2999 decoder = PaginationDecoderDecorator(decoder=decoder) 3000 3001 if self._is_supported_decoder_for_pagination(inner_decoder): 3002 decoder_to_use = decoder 3003 else: 3004 raise ValueError( 3005 self._UNSUPPORTED_DECODER_ERROR.format(decoder_type=type(inner_decoder)) 3006 ) 3007 3008 # Ideally we would instantiate the runtime extractor from highest most level (in this case the SimpleRetriever) 3009 # so that it can be shared by OffSetIncrement and RecordSelector. However, due to how we instantiate the 3010 # decoder with various decorators here, but not in create_record_selector, it is simpler to retain existing 3011 # behavior by having two separate extractors with identical behavior since they use the same extractor model. 3012 # When we have more time to investigate we can look into reusing the same component. 3013 extractor = ( 3014 self._create_component_from_model( 3015 model=extractor_model, config=config, decoder=decoder_to_use 3016 ) 3017 if extractor_model 3018 else None 3019 ) 3020 3021 # Pydantic v1 Union type coercion can convert int to string depending on Union order. 3022 # If page_size is a string that represents an integer (not an interpolation), convert it back. 3023 page_size = model.page_size 3024 if isinstance(page_size, str) and page_size.isdigit(): 3025 page_size = int(page_size) 3026 3027 return OffsetIncrement( 3028 page_size=page_size, 3029 config=config, 3030 decoder=decoder_to_use, 3031 extractor=extractor, 3032 inject_on_first_request=model.inject_on_first_request or False, 3033 parameters=model.parameters or {}, 3034 ) 3035 3036 def create_page_increment( 3037 self, 3038 model: PageIncrementModel, 3039 config: Config, 3040 decoder: Optional[Decoder] = None, 3041 extractor_model: Optional[Union[CustomRecordExtractorModel, DpathExtractorModel]] = None, 3042 **kwargs: Any, 3043 ) -> PageIncrement: 3044 # Like OffsetIncrement, we instantiate a separate extractor with identical behavior to the 3045 # RecordSelector's so the strategy can count the raw records in the response. This ensures 3046 # pagination is driven by the API's page size, not the post-filter record count. 3047 extractor = ( 3048 self._create_component_from_model(model=extractor_model, config=config, decoder=decoder) 3049 if extractor_model 3050 else None 3051 ) 3052 3053 # Pydantic v1 Union type coercion can convert int to string depending on Union order. 3054 # If page_size is a string that represents an integer (not an interpolation), convert it back. 3055 page_size = model.page_size 3056 if isinstance(page_size, str) and page_size.isdigit(): 3057 page_size = int(page_size) 3058 3059 return PageIncrement( 3060 page_size=page_size, 3061 config=config, 3062 start_from_page=model.start_from_page or 0, 3063 inject_on_first_request=model.inject_on_first_request or False, 3064 extractor=extractor, 3065 parameters=model.parameters or {}, 3066 ) 3067 3068 def create_parent_stream_config( 3069 self, model: ParentStreamConfigModel, config: Config, *, stream_name: str, **kwargs: Any 3070 ) -> ParentStreamConfig: 3071 declarative_stream = self._create_component_from_model( 3072 model.stream, 3073 config=config, 3074 is_parent=True, 3075 **kwargs, 3076 ) 3077 request_option = ( 3078 self._create_component_from_model(model.request_option, config=config) 3079 if model.request_option 3080 else None 3081 ) 3082 3083 if model.lazy_read_pointer and any("*" in pointer for pointer in model.lazy_read_pointer): 3084 raise ValueError( 3085 "The '*' wildcard in 'lazy_read_pointer' is not supported — only direct paths are allowed." 3086 ) 3087 3088 model_lazy_read_pointer: List[Union[InterpolatedString, str]] = ( 3089 [x for x in model.lazy_read_pointer] if model.lazy_read_pointer else [] 3090 ) 3091 3092 return ParentStreamConfig( 3093 parent_key=model.parent_key, 3094 request_option=request_option, 3095 stream=declarative_stream, 3096 partition_field=model.partition_field, 3097 config=config, 3098 incremental_dependency=model.incremental_dependency or False, 3099 parameters=model.parameters or {}, 3100 extra_fields=model.extra_fields, 3101 lazy_read_pointer=model_lazy_read_pointer, 3102 ) 3103 3104 def create_properties_from_endpoint( 3105 self, model: PropertiesFromEndpointModel, config: Config, **kwargs: Any 3106 ) -> PropertiesFromEndpoint: 3107 retriever = self._create_component_from_model( 3108 model=model.retriever, 3109 config=config, 3110 name="dynamic_properties", 3111 primary_key=None, 3112 stream_slicer=None, 3113 transformations=[], 3114 use_cache=True, # Enable caching on the HttpRequester/HttpClient because the properties endpoint will be called for every slice being processed, and it is highly unlikely for the response to different 3115 ) 3116 return PropertiesFromEndpoint( 3117 property_field_path=model.property_field_path, 3118 retriever=retriever, 3119 config=config, 3120 parameters=model.parameters or {}, 3121 ) 3122 3123 def create_property_chunking( 3124 self, model: PropertyChunkingModel, config: Config, **kwargs: Any 3125 ) -> PropertyChunking: 3126 record_merge_strategy = ( 3127 self._create_component_from_model( 3128 model=model.record_merge_strategy, config=config, **kwargs 3129 ) 3130 if model.record_merge_strategy 3131 else None 3132 ) 3133 3134 property_limit_type: PropertyLimitType 3135 match model.property_limit_type: 3136 case PropertyLimitTypeModel.property_count: 3137 property_limit_type = PropertyLimitType.property_count 3138 case PropertyLimitTypeModel.characters: 3139 property_limit_type = PropertyLimitType.characters 3140 case _: 3141 raise ValueError(f"Invalid PropertyLimitType {property_limit_type}") 3142 3143 return PropertyChunking( 3144 property_limit_type=property_limit_type, 3145 property_limit=model.property_limit, 3146 record_merge_strategy=record_merge_strategy, 3147 config=config, 3148 parameters=model.parameters or {}, 3149 ) 3150 3151 def create_query_properties( 3152 self, model: QueryPropertiesModel, config: Config, *, stream_name: str, **kwargs: Any 3153 ) -> QueryProperties: 3154 if isinstance(model.property_list, list): 3155 property_list = model.property_list 3156 else: 3157 property_list = self._create_component_from_model( 3158 model=model.property_list, config=config, **kwargs 3159 ) 3160 3161 property_chunking = ( 3162 self._create_component_from_model( 3163 model=model.property_chunking, config=config, **kwargs 3164 ) 3165 if model.property_chunking 3166 else None 3167 ) 3168 3169 property_selector = ( 3170 self._create_component_from_model( 3171 model=model.property_selector, config=config, stream_name=stream_name, **kwargs 3172 ) 3173 if model.property_selector 3174 else None 3175 ) 3176 3177 return QueryProperties( 3178 property_list=property_list, 3179 always_include_properties=model.always_include_properties, 3180 property_chunking=property_chunking, 3181 property_selector=property_selector, 3182 config=config, 3183 parameters=model.parameters or {}, 3184 ) 3185 3186 def create_json_schema_property_selector( 3187 self, 3188 model: JsonSchemaPropertySelectorModel, 3189 config: Config, 3190 *, 3191 stream_name: str, 3192 **kwargs: Any, 3193 ) -> JsonSchemaPropertySelector: 3194 configured_stream = self._stream_name_to_configured_stream.get(stream_name) 3195 3196 transformations = [] 3197 if model.transformations: 3198 for transformation_model in model.transformations: 3199 transformations.append( 3200 self._create_component_from_model(model=transformation_model, config=config) 3201 ) 3202 3203 return JsonSchemaPropertySelector( 3204 configured_stream=configured_stream, 3205 properties_transformations=transformations, 3206 config=config, 3207 parameters=model.parameters or {}, 3208 ) 3209 3210 @staticmethod 3211 def create_record_filter( 3212 model: RecordFilterModel, config: Config, **kwargs: Any 3213 ) -> RecordFilter: 3214 return RecordFilter( 3215 condition=model.condition or "", config=config, parameters=model.parameters or {} 3216 ) 3217 3218 @staticmethod 3219 def create_request_path(model: RequestPathModel, config: Config, **kwargs: Any) -> RequestPath: 3220 return RequestPath(parameters={}) 3221 3222 @staticmethod 3223 def create_request_option( 3224 model: RequestOptionModel, config: Config, **kwargs: Any 3225 ) -> RequestOption: 3226 inject_into = RequestOptionType(model.inject_into.value) 3227 field_path: Optional[List[Union[InterpolatedString, str]]] = ( 3228 [ 3229 InterpolatedString.create(segment, parameters=kwargs.get("parameters", {})) 3230 for segment in model.field_path 3231 ] 3232 if model.field_path 3233 else None 3234 ) 3235 field_name = ( 3236 InterpolatedString.create(model.field_name, parameters=kwargs.get("parameters", {})) 3237 if model.field_name 3238 else None 3239 ) 3240 return RequestOption( 3241 field_name=field_name, 3242 field_path=field_path, 3243 inject_into=inject_into, 3244 parameters=kwargs.get("parameters", {}), 3245 ) 3246 3247 def create_record_selector( 3248 self, 3249 model: RecordSelectorModel, 3250 config: Config, 3251 *, 3252 name: str, 3253 transformations: List[RecordTransformation] | None = None, 3254 decoder: Decoder | None = None, 3255 client_side_incremental_sync_cursor: Optional[Cursor] = None, 3256 file_uploader: Optional[DefaultFileUploader] = None, 3257 **kwargs: Any, 3258 ) -> RecordSelector: 3259 extractor = self._create_component_from_model( 3260 model=model.extractor, decoder=decoder, config=config 3261 ) 3262 record_filter = ( 3263 self._create_component_from_model(model.record_filter, config=config) 3264 if model.record_filter 3265 else None 3266 ) 3267 3268 transform_before_filtering = ( 3269 False if model.transform_before_filtering is None else model.transform_before_filtering 3270 ) 3271 if client_side_incremental_sync_cursor: 3272 record_filter = ClientSideIncrementalRecordFilterDecorator( 3273 config=config, 3274 parameters=model.parameters, 3275 condition=model.record_filter.condition 3276 if (model.record_filter and hasattr(model.record_filter, "condition")) 3277 else None, 3278 cursor=client_side_incremental_sync_cursor, 3279 ) 3280 transform_before_filtering = ( 3281 True 3282 if model.transform_before_filtering is None 3283 else model.transform_before_filtering 3284 ) 3285 3286 if model.schema_normalization is None: 3287 # default to no schema normalization if not set 3288 model.schema_normalization = SchemaNormalizationModel.None_ 3289 3290 schema_normalization = ( 3291 TypeTransformer(SCHEMA_TRANSFORMER_TYPE_MAPPING[model.schema_normalization]) 3292 if isinstance(model.schema_normalization, SchemaNormalizationModel) 3293 else self._create_component_from_model(model.schema_normalization, config=config) # type: ignore[arg-type] # custom normalization model expected here 3294 ) 3295 3296 return RecordSelector( 3297 extractor=extractor, 3298 name=name, 3299 config=config, 3300 record_filter=record_filter, 3301 transformations=transformations or [], 3302 file_uploader=file_uploader, 3303 schema_normalization=schema_normalization, 3304 parameters=model.parameters or {}, 3305 transform_before_filtering=transform_before_filtering, 3306 ) 3307 3308 @staticmethod 3309 def create_remove_fields( 3310 model: RemoveFieldsModel, config: Config, **kwargs: Any 3311 ) -> RemoveFields: 3312 return RemoveFields( 3313 field_pointers=model.field_pointers, condition=model.condition or "", parameters={} 3314 ) 3315 3316 def create_selective_authenticator( 3317 self, model: SelectiveAuthenticatorModel, config: Config, **kwargs: Any 3318 ) -> DeclarativeAuthenticator: 3319 authenticators = { 3320 name: self._create_component_from_model(model=auth, config=config) 3321 for name, auth in model.authenticators.items() 3322 } 3323 # SelectiveAuthenticator will return instance of DeclarativeAuthenticator or raise ValueError error 3324 return SelectiveAuthenticator( # type: ignore[abstract] 3325 config=config, 3326 authenticators=authenticators, 3327 authenticator_selection_path=model.authenticator_selection_path, 3328 **kwargs, 3329 ) 3330 3331 @staticmethod 3332 def create_legacy_session_token_authenticator( 3333 model: LegacySessionTokenAuthenticatorModel, config: Config, *, url_base: str, **kwargs: Any 3334 ) -> LegacySessionTokenAuthenticator: 3335 return LegacySessionTokenAuthenticator( 3336 api_url=url_base, 3337 header=model.header, 3338 login_url=model.login_url, 3339 password=model.password or "", 3340 session_token=model.session_token or "", 3341 session_token_response_key=model.session_token_response_key or "", 3342 username=model.username or "", 3343 validate_session_url=model.validate_session_url, 3344 config=config, 3345 parameters=model.parameters or {}, 3346 ) 3347 3348 def create_simple_retriever( 3349 self, 3350 model: SimpleRetrieverModel, 3351 config: Config, 3352 *, 3353 name: str, 3354 primary_key: Optional[Union[str, List[str], List[List[str]]]], 3355 request_options_provider: Optional[RequestOptionsProvider] = None, 3356 cursor: Optional[Cursor] = None, 3357 has_stop_condition_cursor: bool = False, 3358 is_client_side_incremental_sync: bool = False, 3359 transformations: List[RecordTransformation], 3360 file_uploader: Optional[DefaultFileUploader] = None, 3361 incremental_sync: Optional[ 3362 Union[IncrementingCountCursorModel, DatetimeBasedCursorModel] 3363 ] = None, 3364 use_cache: Optional[bool] = None, 3365 log_formatter: Optional[Callable[[Response], Any]] = None, 3366 partition_router: Optional[PartitionRouter] = None, 3367 **kwargs: Any, 3368 ) -> SimpleRetriever: 3369 def _get_url(req: Requester) -> str: 3370 """ 3371 Closure to get the URL from the requester. This is used to get the URL in the case of a lazy retriever. 3372 This is needed because the URL is not set until the requester is created. 3373 """ 3374 3375 _url: str = ( 3376 model.requester.url 3377 if hasattr(model.requester, "url") and model.requester.url is not None 3378 else req.get_url(stream_state=None, stream_slice=None, next_page_token=None) 3379 ) 3380 _url_base: str = ( 3381 model.requester.url_base 3382 if hasattr(model.requester, "url_base") and model.requester.url_base is not None 3383 else req.get_url_base(stream_state=None, stream_slice=None, next_page_token=None) 3384 ) 3385 3386 return _url or _url_base 3387 3388 if cursor is None: 3389 cursor = FinalStateCursor(name, None, self._message_repository) 3390 3391 decoder = ( 3392 self._create_component_from_model(model=model.decoder, config=config) 3393 if model.decoder 3394 else JsonDecoder(parameters={}) 3395 ) 3396 record_selector = self._create_component_from_model( 3397 model=model.record_selector, 3398 name=name, 3399 config=config, 3400 decoder=decoder, 3401 transformations=transformations, 3402 client_side_incremental_sync_cursor=cursor if is_client_side_incremental_sync else None, 3403 file_uploader=file_uploader, 3404 ) 3405 3406 query_properties: Optional[QueryProperties] = None 3407 query_properties_key: Optional[str] = None 3408 self._ensure_query_properties_to_model(model.requester) 3409 if self._has_query_properties_in_request_parameters(model.requester): 3410 # It is better to be explicit about an error if PropertiesFromEndpoint is defined in multiple 3411 # places instead of default to request_parameters which isn't clearly documented 3412 if ( 3413 hasattr(model.requester, "fetch_properties_from_endpoint") 3414 and model.requester.fetch_properties_from_endpoint 3415 ): 3416 raise ValueError( 3417 f"PropertiesFromEndpoint should only be specified once per stream, but found in {model.requester.type}.fetch_properties_from_endpoint and {model.requester.type}.request_parameters" 3418 ) 3419 3420 query_properties_definitions = [] 3421 for key, request_parameter in model.requester.request_parameters.items(): # type: ignore # request_parameters is already validated to be a Mapping using _has_query_properties_in_request_parameters() 3422 if isinstance(request_parameter, QueryPropertiesModel): 3423 query_properties_key = key 3424 query_properties_definitions.append(request_parameter) 3425 3426 if len(query_properties_definitions) > 1: 3427 raise ValueError( 3428 f"request_parameters only supports defining one QueryProperties field, but found {len(query_properties_definitions)} usages" 3429 ) 3430 3431 if len(query_properties_definitions) == 1: 3432 query_properties = self._create_component_from_model( 3433 model=query_properties_definitions[0], stream_name=name, config=config 3434 ) 3435 3436 # Removes QueryProperties components from the interpolated mappings because it has been designed 3437 # to be used by the SimpleRetriever and will be resolved from the provider from the slice directly 3438 # instead of through jinja interpolation 3439 if hasattr(model.requester, "request_parameters") and isinstance( 3440 model.requester.request_parameters, Mapping 3441 ): 3442 model.requester.request_parameters = self._remove_query_properties( 3443 model.requester.request_parameters 3444 ) 3445 elif ( 3446 hasattr(model.requester, "fetch_properties_from_endpoint") 3447 and model.requester.fetch_properties_from_endpoint 3448 ): 3449 # todo: Deprecate this condition once dependent connectors migrate to query_properties 3450 query_properties_definition = QueryPropertiesModel( 3451 type="QueryProperties", 3452 property_list=model.requester.fetch_properties_from_endpoint, 3453 always_include_properties=None, 3454 property_chunking=None, 3455 ) # type: ignore # $parameters has a default value 3456 3457 query_properties = self.create_query_properties( 3458 model=query_properties_definition, 3459 stream_name=name, 3460 config=config, 3461 ) 3462 elif hasattr(model.requester, "query_properties") and model.requester.query_properties: 3463 query_properties = self.create_query_properties( 3464 model=model.requester.query_properties, 3465 stream_name=name, 3466 config=config, 3467 ) 3468 3469 requester = self._create_component_from_model( 3470 model=model.requester, 3471 decoder=decoder, 3472 name=name, 3473 query_properties_key=query_properties_key, 3474 use_cache=use_cache, 3475 config=config, 3476 ) 3477 3478 if not request_options_provider: 3479 request_options_provider = DefaultRequestOptionsProvider(parameters={}) 3480 if isinstance(request_options_provider, DefaultRequestOptionsProvider) and isinstance( 3481 partition_router, PartitionRouter 3482 ): 3483 request_options_provider = partition_router 3484 3485 paginator = ( 3486 self._create_component_from_model( 3487 model=model.paginator, 3488 config=config, 3489 url_base=_get_url(requester), 3490 extractor_model=model.record_selector.extractor, 3491 decoder=decoder, 3492 cursor_used_for_stop_condition=cursor if has_stop_condition_cursor else None, 3493 ) 3494 if model.paginator 3495 else NoPagination(parameters={}) 3496 ) 3497 3498 ignore_stream_slicer_parameters_on_paginated_requests = ( 3499 model.ignore_stream_slicer_parameters_on_paginated_requests or False 3500 ) 3501 3502 if ( 3503 model.partition_router 3504 and isinstance(model.partition_router, SubstreamPartitionRouterModel) 3505 and not bool(self._connector_state_manager.get_stream_state(name, None)) 3506 and any( 3507 parent_stream_config.lazy_read_pointer 3508 for parent_stream_config in model.partition_router.parent_stream_configs 3509 ) 3510 ): 3511 if incremental_sync: 3512 if incremental_sync.type != "DatetimeBasedCursor": 3513 raise ValueError( 3514 f"LazySimpleRetriever only supports DatetimeBasedCursor. Found: {incremental_sync.type}." 3515 ) 3516 3517 elif incremental_sync.step or incremental_sync.cursor_granularity: 3518 raise ValueError( 3519 f"Found more that one slice per parent. LazySimpleRetriever only supports single slice read for stream - {name}." 3520 ) 3521 3522 if model.decoder and model.decoder.type != "JsonDecoder": 3523 raise ValueError( 3524 f"LazySimpleRetriever only supports JsonDecoder. Found: {model.decoder.type}." 3525 ) 3526 3527 return LazySimpleRetriever( 3528 name=name, 3529 paginator=paginator, 3530 primary_key=primary_key, 3531 requester=requester, 3532 record_selector=record_selector, 3533 stream_slicer=_NO_STREAM_SLICING, 3534 request_option_provider=request_options_provider, 3535 config=config, 3536 ignore_stream_slicer_parameters_on_paginated_requests=ignore_stream_slicer_parameters_on_paginated_requests, 3537 parameters=model.parameters or {}, 3538 ) 3539 3540 if ( 3541 model.record_selector.record_filter 3542 and model.pagination_reset 3543 and model.pagination_reset.limits 3544 ): 3545 raise ValueError("PaginationResetLimits are not supported while having record filter.") 3546 3547 return SimpleRetriever( 3548 name=name, 3549 paginator=paginator, 3550 primary_key=primary_key, 3551 requester=requester, 3552 record_selector=record_selector, 3553 stream_slicer=_NO_STREAM_SLICING, 3554 request_option_provider=request_options_provider, 3555 config=config, 3556 ignore_stream_slicer_parameters_on_paginated_requests=ignore_stream_slicer_parameters_on_paginated_requests, 3557 additional_query_properties=query_properties, 3558 log_formatter=self._get_log_formatter(log_formatter, name), 3559 pagination_tracker_factory=self._create_pagination_tracker_factory( 3560 model.pagination_reset, cursor 3561 ), 3562 parameters=model.parameters or {}, 3563 ) 3564 3565 def _create_pagination_tracker_factory( 3566 self, model: Optional[PaginationResetModel], cursor: Cursor 3567 ) -> Callable[[], PaginationTracker]: 3568 if model is None: 3569 return lambda: PaginationTracker() 3570 3571 # Until we figure out a way to use any cursor for PaginationTracker, we will have to have this cursor selector logic 3572 cursor_factory: Callable[[], Optional[ConcurrentCursor]] = lambda: None 3573 if model.action == PaginationResetActionModel.RESET: 3574 # in that case, we will let cursor_factory to return None even if the stream has a cursor 3575 pass 3576 elif model.action == PaginationResetActionModel.SPLIT_USING_CURSOR: 3577 if isinstance(cursor, ConcurrentCursor): 3578 cursor_factory = lambda: cursor.copy_without_state() # type: ignore # the if condition validates that it is a ConcurrentCursor 3579 elif isinstance(cursor, ConcurrentPerPartitionCursor): 3580 cursor_factory = lambda: cursor._cursor_factory.create( # type: ignore # if this becomes a problem, we would need to extract the cursor_factory instantiation logic and make it accessible here 3581 {}, datetime.timedelta(0) 3582 ) 3583 elif not isinstance(cursor, FinalStateCursor): 3584 LOGGER.warning( 3585 "Unknown cursor for PaginationTracker. Pagination resets might not work properly" 3586 ) 3587 else: 3588 raise ValueError(f"Unknown PaginationReset action: {model.action}") 3589 3590 limit = model.limits.number_of_records if model and model.limits else None 3591 return lambda: PaginationTracker(cursor_factory(), limit) 3592 3593 def _get_log_formatter( 3594 self, log_formatter: Callable[[Response], Any] | None, name: str 3595 ) -> Callable[[Response], Any] | None: 3596 if self._should_limit_slices_fetched(): 3597 return ( 3598 ( 3599 lambda response: format_http_message( 3600 response, 3601 f"Stream '{name}' request", 3602 f"Request performed in order to extract records for stream '{name}'", 3603 name, 3604 ) 3605 ) 3606 if not log_formatter 3607 else log_formatter 3608 ) 3609 return None 3610 3611 def _should_limit_slices_fetched(self) -> bool: 3612 """ 3613 Returns True if the number of slices fetched should be limited, False otherwise. 3614 This is used to limit the number of slices fetched during tests. 3615 """ 3616 return bool(self._limit_slices_fetched or self._emit_connector_builder_messages) 3617 3618 @staticmethod 3619 def _has_query_properties_in_request_parameters( 3620 requester: Union[HttpRequesterModel, CustomRequesterModel], 3621 ) -> bool: 3622 if not hasattr(requester, "request_parameters"): 3623 return False 3624 request_parameters = requester.request_parameters 3625 if request_parameters and isinstance(request_parameters, Mapping): 3626 for request_parameter in request_parameters.values(): 3627 if isinstance(request_parameter, QueryPropertiesModel): 3628 return True 3629 return False 3630 3631 @staticmethod 3632 def _remove_query_properties( 3633 request_parameters: Mapping[str, Union[str, QueryPropertiesModel]], 3634 ) -> Mapping[str, str]: 3635 return { 3636 parameter_field: request_parameter 3637 for parameter_field, request_parameter in request_parameters.items() 3638 if not isinstance(request_parameter, QueryPropertiesModel) 3639 } 3640 3641 def create_state_delegating_stream( 3642 self, 3643 model: StateDelegatingStreamModel, 3644 config: Config, 3645 **kwargs: Any, 3646 ) -> DefaultStream: 3647 if ( 3648 model.full_refresh_stream.name != model.name 3649 or model.name != model.incremental_stream.name 3650 ): 3651 raise ValueError( 3652 f"state_delegating_stream, full_refresh_stream name and incremental_stream must have equal names. Instead has {model.name}, {model.full_refresh_stream.name} and {model.incremental_stream.name}." 3653 ) 3654 3655 # Resolve api_retention_period with config context (supports Jinja2 interpolation) 3656 resolved_retention_period: Optional[str] = None 3657 if model.api_retention_period: 3658 interpolated_retention = InterpolatedString.create( 3659 model.api_retention_period, parameters=model.parameters or {} 3660 ) 3661 resolved_value = interpolated_retention.eval(config=config) 3662 if resolved_value: 3663 resolved_retention_period = str(resolved_value) 3664 3665 if resolved_retention_period: 3666 for stream_model in (model.full_refresh_stream, model.incremental_stream): 3667 if isinstance(stream_model.incremental_sync, IncrementingCountCursorModel): 3668 raise ValueError( 3669 f"Stream '{model.name}' uses IncrementingCountCursor which is not supported " 3670 f"with api_retention_period. IncrementingCountCursor does not use datetime-based " 3671 f"cursors, so cursor age validation cannot be performed." 3672 ) 3673 3674 stream_state = self._connector_state_manager.get_stream_state(model.name, None) 3675 3676 if not stream_state: 3677 return self._create_component_from_model( # type: ignore[no-any-return] 3678 model.full_refresh_stream, config=config, **kwargs 3679 ) 3680 3681 incremental_stream: DefaultStream = self._create_component_from_model( 3682 model.incremental_stream, config=config, **kwargs 3683 ) # type: ignore[assignment] 3684 3685 # Only run cursor age validation for streams that are in the configured 3686 # catalog (or when no catalog was provided, e.g. during discover / connector 3687 # builder). Streams not selected by the user but instantiated as parent-stream 3688 # dependencies must not go through this path because it emits state messages 3689 # that the destination does not know about, causing "Stream not found" crashes. 3690 stream_is_in_catalog = ( 3691 not self._stream_name_to_configured_stream # no catalog → validate by default 3692 or model.name in self._stream_name_to_configured_stream 3693 ) 3694 if resolved_retention_period and stream_is_in_catalog: 3695 full_refresh_stream: DefaultStream = self._create_component_from_model( 3696 model.full_refresh_stream, config=config, **kwargs 3697 ) # type: ignore[assignment] 3698 if self._is_cursor_older_than_retention_period( 3699 stream_state, 3700 full_refresh_stream.cursor, 3701 incremental_stream.cursor, 3702 resolved_retention_period, 3703 model.name, 3704 ): 3705 # Clear state BEFORE constructing the full_refresh_stream so that 3706 # its cursor starts from start_date instead of the stale cursor. 3707 self._connector_state_manager.update_state_for_stream(model.name, None, {}) 3708 state_message = self._connector_state_manager.create_state_message(model.name, None) 3709 self._message_repository.emit_message(state_message) 3710 return self._create_component_from_model( # type: ignore[no-any-return] 3711 model.full_refresh_stream, config=config, **kwargs 3712 ) 3713 3714 return incremental_stream 3715 3716 @staticmethod 3717 def _is_cursor_older_than_retention_period( 3718 stream_state: Mapping[str, Any], 3719 full_refresh_cursor: Cursor, 3720 incremental_cursor: Cursor, 3721 api_retention_period: str, 3722 stream_name: str, 3723 ) -> bool: 3724 """Check if the cursor value in the state is older than the API's retention period. 3725 3726 Checks cursors in sequence: full refresh cursor first, then incremental cursor. 3727 FinalStateCursor returns now() for completed full refresh state (NO_CURSOR_STATE_KEY), 3728 which is always within retention, so we use incremental. For other states, it returns 3729 None and we fall back to checking the incremental cursor. 3730 3731 Returns True if the cursor is older than the retention period (should use full refresh). 3732 Returns False if the cursor is within the retention period (safe to use incremental). 3733 """ 3734 retention_duration = parse_duration(api_retention_period) 3735 retention_cutoff = datetime.datetime.now(datetime.timezone.utc) - retention_duration 3736 3737 # Check full refresh cursor first 3738 cursor_datetime = full_refresh_cursor.get_cursor_datetime_from_state(stream_state) 3739 3740 # If full refresh cursor returns None, check incremental cursor 3741 if cursor_datetime is None: 3742 cursor_datetime = incremental_cursor.get_cursor_datetime_from_state(stream_state) 3743 3744 if cursor_datetime is None: 3745 # Neither cursor could parse the state - fall back to full refresh to be safe 3746 return True 3747 3748 if cursor_datetime < retention_cutoff: 3749 logging.warning( 3750 f"Stream '{stream_name}' has a cursor value older than " 3751 f"the API's retention period of {api_retention_period} " 3752 f"(cutoff: {retention_cutoff.isoformat()}). " 3753 f"Falling back to full refresh to avoid data loss." 3754 ) 3755 return True 3756 3757 return False 3758 3759 def _get_state_delegating_stream_model( 3760 self, 3761 model: StateDelegatingStreamModel, 3762 parent_state: Optional[Mapping[str, Any]] = None, 3763 ) -> DeclarativeStreamModel: 3764 """Return the appropriate underlying stream model based on state.""" 3765 return ( 3766 model.incremental_stream 3767 if self._connector_state_manager.get_stream_state(model.name, None) or parent_state 3768 else model.full_refresh_stream 3769 ) 3770 3771 _OPTIONAL_ASYNC_STATUS_FIELDS = {"skipped"} 3772 3773 def _create_async_job_status_mapping( 3774 self, model: AsyncJobStatusMapModel, config: Config, **kwargs: Any 3775 ) -> Mapping[str, AsyncJobStatus]: 3776 api_status_to_cdk_status = {} 3777 for cdk_status, api_statuses in model.dict().items(): 3778 if cdk_status == "type": 3779 # This is an element of the dict because of the typing of the CDK but it is not a CDK status 3780 continue 3781 3782 if api_statuses is None: 3783 if cdk_status in self._OPTIONAL_ASYNC_STATUS_FIELDS: 3784 continue 3785 raise ValueError( 3786 f"Required CDK status '{cdk_status}' has no API statuses mapped. " 3787 f"Please provide at least an empty list for required status fields." 3788 ) 3789 3790 for status in api_statuses: 3791 if status in api_status_to_cdk_status: 3792 raise ValueError( 3793 f"API status {status} is already set for CDK status {cdk_status}. Please ensure API statuses are only provided once" 3794 ) 3795 api_status_to_cdk_status[status] = self._get_async_job_status(cdk_status) 3796 return api_status_to_cdk_status 3797 3798 def _get_async_job_status(self, status: str) -> AsyncJobStatus: 3799 match status: 3800 case "running": 3801 return AsyncJobStatus.RUNNING 3802 case "completed": 3803 return AsyncJobStatus.COMPLETED 3804 case "failed": 3805 return AsyncJobStatus.FAILED 3806 case "timeout": 3807 return AsyncJobStatus.TIMED_OUT 3808 case "skipped": 3809 return AsyncJobStatus.SKIPPED 3810 case _: 3811 raise ValueError(f"Unsupported CDK status {status}") 3812 3813 def create_async_retriever( 3814 self, 3815 model: AsyncRetrieverModel, 3816 config: Config, 3817 *, 3818 name: str, 3819 primary_key: Optional[ 3820 Union[str, List[str], List[List[str]]] 3821 ], # this seems to be needed to match create_simple_retriever 3822 stream_slicer: Optional[StreamSlicer], 3823 client_side_incremental_sync: Optional[Dict[str, Any]] = None, 3824 transformations: List[RecordTransformation], 3825 **kwargs: Any, 3826 ) -> AsyncRetriever: 3827 if model.download_target_requester and not model.download_target_extractor: 3828 raise ValueError( 3829 f"`download_target_extractor` required if using a `download_target_requester`" 3830 ) 3831 3832 def _get_download_retriever( 3833 requester: Requester, extractor: RecordExtractor, _decoder: Decoder 3834 ) -> SimpleRetriever: 3835 # We create a record selector for the download retriever 3836 # with no schema normalization and no transformations, neither record filter 3837 # as all this occurs in the record_selector of the AsyncRetriever 3838 record_selector = RecordSelector( 3839 extractor=extractor, 3840 name=name, 3841 record_filter=None, 3842 transformations=[], 3843 schema_normalization=TypeTransformer(TransformConfig.NoTransform), 3844 config=config, 3845 parameters={}, 3846 ) 3847 paginator = ( 3848 self._create_component_from_model( 3849 model=model.download_paginator, 3850 decoder=_decoder, 3851 config=config, 3852 url_base="", 3853 ) 3854 if model.download_paginator 3855 else NoPagination(parameters={}) 3856 ) 3857 3858 return SimpleRetriever( 3859 requester=requester, 3860 record_selector=record_selector, 3861 primary_key=None, 3862 name=name, 3863 paginator=paginator, 3864 config=config, 3865 parameters={}, 3866 log_formatter=self._get_log_formatter(None, name), 3867 ) 3868 3869 def _get_job_timeout() -> datetime.timedelta: 3870 user_defined_timeout: Optional[int] = ( 3871 int( 3872 InterpolatedString.create( 3873 str(model.polling_job_timeout), 3874 parameters={}, 3875 ).eval(config) 3876 ) 3877 if model.polling_job_timeout 3878 else None 3879 ) 3880 3881 # check for user defined timeout during the test read or 15 minutes 3882 test_read_timeout = datetime.timedelta(minutes=user_defined_timeout or 15) 3883 # default value for non-connector builder is 60 minutes. 3884 default_sync_timeout = datetime.timedelta(minutes=user_defined_timeout or 60) 3885 3886 return ( 3887 test_read_timeout if self._emit_connector_builder_messages else default_sync_timeout 3888 ) 3889 3890 decoder = ( 3891 self._create_component_from_model(model=model.decoder, config=config) 3892 if model.decoder 3893 else JsonDecoder(parameters={}) 3894 ) 3895 record_selector = self._create_component_from_model( 3896 model=model.record_selector, 3897 config=config, 3898 decoder=decoder, 3899 name=name, 3900 transformations=transformations, 3901 client_side_incremental_sync=client_side_incremental_sync, 3902 ) 3903 3904 stream_slicer = stream_slicer or SinglePartitionRouter(parameters={}) 3905 if self._should_limit_slices_fetched(): 3906 stream_slicer = cast( 3907 StreamSlicer, 3908 StreamSlicerTestReadDecorator( 3909 wrapped_slicer=stream_slicer, 3910 maximum_number_of_slices=self._limit_slices_fetched or 5, 3911 ), 3912 ) 3913 3914 creation_requester = self._create_component_from_model( 3915 model=model.creation_requester, 3916 decoder=decoder, 3917 config=config, 3918 name=f"job creation - {name}", 3919 ) 3920 polling_requester = self._create_component_from_model( 3921 model=model.polling_requester, 3922 decoder=decoder, 3923 config=config, 3924 name=f"job polling - {name}", 3925 ) 3926 job_download_components_name = f"job download - {name}" 3927 download_decoder = ( 3928 self._create_component_from_model(model=model.download_decoder, config=config) 3929 if model.download_decoder 3930 else JsonDecoder(parameters={}) 3931 ) 3932 download_extractor = ( 3933 self._create_component_from_model( 3934 model=model.download_extractor, 3935 config=config, 3936 decoder=download_decoder, 3937 parameters=model.parameters, 3938 ) 3939 if model.download_extractor 3940 else DpathExtractor( 3941 [], 3942 config=config, 3943 decoder=download_decoder, 3944 parameters=model.parameters or {}, 3945 ) 3946 ) 3947 download_requester = self._create_component_from_model( 3948 model=model.download_requester, 3949 decoder=download_decoder, 3950 config=config, 3951 name=job_download_components_name, 3952 ) 3953 download_retriever = _get_download_retriever( 3954 download_requester, download_extractor, download_decoder 3955 ) 3956 abort_requester = ( 3957 self._create_component_from_model( 3958 model=model.abort_requester, 3959 decoder=decoder, 3960 config=config, 3961 name=f"job abort - {name}", 3962 ) 3963 if model.abort_requester 3964 else None 3965 ) 3966 delete_requester = ( 3967 self._create_component_from_model( 3968 model=model.delete_requester, 3969 decoder=decoder, 3970 config=config, 3971 name=f"job delete - {name}", 3972 ) 3973 if model.delete_requester 3974 else None 3975 ) 3976 download_target_requester = ( 3977 self._create_component_from_model( 3978 model=model.download_target_requester, 3979 decoder=decoder, 3980 config=config, 3981 name=f"job extract_url - {name}", 3982 ) 3983 if model.download_target_requester 3984 else None 3985 ) 3986 status_extractor = self._create_component_from_model( 3987 model=model.status_extractor, decoder=decoder, config=config, name=name 3988 ) 3989 download_target_extractor = ( 3990 self._create_component_from_model( 3991 model=model.download_target_extractor, 3992 decoder=decoder, 3993 config=config, 3994 name=name, 3995 ) 3996 if model.download_target_extractor 3997 else None 3998 ) 3999 4000 job_repository: AsyncJobRepository = AsyncHttpJobRepository( 4001 creation_requester=creation_requester, 4002 polling_requester=polling_requester, 4003 download_retriever=download_retriever, 4004 download_target_requester=download_target_requester, 4005 abort_requester=abort_requester, 4006 delete_requester=delete_requester, 4007 status_extractor=status_extractor, 4008 status_mapping=self._create_async_job_status_mapping(model.status_mapping, config), 4009 download_target_extractor=download_target_extractor, 4010 job_timeout=_get_job_timeout(), 4011 ) 4012 4013 failed_retry_wait_time_in_seconds: Optional[int] = ( 4014 int( 4015 InterpolatedString.create( 4016 str(model.failed_retry_wait_time_in_seconds), 4017 parameters={}, 4018 ).eval(config) 4019 ) 4020 if model.failed_retry_wait_time_in_seconds 4021 else None 4022 ) 4023 4024 async_job_partition_router = AsyncJobPartitionRouter( 4025 job_orchestrator_factory=lambda stream_slices: AsyncJobOrchestrator( 4026 job_repository, 4027 stream_slices, 4028 self._job_tracker, 4029 self._message_repository, 4030 # FIXME work would need to be done here in order to detect if a stream as a parent stream that is bulk 4031 has_bulk_parent=False, 4032 # set the `job_max_retry` to 1 for the `Connector Builder`` use-case. 4033 # `None` == default retry is set to 3 attempts, under the hood. 4034 job_max_retry=1 if self._emit_connector_builder_messages else None, 4035 failed_retry_wait_time_in_seconds=failed_retry_wait_time_in_seconds, 4036 ), 4037 stream_slicer=stream_slicer, 4038 config=config, 4039 parameters=model.parameters or {}, 4040 ) 4041 4042 return AsyncRetriever( 4043 record_selector=record_selector, 4044 stream_slicer=async_job_partition_router, 4045 config=config, 4046 parameters=model.parameters or {}, 4047 ) 4048 4049 def create_spec(self, model: SpecModel, config: Config, **kwargs: Any) -> Spec: 4050 config_migrations = [ 4051 self._create_component_from_model(migration, config) 4052 for migration in ( 4053 model.config_normalization_rules.config_migrations 4054 if ( 4055 model.config_normalization_rules 4056 and model.config_normalization_rules.config_migrations 4057 ) 4058 else [] 4059 ) 4060 ] 4061 config_transformations = [ 4062 self._create_component_from_model(transformation, config) 4063 for transformation in ( 4064 model.config_normalization_rules.transformations 4065 if ( 4066 model.config_normalization_rules 4067 and model.config_normalization_rules.transformations 4068 ) 4069 else [] 4070 ) 4071 ] 4072 config_validations = [ 4073 self._create_component_from_model(validation, config) 4074 for validation in ( 4075 model.config_normalization_rules.validations 4076 if ( 4077 model.config_normalization_rules 4078 and model.config_normalization_rules.validations 4079 ) 4080 else [] 4081 ) 4082 ] 4083 4084 return Spec( 4085 connection_specification=model.connection_specification, 4086 documentation_url=model.documentation_url, 4087 advanced_auth=model.advanced_auth, 4088 parameters={}, 4089 config_migrations=config_migrations, 4090 config_transformations=config_transformations, 4091 config_validations=config_validations, 4092 ) 4093 4094 def create_substream_partition_router( 4095 self, 4096 model: SubstreamPartitionRouterModel, 4097 config: Config, 4098 *, 4099 stream_name: str, 4100 **kwargs: Any, 4101 ) -> SubstreamPartitionRouter: 4102 parent_stream_configs = [] 4103 if model.parent_stream_configs: 4104 parent_stream_configs.extend( 4105 [ 4106 self.create_parent_stream_config_with_substream_wrapper( 4107 model=parent_stream_config, config=config, stream_name=stream_name, **kwargs 4108 ) 4109 for parent_stream_config in model.parent_stream_configs 4110 ] 4111 ) 4112 4113 return SubstreamPartitionRouter( 4114 parent_stream_configs=parent_stream_configs, 4115 parameters=model.parameters or {}, 4116 config=config, 4117 ) 4118 4119 def create_parent_stream_config_with_substream_wrapper( 4120 self, model: ParentStreamConfigModel, config: Config, *, stream_name: str, **kwargs: Any 4121 ) -> Any: 4122 child_state = self._connector_state_manager.get_stream_state(stream_name, None) 4123 4124 parent_state: Optional[Mapping[str, Any]] = ( 4125 child_state if model.incremental_dependency and child_state else None 4126 ) 4127 connector_state_manager = self._instantiate_parent_stream_state_manager( 4128 child_state, config, model, parent_state 4129 ) 4130 4131 substream_factory = ModelToComponentFactory( 4132 connector_state_manager=connector_state_manager, 4133 limit_pages_fetched_per_slice=self._limit_pages_fetched_per_slice, 4134 limit_slices_fetched=self._limit_slices_fetched, 4135 emit_connector_builder_messages=self._emit_connector_builder_messages, 4136 disable_retries=self._disable_retries, 4137 disable_cache=self._disable_cache, 4138 message_repository=StateFilteringMessageRepository( 4139 LogAppenderMessageRepositoryDecorator( 4140 { 4141 "airbyte_cdk": {"stream": {"is_substream": True}}, 4142 "http": {"is_auxiliary": True}, 4143 }, 4144 self._message_repository, 4145 self._evaluate_log_level(self._emit_connector_builder_messages), 4146 ), 4147 ), 4148 api_budget=self._api_budget, 4149 ) 4150 4151 return substream_factory.create_parent_stream_config( 4152 model=model, config=config, stream_name=stream_name, **kwargs 4153 ) 4154 4155 def _instantiate_parent_stream_state_manager( 4156 self, 4157 child_state: MutableMapping[str, Any], 4158 config: Config, 4159 model: ParentStreamConfigModel, 4160 parent_state: Optional[Mapping[str, Any]] = None, 4161 ) -> ConnectorStateManager: 4162 """ 4163 With DefaultStream, the state needs to be provided during __init__ of the cursor as opposed to the 4164 `set_initial_state` flow that existed for the declarative cursors. This state is taken from 4165 self._connector_state_manager.get_stream_state (`self` being a newly created ModelToComponentFactory to account 4166 for the MessageRepository being different). So we need to pass a ConnectorStateManager to the 4167 ModelToComponentFactory that has the parent states. This method populates this if there is a child state and if 4168 incremental_dependency is set. 4169 """ 4170 if model.incremental_dependency and child_state: 4171 parent_stream_name = model.stream.name or "" 4172 extracted_parent_state = ConcurrentPerPartitionCursor.get_parent_state( 4173 child_state, parent_stream_name 4174 ) 4175 4176 if not extracted_parent_state: 4177 extracted_parent_state = ConcurrentPerPartitionCursor.get_global_state( 4178 child_state, parent_stream_name 4179 ) 4180 4181 if not extracted_parent_state and not isinstance(extracted_parent_state, dict): 4182 cursor_values = child_state.values() 4183 if cursor_values and len(cursor_values) == 1: 4184 incremental_sync_model: Union[ 4185 DatetimeBasedCursorModel, 4186 IncrementingCountCursorModel, 4187 ] = ( 4188 model.stream.incremental_sync # type: ignore # if we are there, it is because there is incremental_dependency and therefore there is an incremental_sync on the parent stream 4189 if isinstance(model.stream, DeclarativeStreamModel) 4190 else self._get_state_delegating_stream_model( 4191 model.stream, parent_state=parent_state 4192 ).incremental_sync 4193 ) 4194 cursor_field = InterpolatedString.create( 4195 incremental_sync_model.cursor_field, 4196 parameters=incremental_sync_model.parameters or {}, 4197 ).eval(config) 4198 extracted_parent_state = AirbyteStateMessage( 4199 type=AirbyteStateType.STREAM, 4200 stream=AirbyteStreamState( 4201 stream_descriptor=StreamDescriptor( 4202 name=parent_stream_name, namespace=None 4203 ), 4204 stream_state=AirbyteStateBlob( 4205 {cursor_field: list(cursor_values)[0]} 4206 ), 4207 ), 4208 ) 4209 return ConnectorStateManager([extracted_parent_state] if extracted_parent_state else []) 4210 4211 return ConnectorStateManager([]) 4212 4213 @staticmethod 4214 def create_wait_time_from_header( 4215 model: WaitTimeFromHeaderModel, config: Config, **kwargs: Any 4216 ) -> WaitTimeFromHeaderBackoffStrategy: 4217 return WaitTimeFromHeaderBackoffStrategy( 4218 header=model.header, 4219 parameters=model.parameters or {}, 4220 config=config, 4221 regex=model.regex, 4222 max_waiting_time_in_seconds=model.max_waiting_time_in_seconds 4223 if model.max_waiting_time_in_seconds is not None 4224 else None, 4225 ) 4226 4227 @staticmethod 4228 def create_wait_until_time_from_header( 4229 model: WaitUntilTimeFromHeaderModel, config: Config, **kwargs: Any 4230 ) -> WaitUntilTimeFromHeaderBackoffStrategy: 4231 return WaitUntilTimeFromHeaderBackoffStrategy( 4232 header=model.header, 4233 parameters=model.parameters or {}, 4234 config=config, 4235 min_wait=model.min_wait, 4236 regex=model.regex, 4237 ) 4238 4239 def get_message_repository(self) -> MessageRepository: 4240 return self._message_repository 4241 4242 def _evaluate_log_level(self, emit_connector_builder_messages: bool) -> Level: 4243 return Level.DEBUG if emit_connector_builder_messages else Level.INFO 4244 4245 @staticmethod 4246 def create_components_mapping_definition( 4247 model: ComponentMappingDefinitionModel, config: Config, **kwargs: Any 4248 ) -> ComponentMappingDefinition: 4249 interpolated_value = InterpolatedString.create( 4250 model.value, parameters=model.parameters or {} 4251 ) 4252 field_path = [ 4253 InterpolatedString.create(path, parameters=model.parameters or {}) 4254 for path in model.field_path 4255 ] 4256 return ComponentMappingDefinition( 4257 field_path=field_path, # type: ignore[arg-type] # field_path can be str and InterpolatedString 4258 value=interpolated_value, 4259 value_type=ModelToComponentFactory._json_schema_type_name_to_type(model.value_type), 4260 create_or_update=model.create_or_update, 4261 condition=model.condition, 4262 parameters=model.parameters or {}, 4263 ) 4264 4265 def create_http_components_resolver( 4266 self, model: HttpComponentsResolverModel, config: Config, stream_name: Optional[str] = None 4267 ) -> Any: 4268 retriever = self._create_component_from_model( 4269 model=model.retriever, 4270 config=config, 4271 name=f"{stream_name if stream_name else '__http_components_resolver'}", 4272 primary_key=None, 4273 stream_slicer=self._build_stream_slicer_from_partition_router(model.retriever, config), 4274 transformations=[], 4275 ) 4276 4277 components_mapping = [] 4278 for component_mapping_definition_model in model.components_mapping: 4279 if component_mapping_definition_model.condition: 4280 raise ValueError("`condition` is only supported for `ConfigComponentsResolver`") 4281 components_mapping.append( 4282 self._create_component_from_model( 4283 model=component_mapping_definition_model, 4284 value_type=ModelToComponentFactory._json_schema_type_name_to_type( 4285 component_mapping_definition_model.value_type 4286 ), 4287 config=config, 4288 ) 4289 ) 4290 4291 return HttpComponentsResolver( 4292 retriever=retriever, 4293 stream_slicer=self._build_stream_slicer_from_partition_router(model.retriever, config), 4294 config=config, 4295 components_mapping=components_mapping, 4296 parameters=model.parameters or {}, 4297 ) 4298 4299 @staticmethod 4300 def create_stream_config( 4301 model: StreamConfigModel, config: Config, **kwargs: Any 4302 ) -> StreamConfig: 4303 model_configs_pointer: List[Union[InterpolatedString, str]] = ( 4304 [x for x in model.configs_pointer] if model.configs_pointer else [] 4305 ) 4306 4307 return StreamConfig( 4308 configs_pointer=model_configs_pointer, 4309 default_values=model.default_values, 4310 parameters=model.parameters or {}, 4311 ) 4312 4313 def create_config_components_resolver( 4314 self, 4315 model: ConfigComponentsResolverModel, 4316 config: Config, 4317 ) -> Any: 4318 model_stream_configs = ( 4319 model.stream_config if isinstance(model.stream_config, list) else [model.stream_config] 4320 ) 4321 4322 stream_configs = [ 4323 self._create_component_from_model( 4324 stream_config, config=config, parameters=model.parameters or {} 4325 ) 4326 for stream_config in model_stream_configs 4327 ] 4328 4329 components_mapping = [ 4330 self._create_component_from_model( 4331 model=components_mapping_definition_model, 4332 value_type=ModelToComponentFactory._json_schema_type_name_to_type( 4333 components_mapping_definition_model.value_type 4334 ), 4335 config=config, 4336 parameters=model.parameters, 4337 ) 4338 for components_mapping_definition_model in model.components_mapping 4339 ] 4340 4341 return ConfigComponentsResolver( 4342 stream_configs=stream_configs, 4343 config=config, 4344 components_mapping=components_mapping, 4345 parameters=model.parameters or {}, 4346 ) 4347 4348 def create_parametrized_components_resolver( 4349 self, 4350 model: ParametrizedComponentsResolverModel, 4351 config: Config, 4352 ) -> ParametrizedComponentsResolver: 4353 stream_parameters = StreamParametersDefinition( 4354 list_of_parameters_for_stream=model.stream_parameters.list_of_parameters_for_stream 4355 ) 4356 4357 components_mapping = [] 4358 for components_mapping_definition_model in model.components_mapping: 4359 if components_mapping_definition_model.condition: 4360 raise ValueError("`condition` is only supported for `ConfigComponentsResolver`") 4361 components_mapping.append( 4362 self._create_component_from_model( 4363 model=components_mapping_definition_model, 4364 value_type=ModelToComponentFactory._json_schema_type_name_to_type( 4365 components_mapping_definition_model.value_type 4366 ), 4367 config=config, 4368 ) 4369 ) 4370 return ParametrizedComponentsResolver( 4371 stream_parameters=stream_parameters, 4372 config=config, 4373 components_mapping=components_mapping, 4374 parameters=model.parameters or {}, 4375 ) 4376 4377 _UNSUPPORTED_DECODER_ERROR = ( 4378 "Specified decoder of {decoder_type} is not supported for pagination." 4379 "Please set as `JsonDecoder`, `XmlDecoder`, or a `CompositeRawDecoder` with an inner_parser of `JsonParser` or `GzipParser` instead." 4380 "If using `GzipParser`, please ensure that the lowest level inner_parser is a `JsonParser`." 4381 ) 4382 4383 def _is_supported_decoder_for_pagination(self, decoder: Decoder) -> bool: 4384 if isinstance(decoder, (JsonDecoder, XmlDecoder)): 4385 return True 4386 elif isinstance(decoder, CompositeRawDecoder): 4387 return self._is_supported_parser_for_pagination(decoder.parser) 4388 else: 4389 return False 4390 4391 def _is_supported_parser_for_pagination(self, parser: Parser) -> bool: 4392 if isinstance(parser, JsonParser): 4393 return True 4394 elif isinstance(parser, GzipParser): 4395 return isinstance(parser.inner_parser, JsonParser) 4396 else: 4397 return False 4398 4399 def create_http_api_budget( 4400 self, model: HTTPAPIBudgetModel, config: Config, **kwargs: Any 4401 ) -> HttpAPIBudget: 4402 policies = [ 4403 self._create_component_from_model(model=policy, config=config) 4404 for policy in model.policies 4405 ] 4406 4407 return HttpAPIBudget( 4408 policies=policies, 4409 ratelimit_reset_header=model.ratelimit_reset_header or "ratelimit-reset", 4410 ratelimit_remaining_header=model.ratelimit_remaining_header or "ratelimit-remaining", 4411 status_codes_for_ratelimit_hit=model.status_codes_for_ratelimit_hit or [429], 4412 ) 4413 4414 def create_fixed_window_call_rate_policy( 4415 self, model: FixedWindowCallRatePolicyModel, config: Config, **kwargs: Any 4416 ) -> FixedWindowCallRatePolicy: 4417 matchers = [ 4418 self._create_component_from_model(model=matcher, config=config) 4419 for matcher in model.matchers 4420 ] 4421 4422 # Set the initial reset timestamp to 10 days from now. 4423 # This value will be updated by the first request. 4424 return FixedWindowCallRatePolicy( 4425 next_reset_ts=datetime.datetime.now() + datetime.timedelta(days=10), 4426 period=parse_duration(model.period), 4427 call_limit=model.call_limit, 4428 matchers=matchers, 4429 ) 4430 4431 def create_file_uploader( 4432 self, model: FileUploaderModel, config: Config, **kwargs: Any 4433 ) -> FileUploader: 4434 name = "File Uploader" 4435 requester = self._create_component_from_model( 4436 model=model.requester, 4437 config=config, 4438 name=name, 4439 **kwargs, 4440 ) 4441 download_target_extractor = self._create_component_from_model( 4442 model=model.download_target_extractor, 4443 config=config, 4444 name=name, 4445 **kwargs, 4446 ) 4447 emit_connector_builder_messages = self._emit_connector_builder_messages 4448 file_uploader = DefaultFileUploader( 4449 requester=requester, 4450 download_target_extractor=download_target_extractor, 4451 config=config, 4452 file_writer=NoopFileWriter() 4453 if emit_connector_builder_messages 4454 else LocalFileSystemFileWriter(), 4455 parameters=model.parameters or {}, 4456 filename_extractor=model.filename_extractor if model.filename_extractor else None, 4457 ) 4458 4459 return ( 4460 ConnectorBuilderFileUploader(file_uploader) 4461 if emit_connector_builder_messages 4462 else file_uploader 4463 ) 4464 4465 def create_moving_window_call_rate_policy( 4466 self, model: MovingWindowCallRatePolicyModel, config: Config, **kwargs: Any 4467 ) -> MovingWindowCallRatePolicy: 4468 rates = [ 4469 self._create_component_from_model(model=rate, config=config) for rate in model.rates 4470 ] 4471 matchers = [ 4472 self._create_component_from_model(model=matcher, config=config) 4473 for matcher in model.matchers 4474 ] 4475 return MovingWindowCallRatePolicy( 4476 rates=rates, 4477 matchers=matchers, 4478 ) 4479 4480 def create_unlimited_call_rate_policy( 4481 self, model: UnlimitedCallRatePolicyModel, config: Config, **kwargs: Any 4482 ) -> UnlimitedCallRatePolicy: 4483 matchers = [ 4484 self._create_component_from_model(model=matcher, config=config) 4485 for matcher in model.matchers 4486 ] 4487 4488 return UnlimitedCallRatePolicy( 4489 matchers=matchers, 4490 ) 4491 4492 def create_rate(self, model: RateModel, config: Config, **kwargs: Any) -> Rate: 4493 interpolated_limit = InterpolatedString.create(str(model.limit), parameters={}) 4494 return Rate( 4495 limit=int(interpolated_limit.eval(config=config)), 4496 interval=parse_duration(model.interval), 4497 ) 4498 4499 def create_http_request_matcher( 4500 self, model: HttpRequestRegexMatcherModel, config: Config, **kwargs: Any 4501 ) -> HttpRequestRegexMatcher: 4502 weight = model.weight 4503 if weight is not None: 4504 if isinstance(weight, str): 4505 weight = int(InterpolatedString.create(weight, parameters={}).eval(config)) 4506 else: 4507 weight = int(weight) 4508 if weight < 1: 4509 raise ValueError(f"weight must be >= 1, got {weight}") 4510 return HttpRequestRegexMatcher( 4511 method=model.method, 4512 url_base=model.url_base, 4513 url_path_pattern=model.url_path_pattern, 4514 params=model.params, 4515 headers=model.headers, 4516 weight=weight, 4517 ) 4518 4519 def set_api_budget(self, component_definition: ComponentDefinition, config: Config) -> None: 4520 self._api_budget = self.create_component( 4521 model_type=HTTPAPIBudgetModel, component_definition=component_definition, config=config 4522 ) 4523 4524 def create_grouping_partition_router( 4525 self, 4526 model: GroupingPartitionRouterModel, 4527 config: Config, 4528 *, 4529 stream_name: str, 4530 **kwargs: Any, 4531 ) -> GroupingPartitionRouter: 4532 underlying_router = self._create_component_from_model( 4533 model=model.underlying_partition_router, 4534 config=config, 4535 stream_name=stream_name, 4536 **kwargs, 4537 ) 4538 if model.group_size < 1: 4539 raise ValueError(f"Group size must be greater than 0, got {model.group_size}") 4540 4541 # Request options in underlying partition routers are not supported for GroupingPartitionRouter 4542 # because they are specific to individual partitions and cannot be aggregated or handled 4543 # when grouping, potentially leading to incorrect API calls. Any request customization 4544 # should be managed at the stream level through the requester's configuration. 4545 if isinstance(underlying_router, SubstreamPartitionRouter): 4546 if any( 4547 parent_config.request_option 4548 for parent_config in underlying_router.parent_stream_configs 4549 ): 4550 raise ValueError("Request options are not supported for GroupingPartitionRouter.") 4551 4552 if isinstance(underlying_router, ListPartitionRouter): 4553 if underlying_router.request_option: 4554 raise ValueError("Request options are not supported for GroupingPartitionRouter.") 4555 4556 return GroupingPartitionRouter( 4557 group_size=model.group_size, 4558 underlying_partition_router=underlying_router, 4559 deduplicate=model.deduplicate if model.deduplicate is not None else True, 4560 config=config, 4561 ) 4562 4563 def _ensure_query_properties_to_model( 4564 self, requester: Union[HttpRequesterModel, CustomRequesterModel] 4565 ) -> None: 4566 """ 4567 For some reason, it seems like CustomRequesterModel request_parameters stays as dictionaries which means that 4568 the other conditions relying on it being QueryPropertiesModel instead of a dict fail. Here, we migrate them to 4569 proper model. 4570 """ 4571 if not hasattr(requester, "request_parameters"): 4572 return 4573 4574 request_parameters = requester.request_parameters 4575 if request_parameters and isinstance(request_parameters, Dict): 4576 for request_parameter_key in request_parameters.keys(): 4577 request_parameter = request_parameters[request_parameter_key] 4578 if ( 4579 isinstance(request_parameter, Dict) 4580 and request_parameter.get("type") == "QueryProperties" 4581 ): 4582 request_parameters[request_parameter_key] = QueryPropertiesModel.parse_obj( 4583 request_parameter 4584 ) 4585 4586 def _get_catalog_defined_cursor_field( 4587 self, stream_name: str, allow_catalog_defined_cursor_field: bool 4588 ) -> Optional[CursorField]: 4589 if not allow_catalog_defined_cursor_field: 4590 return None 4591 4592 configured_stream = self._stream_name_to_configured_stream.get(stream_name) 4593 4594 # Depending on the operation is being performed, there may not be a configured stream yet. In this 4595 # case we return None which will then use the default cursor field defined on the cursor model. 4596 # We also treat cursor_field: [""] (list with empty string) as no cursor field, since this can 4597 # occur when the platform serializes "no cursor configured" streams incorrectly. 4598 if ( 4599 not configured_stream 4600 or not configured_stream.cursor_field 4601 or not configured_stream.cursor_field[0] 4602 ): 4603 return None 4604 elif len(configured_stream.cursor_field) > 1: 4605 raise ValueError( 4606 f"The `{stream_name}` stream does not support nested cursor_field. Please specify only a single cursor_field for the stream in the configured catalog." 4607 ) 4608 else: 4609 return CursorField( 4610 cursor_field_key=configured_stream.cursor_field[0], 4611 supports_catalog_defined_cursor_field=allow_catalog_defined_cursor_field, 4612 )
ModelToComponentFactory( limit_pages_fetched_per_slice: Optional[int] = None, limit_slices_fetched: Optional[int] = None, emit_connector_builder_messages: bool = False, disable_retries: bool = False, disable_cache: bool = False, message_repository: Optional[airbyte_cdk.MessageRepository] = None, connector_state_manager: Optional[airbyte_cdk.ConnectorStateManager] = None, max_concurrent_async_job_count: Optional[int] = None, configured_catalog: Optional[airbyte_protocol_dataclasses.models.airbyte_protocol.ConfiguredAirbyteCatalog] = None, api_budget: Optional[airbyte_cdk.sources.streams.call_rate.APIBudget] = None)
692 def __init__( 693 self, 694 limit_pages_fetched_per_slice: Optional[int] = None, 695 limit_slices_fetched: Optional[int] = None, 696 emit_connector_builder_messages: bool = False, 697 disable_retries: bool = False, 698 disable_cache: bool = False, 699 message_repository: Optional[MessageRepository] = None, 700 connector_state_manager: Optional[ConnectorStateManager] = None, 701 max_concurrent_async_job_count: Optional[int] = None, 702 configured_catalog: Optional[ConfiguredAirbyteCatalog] = None, 703 api_budget: Optional[APIBudget] = None, 704 ): 705 self._init_mappings() 706 self._limit_pages_fetched_per_slice = limit_pages_fetched_per_slice 707 self._limit_slices_fetched = limit_slices_fetched 708 self._emit_connector_builder_messages = emit_connector_builder_messages 709 self._disable_retries = disable_retries 710 self._disable_cache = disable_cache 711 self._message_repository = message_repository or InMemoryMessageRepository( 712 self._evaluate_log_level(emit_connector_builder_messages) 713 ) 714 self._stream_name_to_configured_stream = self._create_stream_name_to_configured_stream( 715 configured_catalog 716 ) 717 self._connector_state_manager = connector_state_manager or ConnectorStateManager() 718 self._api_budget: Optional[Union[APIBudget]] = api_budget 719 self._job_tracker: JobTracker = JobTracker(max_concurrent_async_job_count or 1) 720 # placeholder for deprecation warnings 721 self._collected_deprecation_logs: List[ConnectorBuilderLogMessage] = []
def
create_component( self, model_type: Type[pydantic.v1.main.BaseModel], component_definition: Mapping[str, Any], config: Mapping[str, Any], **kwargs: Any) -> Any:
846 def create_component( 847 self, 848 model_type: Type[BaseModel], 849 component_definition: ComponentDefinition, 850 config: Config, 851 **kwargs: Any, 852 ) -> Any: 853 """ 854 Takes a given Pydantic model type and Mapping representing a component definition and creates a declarative component and 855 subcomponents which will be used at runtime. This is done by first parsing the mapping into a Pydantic model and then creating 856 creating declarative components from that model. 857 858 :param model_type: The type of declarative component that is being initialized 859 :param component_definition: The mapping that represents a declarative component 860 :param config: The connector config that is provided by the customer 861 :return: The declarative component to be used at runtime 862 """ 863 864 component_type = component_definition.get("type") 865 if component_definition.get("type") != model_type.__name__: 866 raise ValueError( 867 f"Expected manifest component of type {model_type.__name__}, but received {component_type} instead" 868 ) 869 870 declarative_component_model = model_type.parse_obj(component_definition) 871 872 if not isinstance(declarative_component_model, model_type): 873 raise ValueError( 874 f"Expected {model_type.__name__} component, but received {declarative_component_model.__class__.__name__}" 875 ) 876 877 return self._create_component_from_model( 878 model=declarative_component_model, config=config, **kwargs 879 )
Takes a given Pydantic model type and Mapping representing a component definition and creates a declarative component and subcomponents which will be used at runtime. This is done by first parsing the mapping into a Pydantic model and then creating creating declarative components from that model.
Parameters
- model_type: The type of declarative component that is being initialized
- component_definition: The mapping that represents a declarative component
- config: The connector config that is provided by the customer
Returns
The declarative component to be used at runtime
896 def get_model_deprecations(self) -> List[ConnectorBuilderLogMessage]: 897 """ 898 Returns the deprecation warnings that were collected during the creation of components. 899 """ 900 return self._collected_deprecation_logs
Returns the deprecation warnings that were collected during the creation of components.
def
create_config_migration( self, model: airbyte_cdk.sources.declarative.models.declarative_component_schema.ConfigMigration, config: Mapping[str, Any]) -> airbyte_cdk.sources.declarative.spec.ConfigMigration:
917 def create_config_migration( 918 self, model: ConfigMigrationModel, config: Config 919 ) -> ConfigMigration: 920 transformations: List[ConfigTransformation] = [ 921 self._create_component_from_model(transformation, config) 922 for transformation in model.transformations 923 ] 924 925 return ConfigMigration( 926 description=model.description, 927 transformations=transformations, 928 )
def
create_config_add_fields( self, model: airbyte_cdk.sources.declarative.models.declarative_component_schema.ConfigAddFields, config: Mapping[str, Any], **kwargs: Any) -> airbyte_cdk.sources.declarative.transformations.config_transformations.ConfigAddFields:
930 def create_config_add_fields( 931 self, model: ConfigAddFieldsModel, config: Config, **kwargs: Any 932 ) -> ConfigAddFields: 933 fields = [self._create_component_from_model(field, config) for field in model.fields] 934 return ConfigAddFields( 935 fields=fields, 936 condition=model.condition or "", 937 )
@staticmethod
def
create_config_remove_fields( model: airbyte_cdk.sources.declarative.models.declarative_component_schema.ConfigRemoveFields, config: Mapping[str, Any], **kwargs: Any) -> airbyte_cdk.sources.declarative.transformations.config_transformations.ConfigRemoveFields:
@staticmethod
def
create_config_remap_field( model: airbyte_cdk.sources.declarative.models.declarative_component_schema.ConfigRemapField, config: Mapping[str, Any], **kwargs: Any) -> airbyte_cdk.sources.declarative.transformations.config_transformations.ConfigRemapField:
def
create_dpath_validator( self, model: airbyte_cdk.sources.declarative.models.declarative_component_schema.DpathValidator, config: Mapping[str, Any]) -> airbyte_cdk.sources.declarative.validators.DpathValidator:
def
create_predicate_validator( self, model: airbyte_cdk.sources.declarative.models.declarative_component_schema.PredicateValidator, config: Mapping[str, Any]) -> airbyte_cdk.sources.declarative.validators.PredicateValidator:
@staticmethod
def
create_validate_adheres_to_schema( model: airbyte_cdk.sources.declarative.models.declarative_component_schema.ValidateAdheresToSchema, config: Mapping[str, Any], **kwargs: Any) -> airbyte_cdk.sources.declarative.validators.ValidateAdheresToSchema:
@staticmethod
def
create_added_field_definition( model: airbyte_cdk.sources.declarative.models.declarative_component_schema.AddedFieldDefinition, config: Mapping[str, Any], **kwargs: Any) -> airbyte_cdk.AddedFieldDefinition:
986 @staticmethod 987 def create_added_field_definition( 988 model: AddedFieldDefinitionModel, config: Config, **kwargs: Any 989 ) -> AddedFieldDefinition: 990 interpolated_value = InterpolatedString.create( 991 model.value, parameters=model.parameters or {} 992 ) 993 return AddedFieldDefinition( 994 path=model.path, 995 value=interpolated_value, 996 value_type=ModelToComponentFactory._json_schema_type_name_to_type(model.value_type), 997 parameters=model.parameters or {}, 998 )
def
create_add_fields( self, model: airbyte_cdk.sources.declarative.models.declarative_component_schema.AddFields, config: Mapping[str, Any], **kwargs: Any) -> airbyte_cdk.AddFields:
1000 def create_add_fields(self, model: AddFieldsModel, config: Config, **kwargs: Any) -> AddFields: 1001 added_field_definitions = [ 1002 self._create_component_from_model( 1003 model=added_field_definition_model, 1004 value_type=ModelToComponentFactory._json_schema_type_name_to_type( 1005 added_field_definition_model.value_type 1006 ), 1007 config=config, 1008 ) 1009 for added_field_definition_model in model.fields 1010 ] 1011 return AddFields( 1012 fields=added_field_definitions, 1013 condition=model.condition or "", 1014 parameters=model.parameters or {}, 1015 )
def
create_keys_to_lower_transformation( self, model: airbyte_cdk.sources.declarative.models.declarative_component_schema.KeysToLower, config: Mapping[str, Any], **kwargs: Any) -> airbyte_cdk.sources.declarative.transformations.keys_to_lower_transformation.KeysToLowerTransformation:
def
create_keys_to_snake_transformation( self, model: airbyte_cdk.sources.declarative.models.declarative_component_schema.KeysToSnakeCase, config: Mapping[str, Any], **kwargs: Any) -> airbyte_cdk.sources.declarative.transformations.keys_to_snake_transformation.KeysToSnakeCaseTransformation:
def
create_keys_replace_transformation( self, model: airbyte_cdk.sources.declarative.models.declarative_component_schema.KeysReplace, config: Mapping[str, Any], **kwargs: Any) -> airbyte_cdk.sources.declarative.transformations.keys_replace_transformation.KeysReplaceTransformation:
def
create_flatten_fields( self, model: airbyte_cdk.sources.declarative.models.declarative_component_schema.FlattenFields, config: Mapping[str, Any], **kwargs: Any) -> airbyte_cdk.sources.declarative.transformations.flatten_fields.FlattenFields:
def
create_dpath_flatten_fields( self, model: airbyte_cdk.sources.declarative.models.declarative_component_schema.DpathFlattenFields, config: Mapping[str, Any], **kwargs: Any) -> airbyte_cdk.sources.declarative.transformations.dpath_flatten_fields.DpathFlattenFields:
1041 def create_dpath_flatten_fields( 1042 self, model: DpathFlattenFieldsModel, config: Config, **kwargs: Any 1043 ) -> DpathFlattenFields: 1044 model_field_path: List[Union[InterpolatedString, str]] = [x for x in model.field_path] 1045 key_transformation = ( 1046 KeyTransformation( 1047 config=config, 1048 prefix=model.key_transformation.prefix, 1049 suffix=model.key_transformation.suffix, 1050 parameters=model.parameters or {}, 1051 ) 1052 if model.key_transformation is not None 1053 else None 1054 ) 1055 return DpathFlattenFields( 1056 config=config, 1057 field_path=model_field_path, 1058 delete_origin_value=model.delete_origin_value 1059 if model.delete_origin_value is not None 1060 else False, 1061 replace_record=model.replace_record if model.replace_record is not None else False, 1062 key_transformation=key_transformation, 1063 parameters=model.parameters or {}, 1064 )
def
create_api_key_authenticator( self, model: airbyte_cdk.sources.declarative.models.declarative_component_schema.ApiKeyAuthenticator, config: Mapping[str, Any], token_provider: Optional[airbyte_cdk.sources.declarative.auth.token_provider.TokenProvider] = None, **kwargs: Any) -> airbyte_cdk.ApiKeyAuthenticator:
1078 def create_api_key_authenticator( 1079 self, 1080 model: ApiKeyAuthenticatorModel, 1081 config: Config, 1082 token_provider: Optional[TokenProvider] = None, 1083 **kwargs: Any, 1084 ) -> ApiKeyAuthenticator: 1085 if model.inject_into is None and model.header is None: 1086 raise ValueError( 1087 "Expected either inject_into or header to be set for ApiKeyAuthenticator" 1088 ) 1089 1090 if model.inject_into is not None and model.header is not None: 1091 raise ValueError( 1092 "inject_into and header cannot be set both for ApiKeyAuthenticator - remove the deprecated header option" 1093 ) 1094 1095 if token_provider is not None and model.api_token != "": 1096 raise ValueError( 1097 "If token_provider is set, api_token is ignored and has to be set to empty string." 1098 ) 1099 1100 request_option = ( 1101 self._create_component_from_model( 1102 model.inject_into, config, parameters=model.parameters or {} 1103 ) 1104 if model.inject_into 1105 else RequestOption( 1106 inject_into=RequestOptionType.header, 1107 field_name=model.header or "", 1108 parameters=model.parameters or {}, 1109 ) 1110 ) 1111 1112 return ApiKeyAuthenticator( 1113 token_provider=( 1114 token_provider 1115 if token_provider is not None 1116 else InterpolatedStringTokenProvider( 1117 api_token=model.api_token or "", 1118 config=config, 1119 parameters=model.parameters or {}, 1120 ) 1121 ), 1122 request_option=request_option, 1123 config=config, 1124 parameters=model.parameters or {}, 1125 )
def
create_legacy_to_per_partition_state_migration( self, model: airbyte_cdk.sources.declarative.models.declarative_component_schema.LegacyToPerPartitionStateMigration, config: Mapping[str, Any], declarative_stream: airbyte_cdk.sources.declarative.models.declarative_component_schema.DeclarativeStream) -> airbyte_cdk.LegacyToPerPartitionStateMigration:
1127 def create_legacy_to_per_partition_state_migration( 1128 self, 1129 model: LegacyToPerPartitionStateMigrationModel, 1130 config: Mapping[str, Any], 1131 declarative_stream: DeclarativeStreamModel, 1132 ) -> LegacyToPerPartitionStateMigration: 1133 retriever = declarative_stream.retriever 1134 if not isinstance(retriever, (SimpleRetrieverModel, AsyncRetrieverModel)): 1135 raise ValueError( 1136 f"LegacyToPerPartitionStateMigrations can only be applied on a DeclarativeStream with a SimpleRetriever or AsyncRetriever. Got {type(retriever)}" 1137 ) 1138 partition_router = retriever.partition_router 1139 if not isinstance( 1140 partition_router, (SubstreamPartitionRouterModel, CustomPartitionRouterModel) 1141 ): 1142 raise ValueError( 1143 f"LegacyToPerPartitionStateMigrations can only be applied on a SimpleRetriever with a Substream partition router. Got {type(partition_router)}" 1144 ) 1145 if not hasattr(partition_router, "parent_stream_configs"): 1146 raise ValueError( 1147 "LegacyToPerPartitionStateMigrations can only be applied with a parent stream configuration." 1148 ) 1149 1150 if not hasattr(declarative_stream, "incremental_sync"): 1151 raise ValueError( 1152 "LegacyToPerPartitionStateMigrations can only be applied with an incremental_sync configuration." 1153 ) 1154 1155 return LegacyToPerPartitionStateMigration( 1156 partition_router, # type: ignore # was already checked above 1157 declarative_stream.incremental_sync, # type: ignore # was already checked. Migration can be applied only to incremental streams. 1158 config, 1159 declarative_stream.parameters, # type: ignore # different type is expected here Mapping[str, Any], got Dict[str, Any] 1160 )
def
create_session_token_authenticator( self, model: airbyte_cdk.sources.declarative.models.declarative_component_schema.SessionTokenAuthenticator, config: Mapping[str, Any], name: str, **kwargs: Any) -> Union[airbyte_cdk.ApiKeyAuthenticator, airbyte_cdk.BearerAuthenticator]:
1162 def create_session_token_authenticator( 1163 self, model: SessionTokenAuthenticatorModel, config: Config, name: str, **kwargs: Any 1164 ) -> Union[ApiKeyAuthenticator, BearerAuthenticator]: 1165 decoder = ( 1166 self._create_component_from_model(model=model.decoder, config=config) 1167 if model.decoder 1168 else JsonDecoder(parameters={}) 1169 ) 1170 login_requester = self._create_component_from_model( 1171 model=model.login_requester, 1172 config=config, 1173 name=f"{name}_login_requester", 1174 decoder=decoder, 1175 ) 1176 token_provider = SessionTokenProvider( 1177 login_requester=login_requester, 1178 session_token_path=model.session_token_path, 1179 expiration_duration=parse_duration(model.expiration_duration) 1180 if model.expiration_duration 1181 else None, 1182 parameters=model.parameters or {}, 1183 message_repository=self._message_repository, 1184 decoder=decoder, 1185 ) 1186 if model.request_authentication.type == "Bearer": 1187 return ModelToComponentFactory.create_bearer_authenticator( 1188 BearerAuthenticatorModel(type="BearerAuthenticator", api_token=""), # type: ignore # $parameters has a default value 1189 config, 1190 token_provider=token_provider, 1191 ) 1192 else: 1193 # Get the api_token template if specified, default to just the session token 1194 api_token_template = ( 1195 getattr(model.request_authentication, "api_token", None) or "{{ session_token }}" 1196 ) 1197 final_token_provider: TokenProvider = InterpolatedSessionTokenProvider( 1198 config=config, 1199 api_token=api_token_template, 1200 session_token_provider=token_provider, 1201 parameters=model.parameters or {}, 1202 ) 1203 return self.create_api_key_authenticator( 1204 ApiKeyAuthenticatorModel( 1205 type="ApiKeyAuthenticator", 1206 api_token="", 1207 inject_into=model.request_authentication.inject_into, 1208 ), # type: ignore # $parameters and headers default to None 1209 config=config, 1210 token_provider=final_token_provider, 1211 )
@staticmethod
def
create_basic_http_authenticator( model: airbyte_cdk.sources.declarative.models.declarative_component_schema.BasicHttpAuthenticator, config: Mapping[str, Any], **kwargs: Any) -> airbyte_cdk.BasicHttpAuthenticator:
1213 @staticmethod 1214 def create_basic_http_authenticator( 1215 model: BasicHttpAuthenticatorModel, config: Config, **kwargs: Any 1216 ) -> BasicHttpAuthenticator: 1217 return BasicHttpAuthenticator( 1218 password=model.password or "", 1219 username=model.username, 1220 config=config, 1221 parameters=model.parameters or {}, 1222 )
@staticmethod
def
create_bearer_authenticator( model: airbyte_cdk.sources.declarative.models.declarative_component_schema.BearerAuthenticator, config: Mapping[str, Any], token_provider: Optional[airbyte_cdk.sources.declarative.auth.token_provider.TokenProvider] = None, **kwargs: Any) -> airbyte_cdk.BearerAuthenticator:
1224 @staticmethod 1225 def create_bearer_authenticator( 1226 model: BearerAuthenticatorModel, 1227 config: Config, 1228 token_provider: Optional[TokenProvider] = None, 1229 **kwargs: Any, 1230 ) -> BearerAuthenticator: 1231 if token_provider is not None and model.api_token != "": 1232 raise ValueError( 1233 "If token_provider is set, api_token is ignored and has to be set to empty string." 1234 ) 1235 return BearerAuthenticator( 1236 token_provider=( 1237 token_provider 1238 if token_provider is not None 1239 else InterpolatedStringTokenProvider( 1240 api_token=model.api_token or "", 1241 config=config, 1242 parameters=model.parameters or {}, 1243 ) 1244 ), 1245 config=config, 1246 parameters=model.parameters or {}, 1247 )
@staticmethod
def
create_dynamic_stream_check_config( model: airbyte_cdk.sources.declarative.models.declarative_component_schema.DynamicStreamCheckConfig, config: Mapping[str, Any], **kwargs: Any) -> airbyte_cdk.sources.declarative.checks.DynamicStreamCheckConfig:
1249 @staticmethod 1250 def create_dynamic_stream_check_config( 1251 model: DynamicStreamCheckConfigModel, config: Config, **kwargs: Any 1252 ) -> DynamicStreamCheckConfig: 1253 return DynamicStreamCheckConfig( 1254 dynamic_stream_name=model.dynamic_stream_name, 1255 stream_count=model.stream_count, 1256 )
def
create_check_stream( self, model: airbyte_cdk.sources.declarative.models.declarative_component_schema.CheckStream, config: Mapping[str, Any], **kwargs: Any) -> airbyte_cdk.sources.declarative.checks.CheckStream:
1258 def create_check_stream( 1259 self, model: CheckStreamModel, config: Config, **kwargs: Any 1260 ) -> CheckStream: 1261 if model.dynamic_streams_check_configs is None and model.stream_names is None: 1262 raise ValueError( 1263 "Expected either stream_names or dynamic_streams_check_configs to be set for CheckStream" 1264 ) 1265 1266 dynamic_streams_check_configs = ( 1267 [ 1268 self._create_component_from_model(model=dynamic_stream_check_config, config=config) 1269 for dynamic_stream_check_config in model.dynamic_streams_check_configs 1270 ] 1271 if model.dynamic_streams_check_configs 1272 else [] 1273 ) 1274 1275 return CheckStream( 1276 stream_names=model.stream_names or [], 1277 dynamic_streams_check_configs=dynamic_streams_check_configs, 1278 parameters={}, 1279 )
@staticmethod
def
create_check_dynamic_stream( model: airbyte_cdk.sources.declarative.models.declarative_component_schema.CheckDynamicStream, config: Mapping[str, Any], **kwargs: Any) -> airbyte_cdk.sources.declarative.checks.CheckDynamicStream:
1281 @staticmethod 1282 def create_check_dynamic_stream( 1283 model: CheckDynamicStreamModel, config: Config, **kwargs: Any 1284 ) -> CheckDynamicStream: 1285 assert model.use_check_availability is not None # for mypy 1286 1287 use_check_availability = model.use_check_availability 1288 1289 return CheckDynamicStream( 1290 stream_count=model.stream_count, 1291 use_check_availability=use_check_availability, 1292 parameters={}, 1293 )
def
create_composite_error_handler( self, model: airbyte_cdk.sources.declarative.models.declarative_component_schema.CompositeErrorHandler, config: Mapping[str, Any], **kwargs: Any) -> airbyte_cdk.sources.declarative.requesters.error_handlers.CompositeErrorHandler:
1295 def create_composite_error_handler( 1296 self, model: CompositeErrorHandlerModel, config: Config, **kwargs: Any 1297 ) -> CompositeErrorHandler: 1298 error_handlers = [ 1299 self._create_component_from_model(model=error_handler_model, config=config) 1300 for error_handler_model in model.error_handlers 1301 ] 1302 return CompositeErrorHandler( 1303 error_handlers=error_handlers, parameters=model.parameters or {} 1304 )
@staticmethod
def
create_concurrency_level( model: airbyte_cdk.sources.declarative.models.declarative_component_schema.ConcurrencyLevel, config: Mapping[str, Any], **kwargs: Any) -> airbyte_cdk.sources.declarative.concurrency_level.ConcurrencyLevel:
1306 @staticmethod 1307 def create_concurrency_level( 1308 model: ConcurrencyLevelModel, config: Config, **kwargs: Any 1309 ) -> ConcurrencyLevel: 1310 return ConcurrencyLevel( 1311 default_concurrency=model.default_concurrency, 1312 max_concurrency=model.max_concurrency, 1313 config=config, 1314 parameters={}, 1315 )
@staticmethod
def
apply_stream_state_migrations( stream_state_migrations: Optional[List[Any]], stream_state: MutableMapping[str, Any]) -> MutableMapping[str, Any]:
1317 @staticmethod 1318 def apply_stream_state_migrations( 1319 stream_state_migrations: List[Any] | None, stream_state: MutableMapping[str, Any] 1320 ) -> MutableMapping[str, Any]: 1321 if stream_state_migrations: 1322 for state_migration in stream_state_migrations: 1323 if state_migration.should_migrate(stream_state): 1324 # The state variable is expected to be mutable but the migrate method returns an immutable mapping. 1325 stream_state = dict(state_migration.migrate(stream_state)) 1326 return stream_state
def
create_concurrent_cursor_from_datetime_based_cursor( self, model_type: Type[pydantic.v1.main.BaseModel], component_definition: Mapping[str, Any], stream_name: str, stream_namespace: Optional[str], stream_state: MutableMapping[str, Any], config: Mapping[str, Any], message_repository: Optional[airbyte_cdk.MessageRepository] = None, runtime_lookback_window: Optional[datetime.timedelta] = None, **kwargs: Any) -> airbyte_cdk.ConcurrentCursor:
1328 def create_concurrent_cursor_from_datetime_based_cursor( 1329 self, 1330 model_type: Type[BaseModel], 1331 component_definition: ComponentDefinition, 1332 stream_name: str, 1333 stream_namespace: Optional[str], 1334 stream_state: MutableMapping[str, Any], 1335 config: Config, 1336 message_repository: Optional[MessageRepository] = None, 1337 runtime_lookback_window: Optional[datetime.timedelta] = None, 1338 **kwargs: Any, 1339 ) -> ConcurrentCursor: 1340 component_type = component_definition.get("type") 1341 if component_definition.get("type") != model_type.__name__: 1342 raise ValueError( 1343 f"Expected manifest component of type {model_type.__name__}, but received {component_type} instead" 1344 ) 1345 1346 # FIXME the interfaces of the concurrent cursor are kind of annoying as they take a `ComponentDefinition` instead of the actual model. This was done because the ConcurrentDeclarativeSource didn't have access to the models [here for example](https://github.com/airbytehq/airbyte-python-cdk/blob/f525803b3fec9329e4cc8478996a92bf884bfde9/airbyte_cdk/sources/declarative/concurrent_declarative_source.py#L354C54-L354C91). So now we have two cases: 1347 # * The ComponentDefinition comes from model.__dict__ in which case we have `parameters` 1348 # * The ComponentDefinition comes from the manifest as a dict in which case we have `$parameters` 1349 # We should change those interfaces to use the model once we clean up the code in CDS at which point the parameter propagation should happen as part of the ModelToComponentFactory. 1350 if "$parameters" not in component_definition and "parameters" in component_definition: 1351 component_definition["$parameters"] = component_definition.get("parameters") # type: ignore # This is a dict 1352 datetime_based_cursor_model = model_type.parse_obj(component_definition) 1353 1354 if not isinstance(datetime_based_cursor_model, DatetimeBasedCursorModel): 1355 raise ValueError( 1356 f"Expected {model_type.__name__} component, but received {datetime_based_cursor_model.__class__.__name__}" 1357 ) 1358 1359 model_parameters = datetime_based_cursor_model.parameters or {} 1360 1361 cursor_field = self._get_catalog_defined_cursor_field( 1362 stream_name=stream_name, 1363 allow_catalog_defined_cursor_field=datetime_based_cursor_model.allow_catalog_defined_cursor_field 1364 or False, 1365 ) 1366 1367 if not cursor_field: 1368 interpolated_cursor_field = InterpolatedString.create( 1369 datetime_based_cursor_model.cursor_field, 1370 parameters=model_parameters, 1371 ) 1372 cursor_field = CursorField( 1373 cursor_field_key=interpolated_cursor_field.eval(config=config), 1374 supports_catalog_defined_cursor_field=datetime_based_cursor_model.allow_catalog_defined_cursor_field 1375 or False, 1376 ) 1377 1378 interpolated_partition_field_start = InterpolatedString.create( 1379 datetime_based_cursor_model.partition_field_start or "start_time", 1380 parameters=model_parameters, 1381 ) 1382 interpolated_partition_field_end = InterpolatedString.create( 1383 datetime_based_cursor_model.partition_field_end or "end_time", 1384 parameters=model_parameters, 1385 ) 1386 1387 slice_boundary_fields = ( 1388 interpolated_partition_field_start.eval(config=config), 1389 interpolated_partition_field_end.eval(config=config), 1390 ) 1391 1392 datetime_format = datetime_based_cursor_model.datetime_format 1393 1394 cursor_granularity = ( 1395 parse_duration(datetime_based_cursor_model.cursor_granularity) 1396 if datetime_based_cursor_model.cursor_granularity 1397 else None 1398 ) 1399 1400 lookback_window = None 1401 interpolated_lookback_window = ( 1402 InterpolatedString.create( 1403 datetime_based_cursor_model.lookback_window, 1404 parameters=model_parameters, 1405 ) 1406 if datetime_based_cursor_model.lookback_window 1407 else None 1408 ) 1409 if interpolated_lookback_window: 1410 evaluated_lookback_window = interpolated_lookback_window.eval(config=config) 1411 if evaluated_lookback_window: 1412 lookback_window = parse_duration(evaluated_lookback_window) 1413 1414 connector_state_converter: DateTimeStreamStateConverter 1415 connector_state_converter = CustomFormatConcurrentStreamStateConverter( 1416 datetime_format=datetime_format, 1417 input_datetime_formats=datetime_based_cursor_model.cursor_datetime_formats, 1418 is_sequential_state=True, # ConcurrentPerPartitionCursor only works with sequential state 1419 cursor_granularity=cursor_granularity, 1420 ) 1421 1422 # Adjusts the stream state by applying the runtime lookback window. 1423 # This is used to ensure correct state handling in case of failed partitions. 1424 stream_state_value = stream_state.get(cursor_field.cursor_field_key) 1425 if runtime_lookback_window and stream_state_value: 1426 new_stream_state = ( 1427 connector_state_converter.parse_timestamp(stream_state_value) 1428 - runtime_lookback_window 1429 ) 1430 stream_state[cursor_field.cursor_field_key] = connector_state_converter.output_format( 1431 new_stream_state 1432 ) 1433 1434 start_date_runtime_value: Union[InterpolatedString, str, MinMaxDatetime] 1435 if isinstance(datetime_based_cursor_model.start_datetime, MinMaxDatetimeModel): 1436 start_date_runtime_value = self.create_min_max_datetime( 1437 model=datetime_based_cursor_model.start_datetime, config=config 1438 ) 1439 else: 1440 start_date_runtime_value = datetime_based_cursor_model.start_datetime 1441 1442 end_date_runtime_value: Optional[Union[InterpolatedString, str, MinMaxDatetime]] 1443 if isinstance(datetime_based_cursor_model.end_datetime, MinMaxDatetimeModel): 1444 end_date_runtime_value = self.create_min_max_datetime( 1445 model=datetime_based_cursor_model.end_datetime, config=config 1446 ) 1447 else: 1448 end_date_runtime_value = datetime_based_cursor_model.end_datetime 1449 1450 interpolated_start_date = MinMaxDatetime.create( 1451 interpolated_string_or_min_max_datetime=start_date_runtime_value, 1452 parameters=datetime_based_cursor_model.parameters, 1453 ) 1454 interpolated_end_date = ( 1455 None 1456 if not end_date_runtime_value 1457 else MinMaxDatetime.create( 1458 end_date_runtime_value, datetime_based_cursor_model.parameters 1459 ) 1460 ) 1461 1462 # If datetime format is not specified then start/end datetime should inherit it from the stream slicer 1463 if not interpolated_start_date.datetime_format: 1464 interpolated_start_date.datetime_format = datetime_format 1465 if interpolated_end_date and not interpolated_end_date.datetime_format: 1466 interpolated_end_date.datetime_format = datetime_format 1467 1468 start_date = interpolated_start_date.get_datetime(config=config) 1469 end_date_provider = ( 1470 partial(interpolated_end_date.get_datetime, config) 1471 if interpolated_end_date 1472 else connector_state_converter.get_end_provider() 1473 ) 1474 1475 if ( 1476 datetime_based_cursor_model.step and not datetime_based_cursor_model.cursor_granularity 1477 ) or ( 1478 not datetime_based_cursor_model.step and datetime_based_cursor_model.cursor_granularity 1479 ): 1480 raise ValueError( 1481 f"If step is defined, cursor_granularity should be as well and vice-versa. " 1482 f"Right now, step is `{datetime_based_cursor_model.step}` and cursor_granularity is `{datetime_based_cursor_model.cursor_granularity}`" 1483 ) 1484 1485 # When step is not defined, default to a step size from the starting date to the present moment 1486 step_length = datetime.timedelta.max 1487 interpolated_step = ( 1488 InterpolatedString.create( 1489 datetime_based_cursor_model.step, 1490 parameters=model_parameters, 1491 ) 1492 if datetime_based_cursor_model.step 1493 else None 1494 ) 1495 if interpolated_step: 1496 evaluated_step = interpolated_step.eval(config) 1497 if evaluated_step: 1498 step_length = parse_duration(evaluated_step) 1499 1500 clamping_strategy: ClampingStrategy = NoClamping() 1501 if datetime_based_cursor_model.clamping: 1502 # While it is undesirable to interpolate within the model factory (as opposed to at runtime), 1503 # it is still better than shifting interpolation low-code concept into the ConcurrentCursor runtime 1504 # object which we want to keep agnostic of being low-code 1505 target = InterpolatedString( 1506 string=datetime_based_cursor_model.clamping.target, 1507 parameters=model_parameters, 1508 ) 1509 evaluated_target = target.eval(config=config) 1510 match evaluated_target: 1511 case "DAY": 1512 clamping_strategy = DayClampingStrategy() 1513 end_date_provider = ClampingEndProvider( 1514 DayClampingStrategy(is_ceiling=False), 1515 end_date_provider, # type: ignore # Having issues w/ inspection for GapType and CursorValueType as shown in existing tests. Confirmed functionality is working in practice 1516 granularity=cursor_granularity or datetime.timedelta(seconds=1), 1517 ) 1518 case "WEEK": 1519 if ( 1520 not datetime_based_cursor_model.clamping.target_details 1521 or "weekday" not in datetime_based_cursor_model.clamping.target_details 1522 ): 1523 raise ValueError( 1524 "Given WEEK clamping, weekday needs to be provided as target_details" 1525 ) 1526 weekday = self._assemble_weekday( 1527 datetime_based_cursor_model.clamping.target_details["weekday"] 1528 ) 1529 clamping_strategy = WeekClampingStrategy(weekday) 1530 end_date_provider = ClampingEndProvider( 1531 WeekClampingStrategy(weekday, is_ceiling=False), 1532 end_date_provider, # type: ignore # Having issues w/ inspection for GapType and CursorValueType as shown in existing tests. Confirmed functionality is working in practice 1533 granularity=cursor_granularity or datetime.timedelta(days=1), 1534 ) 1535 case "MONTH": 1536 clamping_strategy = MonthClampingStrategy() 1537 end_date_provider = ClampingEndProvider( 1538 MonthClampingStrategy(is_ceiling=False), 1539 end_date_provider, # type: ignore # Having issues w/ inspection for GapType and CursorValueType as shown in existing tests. Confirmed functionality is working in practice 1540 granularity=cursor_granularity or datetime.timedelta(days=1), 1541 ) 1542 case _: 1543 raise ValueError( 1544 f"Invalid clamping target {evaluated_target}, expected DAY, WEEK, MONTH" 1545 ) 1546 1547 return ConcurrentCursor( 1548 stream_name=stream_name, 1549 stream_namespace=stream_namespace, 1550 stream_state=stream_state, 1551 message_repository=message_repository or self._message_repository, 1552 connector_state_manager=self._connector_state_manager, 1553 connector_state_converter=connector_state_converter, 1554 cursor_field=cursor_field, 1555 slice_boundary_fields=slice_boundary_fields, 1556 start=start_date, # type: ignore # Having issues w/ inspection for GapType and CursorValueType as shown in existing tests. Confirmed functionality is working in practice 1557 end_provider=end_date_provider, # type: ignore # Having issues w/ inspection for GapType and CursorValueType as shown in existing tests. Confirmed functionality is working in practice 1558 lookback_window=lookback_window, 1559 slice_range=step_length, 1560 cursor_granularity=cursor_granularity, 1561 clamping_strategy=clamping_strategy, 1562 )
def
create_concurrent_cursor_from_incrementing_count_cursor( self, model_type: Type[pydantic.v1.main.BaseModel], component_definition: Mapping[str, Any], stream_name: str, stream_namespace: Optional[str], stream_state: MutableMapping[str, Any], config: Mapping[str, Any], message_repository: Optional[airbyte_cdk.MessageRepository] = None, **kwargs: Any) -> airbyte_cdk.ConcurrentCursor:
1564 def create_concurrent_cursor_from_incrementing_count_cursor( 1565 self, 1566 model_type: Type[BaseModel], 1567 component_definition: ComponentDefinition, 1568 stream_name: str, 1569 stream_namespace: Optional[str], 1570 stream_state: MutableMapping[str, Any], 1571 config: Config, 1572 message_repository: Optional[MessageRepository] = None, 1573 **kwargs: Any, 1574 ) -> ConcurrentCursor: 1575 component_type = component_definition.get("type") 1576 if component_definition.get("type") != model_type.__name__: 1577 raise ValueError( 1578 f"Expected manifest component of type {model_type.__name__}, but received {component_type} instead" 1579 ) 1580 1581 incrementing_count_cursor_model = model_type.parse_obj(component_definition) 1582 1583 if not isinstance(incrementing_count_cursor_model, IncrementingCountCursorModel): 1584 raise ValueError( 1585 f"Expected {model_type.__name__} component, but received {incrementing_count_cursor_model.__class__.__name__}" 1586 ) 1587 1588 start_value: Union[int, str, None] = incrementing_count_cursor_model.start_value 1589 # Pydantic Union type coercion can convert int 0 to string '0' depending on Union order. 1590 # We need to handle both int and str representations of numeric values. 1591 # Evaluate the InterpolatedString and convert to int for the ConcurrentCursor. 1592 if start_value is not None: 1593 interpolated_start_value = InterpolatedString.create( 1594 str(start_value), # Ensure we pass a string to InterpolatedString.create 1595 parameters=incrementing_count_cursor_model.parameters or {}, 1596 ) 1597 evaluated_start_value: int = int(interpolated_start_value.eval(config=config)) 1598 else: 1599 evaluated_start_value = 0 1600 1601 cursor_field = self._get_catalog_defined_cursor_field( 1602 stream_name=stream_name, 1603 allow_catalog_defined_cursor_field=incrementing_count_cursor_model.allow_catalog_defined_cursor_field 1604 or False, 1605 ) 1606 1607 if not cursor_field: 1608 interpolated_cursor_field = InterpolatedString.create( 1609 incrementing_count_cursor_model.cursor_field, 1610 parameters=incrementing_count_cursor_model.parameters or {}, 1611 ) 1612 cursor_field = CursorField( 1613 cursor_field_key=interpolated_cursor_field.eval(config=config), 1614 supports_catalog_defined_cursor_field=incrementing_count_cursor_model.allow_catalog_defined_cursor_field 1615 or False, 1616 ) 1617 1618 connector_state_converter = IncrementingCountStreamStateConverter( 1619 is_sequential_state=True, # ConcurrentPerPartitionCursor only works with sequential state 1620 ) 1621 1622 return ConcurrentCursor( 1623 stream_name=stream_name, 1624 stream_namespace=stream_namespace, 1625 stream_state=stream_state, 1626 message_repository=message_repository or self._message_repository, 1627 connector_state_manager=self._connector_state_manager, 1628 connector_state_converter=connector_state_converter, 1629 cursor_field=cursor_field, 1630 slice_boundary_fields=None, 1631 start=evaluated_start_value, # type: ignore # Having issues w/ inspection for GapType and CursorValueType as shown in existing tests. Confirmed functionality is working in practice 1632 end_provider=connector_state_converter.get_end_provider(), # type: ignore # Having issues w/ inspection for GapType and CursorValueType as shown in existing tests. Confirmed functionality is working in practice 1633 )
def
create_concurrent_cursor_from_perpartition_cursor( self, state_manager: airbyte_cdk.ConnectorStateManager, model_type: Type[pydantic.v1.main.BaseModel], component_definition: Mapping[str, Any], stream_name: str, stream_namespace: Optional[str], config: Mapping[str, Any], stream_state: MutableMapping[str, Any], partition_router: airbyte_cdk.sources.declarative.partition_routers.PartitionRouter, attempt_to_create_cursor_if_not_provided: bool = False, **kwargs: Any) -> airbyte_cdk.sources.declarative.incremental.ConcurrentPerPartitionCursor:
1654 def create_concurrent_cursor_from_perpartition_cursor( 1655 self, 1656 state_manager: ConnectorStateManager, 1657 model_type: Type[BaseModel], 1658 component_definition: ComponentDefinition, 1659 stream_name: str, 1660 stream_namespace: Optional[str], 1661 config: Config, 1662 stream_state: MutableMapping[str, Any], 1663 partition_router: PartitionRouter, 1664 attempt_to_create_cursor_if_not_provided: bool = False, 1665 **kwargs: Any, 1666 ) -> ConcurrentPerPartitionCursor: 1667 component_type = component_definition.get("type") 1668 if component_definition.get("type") != model_type.__name__: 1669 raise ValueError( 1670 f"Expected manifest component of type {model_type.__name__}, but received {component_type} instead" 1671 ) 1672 1673 # FIXME the interfaces of the concurrent cursor are kind of annoying as they take a `ComponentDefinition` instead of the actual model. This was done because the ConcurrentDeclarativeSource didn't have access to the models [here for example](https://github.com/airbytehq/airbyte-python-cdk/blob/f525803b3fec9329e4cc8478996a92bf884bfde9/airbyte_cdk/sources/declarative/concurrent_declarative_source.py#L354C54-L354C91). So now we have two cases: 1674 # * The ComponentDefinition comes from model.__dict__ in which case we have `parameters` 1675 # * The ComponentDefinition comes from the manifest as a dict in which case we have `$parameters` 1676 # We should change those interfaces to use the model once we clean up the code in CDS at which point the parameter propagation should happen as part of the ModelToComponentFactory. 1677 if "$parameters" not in component_definition and "parameters" in component_definition: 1678 component_definition["$parameters"] = component_definition.get("parameters") # type: ignore # This is a dict 1679 datetime_based_cursor_model = model_type.parse_obj(component_definition) 1680 1681 if not isinstance(datetime_based_cursor_model, DatetimeBasedCursorModel): 1682 raise ValueError( 1683 f"Expected {model_type.__name__} component, but received {datetime_based_cursor_model.__class__.__name__}" 1684 ) 1685 1686 cursor_field = self._get_catalog_defined_cursor_field( 1687 stream_name=stream_name, 1688 allow_catalog_defined_cursor_field=datetime_based_cursor_model.allow_catalog_defined_cursor_field 1689 or False, 1690 ) 1691 1692 if not cursor_field: 1693 interpolated_cursor_field = InterpolatedString.create( 1694 datetime_based_cursor_model.cursor_field, 1695 # FIXME the interfaces of the concurrent cursor are kind of annoying as they take a `ComponentDefinition` instead of the actual model. This was done because the ConcurrentDeclarativeSource didn't have access to the models [here for example](https://github.com/airbytehq/airbyte-python-cdk/blob/f525803b3fec9329e4cc8478996a92bf884bfde9/airbyte_cdk/sources/declarative/concurrent_declarative_source.py#L354C54-L354C91). So now we have two cases: 1696 # * The ComponentDefinition comes from model.__dict__ in which case we have `parameters` 1697 # * The ComponentDefinition comes from the manifest as a dict in which case we have `$parameters` 1698 # We should change those interfaces to use the model once we clean up the code in CDS at which point the parameter propagation should happen as part of the ModelToComponentFactory. 1699 parameters=datetime_based_cursor_model.parameters or {}, 1700 ) 1701 cursor_field = CursorField( 1702 cursor_field_key=interpolated_cursor_field.eval(config=config), 1703 supports_catalog_defined_cursor_field=datetime_based_cursor_model.allow_catalog_defined_cursor_field 1704 or False, 1705 ) 1706 1707 datetime_format = datetime_based_cursor_model.datetime_format 1708 1709 cursor_granularity = ( 1710 parse_duration(datetime_based_cursor_model.cursor_granularity) 1711 if datetime_based_cursor_model.cursor_granularity 1712 else None 1713 ) 1714 1715 connector_state_converter: DateTimeStreamStateConverter 1716 connector_state_converter = CustomFormatConcurrentStreamStateConverter( 1717 datetime_format=datetime_format, 1718 input_datetime_formats=datetime_based_cursor_model.cursor_datetime_formats, 1719 is_sequential_state=True, # ConcurrentPerPartitionCursor only works with sequential state 1720 cursor_granularity=cursor_granularity, 1721 ) 1722 1723 # Create the cursor factory 1724 cursor_factory = ConcurrentCursorFactory( 1725 partial( 1726 self.create_concurrent_cursor_from_datetime_based_cursor, 1727 state_manager=state_manager, 1728 model_type=model_type, 1729 component_definition=component_definition, 1730 stream_name=stream_name, 1731 stream_namespace=stream_namespace, 1732 config=config, 1733 message_repository=NoopMessageRepository(), 1734 ) 1735 ) 1736 1737 # Per-partition state doesn't make sense for GroupingPartitionRouter, so force the global state 1738 use_global_cursor = isinstance( 1739 partition_router, GroupingPartitionRouter 1740 ) or component_definition.get("global_substream_cursor", False) 1741 1742 # Return the concurrent cursor and state converter 1743 return ConcurrentPerPartitionCursor( 1744 cursor_factory=cursor_factory, 1745 partition_router=partition_router, 1746 stream_name=stream_name, 1747 stream_namespace=stream_namespace, 1748 stream_state=stream_state, 1749 message_repository=self._message_repository, # type: ignore 1750 connector_state_manager=state_manager, 1751 connector_state_converter=connector_state_converter, 1752 cursor_field=cursor_field, 1753 use_global_cursor=use_global_cursor, 1754 attempt_to_create_cursor_if_not_provided=attempt_to_create_cursor_if_not_provided, 1755 )
@staticmethod
def
create_constant_backoff_strategy( model: airbyte_cdk.sources.declarative.models.declarative_component_schema.ConstantBackoffStrategy, config: Mapping[str, Any], **kwargs: Any) -> airbyte_cdk.sources.declarative.requesters.error_handlers.backoff_strategies.ConstantBackoffStrategy:
1757 @staticmethod 1758 def create_constant_backoff_strategy( 1759 model: ConstantBackoffStrategyModel, config: Config, **kwargs: Any 1760 ) -> ConstantBackoffStrategy: 1761 ModelToComponentFactory._validate_jitter_range(model.jitter_range_in_seconds) 1762 return ConstantBackoffStrategy( 1763 backoff_time_in_seconds=model.backoff_time_in_seconds, 1764 jitter_range_in_seconds=model.jitter_range_in_seconds, 1765 config=config, 1766 parameters=model.parameters or {}, 1767 )
def
create_cursor_pagination( self, model: airbyte_cdk.sources.declarative.models.declarative_component_schema.CursorPagination, config: Mapping[str, Any], decoder: airbyte_cdk.Decoder, **kwargs: Any) -> airbyte_cdk.CursorPaginationStrategy:
1774 def create_cursor_pagination( 1775 self, model: CursorPaginationModel, config: Config, decoder: Decoder, **kwargs: Any 1776 ) -> CursorPaginationStrategy: 1777 if isinstance(decoder, PaginationDecoderDecorator): 1778 inner_decoder = decoder.decoder 1779 else: 1780 inner_decoder = decoder 1781 decoder = PaginationDecoderDecorator(decoder=decoder) 1782 1783 if self._is_supported_decoder_for_pagination(inner_decoder): 1784 decoder_to_use = decoder 1785 else: 1786 raise ValueError( 1787 self._UNSUPPORTED_DECODER_ERROR.format(decoder_type=type(inner_decoder)) 1788 ) 1789 1790 # Pydantic v1 Union type coercion can convert int to string depending on Union order. 1791 # If page_size is a string that represents an integer (not an interpolation), convert it back. 1792 page_size = model.page_size 1793 if isinstance(page_size, str) and page_size.isdigit(): 1794 page_size = int(page_size) 1795 1796 return CursorPaginationStrategy( 1797 cursor_value=model.cursor_value, 1798 decoder=decoder_to_use, 1799 page_size=page_size, 1800 stop_condition=model.stop_condition, 1801 config=config, 1802 parameters=model.parameters or {}, 1803 )
def
create_custom_component(self, model: Any, config: Mapping[str, Any], **kwargs: Any) -> Any:
1805 def create_custom_component(self, model: Any, config: Config, **kwargs: Any) -> Any: 1806 """ 1807 Generically creates a custom component based on the model type and a class_name reference to the custom Python class being 1808 instantiated. Only the model's additional properties that match the custom class definition are passed to the constructor 1809 :param model: The Pydantic model of the custom component being created 1810 :param config: The custom defined connector config 1811 :return: The declarative component built from the Pydantic model to be used at runtime 1812 """ 1813 custom_component_class = self._get_class_from_fully_qualified_class_name(model.class_name) 1814 component_fields = get_type_hints(custom_component_class) 1815 model_args = model.dict() 1816 model_args["config"] = config 1817 1818 # There are cases where a parent component will pass arguments to a child component via kwargs. When there are field collisions 1819 # we defer to these arguments over the component's definition 1820 for key, arg in kwargs.items(): 1821 model_args[key] = arg 1822 1823 # Pydantic is unable to parse a custom component's fields that are subcomponents into models because their fields and types are not 1824 # defined in the schema. The fields and types are defined within the Python class implementation. Pydantic can only parse down to 1825 # the custom component and this code performs a second parse to convert the sub-fields first into models, then declarative components 1826 for model_field, model_value in model_args.items(): 1827 # If a custom component field doesn't have a type set, we try to use the type hints to infer the type 1828 if ( 1829 isinstance(model_value, dict) 1830 and "type" not in model_value 1831 and model_field in component_fields 1832 ): 1833 derived_type = self._derive_component_type_from_type_hints( 1834 component_fields.get(model_field) 1835 ) 1836 if derived_type: 1837 model_value["type"] = derived_type 1838 1839 if self._is_component(model_value): 1840 model_args[model_field] = self._create_nested_component( 1841 model, 1842 model_field, 1843 model_value, 1844 config, 1845 **kwargs, 1846 ) 1847 elif isinstance(model_value, list): 1848 vals = [] 1849 for v in model_value: 1850 if isinstance(v, dict) and "type" not in v and model_field in component_fields: 1851 derived_type = self._derive_component_type_from_type_hints( 1852 component_fields.get(model_field) 1853 ) 1854 if derived_type: 1855 v["type"] = derived_type 1856 if self._is_component(v): 1857 vals.append( 1858 self._create_nested_component( 1859 model, 1860 model_field, 1861 v, 1862 config, 1863 **kwargs, 1864 ) 1865 ) 1866 else: 1867 vals.append(v) 1868 model_args[model_field] = vals 1869 1870 kwargs = { 1871 class_field: model_args[class_field] 1872 for class_field in component_fields.keys() 1873 if class_field in model_args 1874 } 1875 1876 if "api_budget" in component_fields and kwargs.get("api_budget") is None: 1877 kwargs["api_budget"] = self._api_budget 1878 1879 return custom_component_class(**kwargs)
Generically creates a custom component based on the model type and a class_name reference to the custom Python class being instantiated. Only the model's additional properties that match the custom class definition are passed to the constructor
Parameters
- model: The Pydantic model of the custom component being created
- config: The custom defined connector config
Returns
The declarative component built from the Pydantic model to be used at runtime
def
create_default_stream( self, model: airbyte_cdk.sources.declarative.models.declarative_component_schema.DeclarativeStream, config: Mapping[str, Any], is_parent: bool = False, **kwargs: Any) -> airbyte_cdk.sources.streams.concurrent.abstract_stream.AbstractStream:
2014 def create_default_stream( 2015 self, model: DeclarativeStreamModel, config: Config, is_parent: bool = False, **kwargs: Any 2016 ) -> AbstractStream: 2017 primary_key = model.primary_key.__root__ if model.primary_key else None 2018 self._migrate_state(model, config) 2019 2020 partition_router = self._build_stream_slicer_from_partition_router( 2021 model.retriever, 2022 config, 2023 stream_name=model.name, 2024 **kwargs, 2025 ) 2026 concurrent_cursor = self._build_concurrent_cursor(model, partition_router, config) 2027 if model.incremental_sync and isinstance(model.incremental_sync, DatetimeBasedCursorModel): 2028 cursor_model: DatetimeBasedCursorModel = model.incremental_sync 2029 2030 end_time_option = ( 2031 self._create_component_from_model( 2032 cursor_model.end_time_option, config, parameters=cursor_model.parameters or {} 2033 ) 2034 if cursor_model.end_time_option 2035 else None 2036 ) 2037 start_time_option = ( 2038 self._create_component_from_model( 2039 cursor_model.start_time_option, config, parameters=cursor_model.parameters or {} 2040 ) 2041 if cursor_model.start_time_option 2042 else None 2043 ) 2044 2045 datetime_request_options_provider = DatetimeBasedRequestOptionsProvider( 2046 start_time_option=start_time_option, 2047 end_time_option=end_time_option, 2048 partition_field_start=cursor_model.partition_field_start, 2049 partition_field_end=cursor_model.partition_field_end, 2050 config=config, 2051 parameters=model.parameters or {}, 2052 ) 2053 request_options_provider = ( 2054 datetime_request_options_provider 2055 if not isinstance(concurrent_cursor, ConcurrentPerPartitionCursor) 2056 else PerPartitionRequestOptionsProvider( 2057 partition_router, datetime_request_options_provider 2058 ) 2059 ) 2060 elif model.incremental_sync and isinstance( 2061 model.incremental_sync, IncrementingCountCursorModel 2062 ): 2063 if isinstance(concurrent_cursor, ConcurrentPerPartitionCursor): 2064 raise ValueError( 2065 "PerPartition does not support per partition states because switching to global state is time based" 2066 ) 2067 2068 cursor_model: IncrementingCountCursorModel = model.incremental_sync # type: ignore 2069 2070 start_time_option = ( 2071 self._create_component_from_model( 2072 cursor_model.start_value_option, # type: ignore # mypy still thinks cursor_model of type DatetimeBasedCursor 2073 config, 2074 parameters=cursor_model.parameters or {}, 2075 ) 2076 if cursor_model.start_value_option # type: ignore # mypy still thinks cursor_model of type DatetimeBasedCursor 2077 else None 2078 ) 2079 2080 # The concurrent engine defaults the start/end fields on the slice to "start" and "end", but 2081 # the default DatetimeBasedRequestOptionsProvider() sets them to start_time/end_time 2082 partition_field_start = "start" 2083 2084 request_options_provider = DatetimeBasedRequestOptionsProvider( 2085 start_time_option=start_time_option, 2086 partition_field_start=partition_field_start, 2087 config=config, 2088 parameters=model.parameters or {}, 2089 ) 2090 else: 2091 request_options_provider = None 2092 2093 transformations = [] 2094 if model.transformations: 2095 for transformation_model in model.transformations: 2096 transformations.append( 2097 self._create_component_from_model(model=transformation_model, config=config) 2098 ) 2099 file_uploader = None 2100 if model.file_uploader: 2101 file_uploader = self._create_component_from_model( 2102 model=model.file_uploader, config=config 2103 ) 2104 2105 stream_slicer: ConcurrentStreamSlicer = ( 2106 partition_router 2107 if isinstance(concurrent_cursor, FinalStateCursor) 2108 else concurrent_cursor 2109 ) 2110 2111 retriever = self._create_component_from_model( 2112 model=model.retriever, 2113 config=config, 2114 name=model.name, 2115 primary_key=primary_key, 2116 request_options_provider=request_options_provider, 2117 stream_slicer=stream_slicer, 2118 partition_router=partition_router, 2119 has_stop_condition_cursor=self._is_stop_condition_on_cursor(model), 2120 is_client_side_incremental_sync=self._is_client_side_filtering_enabled(model), 2121 cursor=concurrent_cursor, 2122 transformations=transformations, 2123 file_uploader=file_uploader, 2124 incremental_sync=model.incremental_sync, 2125 ) 2126 if isinstance(retriever, AsyncRetriever): 2127 stream_slicer = retriever.stream_slicer 2128 2129 schema_loader: SchemaLoader 2130 if model.schema_loader and isinstance(model.schema_loader, list): 2131 nested_schema_loaders = [ 2132 self._create_component_from_model(model=nested_schema_loader, config=config) 2133 for nested_schema_loader in model.schema_loader 2134 ] 2135 schema_loader = CompositeSchemaLoader( 2136 schema_loaders=nested_schema_loaders, parameters={} 2137 ) 2138 elif model.schema_loader: 2139 schema_loader = self._create_component_from_model( 2140 model=model.schema_loader, # type: ignore # If defined, schema_loader is guaranteed not to be a list and will be one of the existing base models 2141 config=config, 2142 ) 2143 else: 2144 options = model.parameters or {} 2145 if "name" not in options: 2146 options["name"] = model.name 2147 schema_loader = DefaultSchemaLoader(config=config, parameters=options) 2148 schema_loader = CachingSchemaLoaderDecorator(schema_loader) 2149 2150 stream_name = model.name or "" 2151 return DefaultStream( 2152 partition_generator=StreamSlicerPartitionGenerator( 2153 DeclarativePartitionFactory( 2154 stream_name, 2155 schema_loader, 2156 retriever, 2157 self._message_repository, 2158 ), 2159 stream_slicer, 2160 slice_limit=self._limit_slices_fetched, 2161 ), 2162 name=stream_name, 2163 json_schema=schema_loader.get_json_schema, 2164 primary_key=get_primary_key_from_stream(primary_key), 2165 cursor_field=( 2166 concurrent_cursor.cursor_field 2167 if hasattr(concurrent_cursor, "cursor_field") 2168 else None 2169 ), 2170 logger=logging.getLogger(f"airbyte.{stream_name}"), 2171 cursor=concurrent_cursor, 2172 supports_file_transfer=hasattr(model, "file_uploader") and bool(model.file_uploader), 2173 )
def
create_default_error_handler( self, model: airbyte_cdk.sources.declarative.models.declarative_component_schema.DefaultErrorHandler, config: Mapping[str, Any], **kwargs: Any) -> airbyte_cdk.sources.declarative.requesters.error_handlers.DefaultErrorHandler:
2315 def create_default_error_handler( 2316 self, model: DefaultErrorHandlerModel, config: Config, **kwargs: Any 2317 ) -> DefaultErrorHandler: 2318 backoff_strategies = [] 2319 if model.backoff_strategies: 2320 for backoff_strategy_model in model.backoff_strategies: 2321 backoff_strategies.append( 2322 self._create_component_from_model(model=backoff_strategy_model, config=config) 2323 ) 2324 2325 response_filters = [] 2326 if model.response_filters: 2327 for response_filter_model in model.response_filters: 2328 response_filters.append( 2329 self._create_component_from_model(model=response_filter_model, config=config) 2330 ) 2331 response_filters.append( 2332 HttpResponseFilter(config=config, parameters=model.parameters or {}) 2333 ) 2334 2335 return DefaultErrorHandler( 2336 backoff_strategies=backoff_strategies, 2337 max_retries=model.max_retries, 2338 response_filters=response_filters, 2339 config=config, 2340 parameters=model.parameters or {}, 2341 )
def
create_default_paginator( self, model: airbyte_cdk.sources.declarative.models.declarative_component_schema.DefaultPaginator, config: Mapping[str, Any], *, url_base: str, extractor_model: Union[airbyte_cdk.sources.declarative.models.declarative_component_schema.DpathExtractor, airbyte_cdk.sources.declarative.models.declarative_component_schema.CustomRecordExtractor, NoneType] = None, decoder: Optional[airbyte_cdk.Decoder] = None, cursor_used_for_stop_condition: Optional[airbyte_cdk.Cursor] = None) -> Union[airbyte_cdk.DefaultPaginator, airbyte_cdk.sources.declarative.requesters.paginators.PaginatorTestReadDecorator]:
2343 def create_default_paginator( 2344 self, 2345 model: DefaultPaginatorModel, 2346 config: Config, 2347 *, 2348 url_base: str, 2349 extractor_model: Optional[Union[CustomRecordExtractorModel, DpathExtractorModel]] = None, 2350 decoder: Optional[Decoder] = None, 2351 cursor_used_for_stop_condition: Optional[Cursor] = None, 2352 ) -> Union[DefaultPaginator, PaginatorTestReadDecorator]: 2353 if decoder: 2354 if self._is_supported_decoder_for_pagination(decoder): 2355 decoder_to_use = PaginationDecoderDecorator(decoder=decoder) 2356 else: 2357 raise ValueError(self._UNSUPPORTED_DECODER_ERROR.format(decoder_type=type(decoder))) 2358 else: 2359 decoder_to_use = PaginationDecoderDecorator(decoder=JsonDecoder(parameters={})) 2360 page_size_option = ( 2361 self._create_component_from_model(model=model.page_size_option, config=config) 2362 if model.page_size_option 2363 else None 2364 ) 2365 page_token_option = ( 2366 self._create_component_from_model(model=model.page_token_option, config=config) 2367 if model.page_token_option 2368 else None 2369 ) 2370 pagination_strategy = self._create_component_from_model( 2371 model=model.pagination_strategy, 2372 config=config, 2373 decoder=decoder_to_use, 2374 extractor_model=extractor_model, 2375 ) 2376 if cursor_used_for_stop_condition: 2377 pagination_strategy = StopConditionPaginationStrategyDecorator( 2378 pagination_strategy, CursorStopCondition(cursor_used_for_stop_condition) 2379 ) 2380 paginator = DefaultPaginator( 2381 decoder=decoder_to_use, 2382 page_size_option=page_size_option, 2383 page_token_option=page_token_option, 2384 pagination_strategy=pagination_strategy, 2385 url_base=url_base, 2386 config=config, 2387 parameters=model.parameters or {}, 2388 ) 2389 if self._limit_pages_fetched_per_slice: 2390 return PaginatorTestReadDecorator(paginator, self._limit_pages_fetched_per_slice) 2391 return paginator
def
create_dpath_extractor( self, model: airbyte_cdk.sources.declarative.models.declarative_component_schema.DpathExtractor, config: Mapping[str, Any], decoder: Optional[airbyte_cdk.Decoder] = None, **kwargs: Any) -> airbyte_cdk.DpathExtractor:
2393 def create_dpath_extractor( 2394 self, 2395 model: DpathExtractorModel, 2396 config: Config, 2397 decoder: Optional[Decoder] = None, 2398 **kwargs: Any, 2399 ) -> DpathExtractor: 2400 if decoder: 2401 decoder_to_use = decoder 2402 else: 2403 decoder_to_use = JsonDecoder(parameters={}) 2404 model_field_path: List[Union[InterpolatedString, str]] = [x for x in model.field_path] 2405 2406 record_expander = None 2407 if model.record_expander: 2408 record_expander = self._create_component_from_model( 2409 model=model.record_expander, 2410 config=config, 2411 ) 2412 2413 return DpathExtractor( 2414 decoder=decoder_to_use, 2415 field_path=model_field_path, 2416 config=config, 2417 parameters=model.parameters or {}, 2418 record_expander=record_expander, 2419 )
def
create_record_expander( self, model: airbyte_cdk.sources.declarative.models.declarative_component_schema.RecordExpander, config: Mapping[str, Any], **kwargs: Any) -> airbyte_cdk.sources.declarative.expanders.RecordExpander:
2421 def create_record_expander( 2422 self, 2423 model: RecordExpanderModel, 2424 config: Config, 2425 **kwargs: Any, 2426 ) -> RecordExpander: 2427 return RecordExpander( 2428 expand_records_from_field=model.expand_records_from_field, 2429 config=config, 2430 parameters=model.parameters or {}, 2431 remain_original_record=model.remain_original_record or False, 2432 on_no_records=OnNoRecords(model.on_no_records.value) 2433 if model.on_no_records 2434 else OnNoRecords.skip, 2435 )
@staticmethod
def
create_response_to_file_extractor( model: airbyte_cdk.sources.declarative.models.declarative_component_schema.ResponseToFileExtractor, **kwargs: Any) -> airbyte_cdk.sources.declarative.extractors.ResponseToFileExtractor:
2437 @staticmethod 2438 def create_response_to_file_extractor( 2439 model: ResponseToFileExtractorModel, 2440 **kwargs: Any, 2441 ) -> ResponseToFileExtractor: 2442 return ResponseToFileExtractor( 2443 parameters=model.parameters or {}, 2444 preserve_na_values=model.preserve_na_values or False, 2445 )
@staticmethod
def
create_exponential_backoff_strategy( model: airbyte_cdk.sources.declarative.models.declarative_component_schema.ExponentialBackoffStrategy, config: Mapping[str, Any]) -> airbyte_cdk.sources.declarative.requesters.error_handlers.backoff_strategies.ExponentialBackoffStrategy:
2447 @staticmethod 2448 def create_exponential_backoff_strategy( 2449 model: ExponentialBackoffStrategyModel, config: Config 2450 ) -> ExponentialBackoffStrategy: 2451 ModelToComponentFactory._validate_jitter_range(model.jitter_range_in_seconds) 2452 return ExponentialBackoffStrategy( 2453 factor=model.factor or 5, 2454 jitter_range_in_seconds=model.jitter_range_in_seconds, 2455 parameters=model.parameters or {}, 2456 config=config, 2457 )
@staticmethod
def
create_group_by_key( model: airbyte_cdk.sources.declarative.models.declarative_component_schema.GroupByKeyMergeStrategy, config: Mapping[str, Any]) -> airbyte_cdk.sources.declarative.requesters.query_properties.strategies.GroupByKey:
def
create_http_requester( self, model: airbyte_cdk.sources.declarative.models.declarative_component_schema.HttpRequester, config: Mapping[str, Any], decoder: airbyte_cdk.Decoder = JsonDecoder(), query_properties_key: Optional[str] = None, use_cache: Optional[bool] = None, *, name: str) -> airbyte_cdk.HttpRequester:
2463 def create_http_requester( 2464 self, 2465 model: HttpRequesterModel, 2466 config: Config, 2467 decoder: Decoder = JsonDecoder(parameters={}), 2468 query_properties_key: Optional[str] = None, 2469 use_cache: Optional[bool] = None, 2470 *, 2471 name: str, 2472 ) -> HttpRequester: 2473 authenticator = ( 2474 self._create_component_from_model( 2475 model=model.authenticator, 2476 config=config, 2477 url_base=model.url or model.url_base, 2478 name=name, 2479 decoder=decoder, 2480 ) 2481 if model.authenticator 2482 else None 2483 ) 2484 error_handler = ( 2485 self._create_component_from_model(model=model.error_handler, config=config) 2486 if model.error_handler 2487 else DefaultErrorHandler( 2488 backoff_strategies=[], 2489 response_filters=[], 2490 config=config, 2491 parameters=model.parameters or {}, 2492 ) 2493 ) 2494 2495 api_budget = self._api_budget 2496 2497 request_options_provider = InterpolatedRequestOptionsProvider( 2498 request_body=model.request_body, 2499 request_body_data=model.request_body_data, 2500 request_body_json=model.request_body_json, 2501 request_headers=model.request_headers, 2502 request_parameters=model.request_parameters, # type: ignore # QueryProperties have been removed in `create_simple_retriever` 2503 query_properties_key=query_properties_key, 2504 config=config, 2505 parameters=model.parameters or {}, 2506 ) 2507 2508 assert model.use_cache is not None # for mypy 2509 assert model.http_method is not None # for mypy 2510 2511 should_use_cache = (model.use_cache or bool(use_cache)) and not self._disable_cache 2512 2513 return HttpRequester( 2514 name=name, 2515 url=model.url, 2516 url_base=model.url_base, 2517 path=model.path, 2518 authenticator=authenticator, 2519 error_handler=error_handler, 2520 api_budget=api_budget, 2521 http_method=HttpMethod[model.http_method.value], 2522 request_options_provider=request_options_provider, 2523 config=config, 2524 disable_retries=self._disable_retries, 2525 parameters=model.parameters or {}, 2526 message_repository=self._message_repository, 2527 use_cache=should_use_cache, 2528 decoder=decoder, 2529 stream_response=decoder.is_stream_response() if decoder else False, 2530 )
@staticmethod
def
create_http_response_filter( model: airbyte_cdk.sources.declarative.models.declarative_component_schema.HttpResponseFilter, config: Mapping[str, Any], **kwargs: Any) -> airbyte_cdk.sources.declarative.requesters.error_handlers.HttpResponseFilter:
2532 @staticmethod 2533 def create_http_response_filter( 2534 model: HttpResponseFilterModel, config: Config, **kwargs: Any 2535 ) -> HttpResponseFilter: 2536 if model.action: 2537 action = ResponseAction(model.action.value) 2538 else: 2539 action = None 2540 2541 failure_type = FailureType(model.failure_type.value) if model.failure_type else None 2542 2543 http_codes = ( 2544 set(model.http_codes) if model.http_codes else set() 2545 ) # JSON schema notation has no set data type. The schema enforces an array of unique elements 2546 2547 return HttpResponseFilter( 2548 action=action, 2549 failure_type=failure_type, 2550 error_message=model.error_message or "", 2551 error_message_contains=model.error_message_contains or "", 2552 http_codes=http_codes, 2553 predicate=model.predicate or "", 2554 config=config, 2555 parameters=model.parameters or {}, 2556 )
@staticmethod
def
create_inline_schema_loader( model: airbyte_cdk.sources.declarative.models.declarative_component_schema.InlineSchemaLoader, config: Mapping[str, Any], **kwargs: Any) -> airbyte_cdk.sources.declarative.schema.InlineSchemaLoader:
def
create_complex_field_type( self, model: airbyte_cdk.sources.declarative.models.declarative_component_schema.ComplexFieldType, config: Mapping[str, Any], **kwargs: Any) -> airbyte_cdk.sources.declarative.schema.ComplexFieldType:
2564 def create_complex_field_type( 2565 self, model: ComplexFieldTypeModel, config: Config, **kwargs: Any 2566 ) -> ComplexFieldType: 2567 items = ( 2568 self._create_component_from_model(model=model.items, config=config) 2569 if isinstance(model.items, ComplexFieldTypeModel) 2570 else model.items 2571 ) 2572 2573 return ComplexFieldType(field_type=model.field_type, items=items)
def
create_types_map( self, model: airbyte_cdk.sources.declarative.models.declarative_component_schema.TypesMap, config: Mapping[str, Any], **kwargs: Any) -> airbyte_cdk.sources.declarative.schema.TypesMap:
2575 def create_types_map(self, model: TypesMapModel, config: Config, **kwargs: Any) -> TypesMap: 2576 target_type = ( 2577 self._create_component_from_model(model=model.target_type, config=config) 2578 if isinstance(model.target_type, ComplexFieldTypeModel) 2579 else model.target_type 2580 ) 2581 2582 return TypesMap( 2583 target_type=target_type, 2584 current_type=model.current_type, 2585 condition=model.condition if model.condition is not None else "True", 2586 )
def
create_schema_type_identifier( self, model: airbyte_cdk.sources.declarative.models.declarative_component_schema.SchemaTypeIdentifier, config: Mapping[str, Any], **kwargs: Any) -> airbyte_cdk.sources.declarative.schema.SchemaTypeIdentifier:
2588 def create_schema_type_identifier( 2589 self, model: SchemaTypeIdentifierModel, config: Config, **kwargs: Any 2590 ) -> SchemaTypeIdentifier: 2591 types_mapping = [] 2592 if model.types_mapping: 2593 types_mapping.extend( 2594 [ 2595 self._create_component_from_model(types_map, config=config) 2596 for types_map in model.types_mapping 2597 ] 2598 ) 2599 model_schema_pointer: List[Union[InterpolatedString, str]] = ( 2600 [x for x in model.schema_pointer] if model.schema_pointer else [] 2601 ) 2602 model_key_pointer: List[Union[InterpolatedString, str]] = [x for x in model.key_pointer] 2603 model_type_pointer: Optional[List[Union[InterpolatedString, str]]] = ( 2604 [x for x in model.type_pointer] if model.type_pointer else None 2605 ) 2606 2607 return SchemaTypeIdentifier( 2608 schema_pointer=model_schema_pointer, 2609 key_pointer=model_key_pointer, 2610 type_pointer=model_type_pointer, 2611 types_mapping=types_mapping, 2612 parameters=model.parameters or {}, 2613 )
def
create_dynamic_schema_loader( self, model: airbyte_cdk.sources.declarative.models.declarative_component_schema.DynamicSchemaLoader, config: Mapping[str, Any], **kwargs: Any) -> airbyte_cdk.sources.declarative.schema.DynamicSchemaLoader:
2615 def create_dynamic_schema_loader( 2616 self, model: DynamicSchemaLoaderModel, config: Config, **kwargs: Any 2617 ) -> DynamicSchemaLoader: 2618 schema_transformations = [] 2619 if model.schema_transformations: 2620 for transformation_model in model.schema_transformations: 2621 schema_transformations.append( 2622 self._create_component_from_model(model=transformation_model, config=config) 2623 ) 2624 name = "dynamic_properties" 2625 retriever = self._create_component_from_model( 2626 model=model.retriever, 2627 config=config, 2628 name=name, 2629 primary_key=None, 2630 partition_router=self._build_stream_slicer_from_partition_router( 2631 model.retriever, config 2632 ), 2633 transformations=[], 2634 use_cache=True, 2635 log_formatter=( 2636 lambda response: format_http_message( 2637 response, 2638 f"Schema loader '{name}' request", 2639 f"Request performed in order to extract schema.", 2640 name, 2641 is_auxiliary=True, 2642 ) 2643 ), 2644 ) 2645 schema_type_identifier = self._create_component_from_model( 2646 model.schema_type_identifier, config=config, parameters=model.parameters or {} 2647 ) 2648 schema_filter = ( 2649 self._create_component_from_model( 2650 model.schema_filter, config=config, parameters=model.parameters or {} 2651 ) 2652 if model.schema_filter is not None 2653 else None 2654 ) 2655 2656 return DynamicSchemaLoader( 2657 retriever=retriever, 2658 config=config, 2659 schema_transformations=schema_transformations, 2660 schema_filter=schema_filter, 2661 schema_type_identifier=schema_type_identifier, 2662 parameters=model.parameters or {}, 2663 )
@staticmethod
def
create_json_decoder( model: airbyte_cdk.sources.declarative.models.declarative_component_schema.JsonDecoder, config: Mapping[str, Any], **kwargs: Any) -> airbyte_cdk.Decoder:
def
create_csv_decoder( self, model: airbyte_cdk.sources.declarative.models.declarative_component_schema.CsvDecoder, config: Mapping[str, Any], **kwargs: Any) -> airbyte_cdk.Decoder:
def
create_jsonl_decoder( self, model: airbyte_cdk.sources.declarative.models.declarative_component_schema.JsonlDecoder, config: Mapping[str, Any], **kwargs: Any) -> airbyte_cdk.Decoder:
def
create_json_items_decoder( self, model: airbyte_cdk.sources.declarative.models.declarative_component_schema.JsonItemsDecoder, config: Mapping[str, Any], **kwargs: Any) -> airbyte_cdk.Decoder:
2683 def create_json_items_decoder( 2684 self, model: JsonItemsDecoderModel, config: Config, **kwargs: Any 2685 ) -> Decoder: 2686 return CompositeRawDecoder( 2687 parser=ModelToComponentFactory._get_parser(model, config), 2688 stream_response=False if self._emit_connector_builder_messages else True, 2689 )
def
create_gzip_decoder( self, model: airbyte_cdk.sources.declarative.models.declarative_component_schema.GzipDecoder, config: Mapping[str, Any], **kwargs: Any) -> airbyte_cdk.Decoder:
2691 def create_gzip_decoder( 2692 self, model: GzipDecoderModel, config: Config, **kwargs: Any 2693 ) -> Decoder: 2694 _compressed_response_types = { 2695 "gzip", 2696 "x-gzip", 2697 "gzip, deflate", 2698 "x-gzip, deflate", 2699 "application/zip", 2700 "application/gzip", 2701 "application/x-gzip", 2702 "application/x-zip-compressed", 2703 } 2704 2705 gzip_parser: GzipParser = ModelToComponentFactory._get_parser(model, config) # type: ignore # based on the model, we know this will be a GzipParser 2706 2707 if self._emit_connector_builder_messages: 2708 # This is very surprising but if the response is not streamed, 2709 # CompositeRawDecoder calls response.content and the requests library actually uncompress the data as opposed to response.raw, 2710 # which uses urllib3 directly and does not uncompress the data. 2711 return CompositeRawDecoder(gzip_parser.inner_parser, False) 2712 2713 return CompositeRawDecoder.by_headers( 2714 [({"Content-Encoding", "Content-Type"}, _compressed_response_types, gzip_parser)], 2715 stream_response=True, 2716 fallback_parser=gzip_parser.inner_parser, 2717 )
@staticmethod
def
create_iterable_decoder( model: airbyte_cdk.sources.declarative.models.declarative_component_schema.IterableDecoder, config: Mapping[str, Any], **kwargs: Any) -> airbyte_cdk.sources.declarative.decoders.IterableDecoder:
@staticmethod
def
create_xml_decoder( model: airbyte_cdk.sources.declarative.models.declarative_component_schema.XmlDecoder, config: Mapping[str, Any], **kwargs: Any) -> airbyte_cdk.sources.declarative.decoders.XmlDecoder:
def
create_zipfile_decoder( self, model: airbyte_cdk.sources.declarative.models.declarative_component_schema.ZipfileDecoder, config: Mapping[str, Any], **kwargs: Any) -> airbyte_cdk.sources.declarative.decoders.ZipfileDecoder:
@staticmethod
def
create_json_file_schema_loader( model: airbyte_cdk.sources.declarative.models.declarative_component_schema.JsonFileSchemaLoader, config: Mapping[str, Any], **kwargs: Any) -> airbyte_cdk.JsonFileSchemaLoader:
def
create_jwt_authenticator( self, model: airbyte_cdk.sources.declarative.models.declarative_component_schema.JwtAuthenticator, config: Mapping[str, Any], **kwargs: Any) -> airbyte_cdk.sources.declarative.auth.JwtAuthenticator:
2771 def create_jwt_authenticator( 2772 self, model: JwtAuthenticatorModel, config: Config, **kwargs: Any 2773 ) -> JwtAuthenticator: 2774 jwt_headers = model.jwt_headers or JwtHeadersModel(kid=None, typ="JWT", cty=None) 2775 jwt_payload = model.jwt_payload or JwtPayloadModel(iss=None, sub=None, aud=None) 2776 request_option = ( 2777 self._create_component_from_model(model.request_option, config) 2778 if model.request_option 2779 else None 2780 ) 2781 return JwtAuthenticator( 2782 config=config, 2783 parameters=model.parameters or {}, 2784 algorithm=JwtAlgorithm(model.algorithm.value), 2785 secret_key=model.secret_key, 2786 base64_encode_secret_key=model.base64_encode_secret_key, 2787 token_duration=model.token_duration, 2788 header_prefix=model.header_prefix, 2789 kid=jwt_headers.kid, 2790 typ=jwt_headers.typ, 2791 cty=jwt_headers.cty, 2792 iss=jwt_payload.iss, 2793 sub=jwt_payload.sub, 2794 aud=jwt_payload.aud, 2795 additional_jwt_headers=model.additional_jwt_headers, 2796 additional_jwt_payload=model.additional_jwt_payload, 2797 passphrase=model.passphrase, 2798 request_option=request_option, 2799 )
def
create_list_partition_router( self, model: airbyte_cdk.sources.declarative.models.declarative_component_schema.ListPartitionRouter, config: Mapping[str, Any], **kwargs: Any) -> airbyte_cdk.sources.declarative.partition_routers.ListPartitionRouter:
2801 def create_list_partition_router( 2802 self, model: ListPartitionRouterModel, config: Config, **kwargs: Any 2803 ) -> ListPartitionRouter: 2804 request_option = ( 2805 self._create_component_from_model(model.request_option, config) 2806 if model.request_option 2807 else None 2808 ) 2809 return ListPartitionRouter( 2810 cursor_field=model.cursor_field, 2811 request_option=request_option, 2812 values=model.values, 2813 config=config, 2814 parameters=model.parameters or {}, 2815 )
@staticmethod
def
create_min_max_datetime( model: airbyte_cdk.sources.declarative.models.declarative_component_schema.MinMaxDatetime, config: Mapping[str, Any], **kwargs: Any) -> airbyte_cdk.MinMaxDatetime:
2817 @staticmethod 2818 def create_min_max_datetime( 2819 model: MinMaxDatetimeModel, config: Config, **kwargs: Any 2820 ) -> MinMaxDatetime: 2821 return MinMaxDatetime( 2822 datetime=model.datetime, 2823 datetime_format=model.datetime_format or "", 2824 max_datetime=model.max_datetime or "", 2825 min_datetime=model.min_datetime or "", 2826 parameters=model.parameters or {}, 2827 )
@staticmethod
def
create_no_auth( model: airbyte_cdk.sources.declarative.models.declarative_component_schema.NoAuth, config: Mapping[str, Any], **kwargs: Any) -> airbyte_cdk.NoAuth:
@staticmethod
def
create_no_pagination( model: airbyte_cdk.sources.declarative.models.declarative_component_schema.NoPagination, config: Mapping[str, Any], **kwargs: Any) -> airbyte_cdk.sources.declarative.requesters.paginators.NoPagination:
def
create_oauth_authenticator( self, model: airbyte_cdk.sources.declarative.models.declarative_component_schema.OAuthAuthenticator, config: Mapping[str, Any], **kwargs: Any) -> airbyte_cdk.DeclarativeOauth2Authenticator:
2839 def create_oauth_authenticator( 2840 self, model: OAuthAuthenticatorModel, config: Config, **kwargs: Any 2841 ) -> DeclarativeOauth2Authenticator: 2842 profile_assertion = ( 2843 self._create_component_from_model(model.profile_assertion, config=config) 2844 if model.profile_assertion 2845 else None 2846 ) 2847 2848 refresh_token_error_status_codes, refresh_token_error_key, refresh_token_error_values = ( 2849 self._get_refresh_token_error_information(model) 2850 ) 2851 if model.refresh_token_updater: 2852 # ignore type error because fixing it would have a lot of dependencies, revisit later 2853 return DeclarativeSingleUseRefreshTokenOauth2Authenticator( # type: ignore 2854 config, 2855 InterpolatedString.create( 2856 model.token_refresh_endpoint, # type: ignore 2857 parameters=model.parameters or {}, 2858 ).eval(config), 2859 access_token_name=InterpolatedString.create( 2860 model.access_token_name or "access_token", parameters=model.parameters or {} 2861 ).eval(config), 2862 refresh_token_name=model.refresh_token_updater.refresh_token_name, 2863 expires_in_name=InterpolatedString.create( 2864 model.expires_in_name or "expires_in", parameters=model.parameters or {} 2865 ).eval(config), 2866 client_id_name=InterpolatedString.create( 2867 model.client_id_name or "client_id", parameters=model.parameters or {} 2868 ).eval(config), 2869 client_id=InterpolatedString.create( 2870 model.client_id, parameters=model.parameters or {} 2871 ).eval(config) 2872 if model.client_id 2873 else model.client_id, 2874 client_secret_name=InterpolatedString.create( 2875 model.client_secret_name or "client_secret", parameters=model.parameters or {} 2876 ).eval(config), 2877 client_secret=InterpolatedString.create( 2878 model.client_secret, parameters=model.parameters or {} 2879 ).eval(config) 2880 if model.client_secret 2881 else model.client_secret, 2882 access_token_config_path=model.refresh_token_updater.access_token_config_path, 2883 refresh_token_config_path=model.refresh_token_updater.refresh_token_config_path, 2884 token_expiry_date_config_path=model.refresh_token_updater.token_expiry_date_config_path, 2885 grant_type_name=InterpolatedString.create( 2886 model.grant_type_name or "grant_type", parameters=model.parameters or {} 2887 ).eval(config), 2888 grant_type=InterpolatedString.create( 2889 model.grant_type or "refresh_token", parameters=model.parameters or {} 2890 ).eval(config), 2891 refresh_request_body=InterpolatedMapping( 2892 model.refresh_request_body or {}, parameters=model.parameters or {} 2893 ).eval(config), 2894 refresh_request_headers=InterpolatedMapping( 2895 model.refresh_request_headers or {}, parameters=model.parameters or {} 2896 ).eval(config), 2897 send_refresh_request_as_query_params=bool( 2898 model.send_refresh_request_as_query_params 2899 ), 2900 scopes=model.scopes, 2901 token_expiry_date_format=model.token_expiry_date_format, 2902 token_expiry_is_time_of_expiration=bool(model.token_expiry_date_format), 2903 message_repository=self._message_repository, 2904 refresh_token_error_status_codes=refresh_token_error_status_codes, 2905 refresh_token_error_key=refresh_token_error_key, 2906 refresh_token_error_values=refresh_token_error_values, 2907 ) 2908 # ignore type error because fixing it would have a lot of dependencies, revisit later 2909 return DeclarativeOauth2Authenticator( # type: ignore 2910 access_token_name=model.access_token_name or "access_token", 2911 access_token_value=model.access_token_value, 2912 client_id_name=model.client_id_name or "client_id", 2913 client_id=model.client_id, 2914 client_secret_name=model.client_secret_name or "client_secret", 2915 client_secret=model.client_secret, 2916 expires_in_name=model.expires_in_name or "expires_in", 2917 grant_type_name=model.grant_type_name or "grant_type", 2918 grant_type=model.grant_type or "refresh_token", 2919 refresh_request_body=model.refresh_request_body, 2920 refresh_request_headers=model.refresh_request_headers, 2921 send_refresh_request_as_query_params=bool(model.send_refresh_request_as_query_params), 2922 refresh_token_name=model.refresh_token_name or "refresh_token", 2923 refresh_token=model.refresh_token, 2924 scopes=model.scopes, 2925 token_expiry_date=model.token_expiry_date, 2926 token_expiry_date_format=model.token_expiry_date_format, 2927 token_expiry_is_time_of_expiration=bool(model.token_expiry_date_format), 2928 token_refresh_endpoint=model.token_refresh_endpoint, 2929 config=config, 2930 parameters=model.parameters or {}, 2931 message_repository=self._message_repository, 2932 profile_assertion=profile_assertion, 2933 use_profile_assertion=model.use_profile_assertion, 2934 refresh_token_error_status_codes=refresh_token_error_status_codes, 2935 refresh_token_error_key=refresh_token_error_key, 2936 refresh_token_error_values=refresh_token_error_values, 2937 )
def
create_offset_increment( self, model: airbyte_cdk.sources.declarative.models.declarative_component_schema.OffsetIncrement, config: Mapping[str, Any], decoder: airbyte_cdk.Decoder, extractor_model: Union[airbyte_cdk.sources.declarative.models.declarative_component_schema.DpathExtractor, airbyte_cdk.sources.declarative.models.declarative_component_schema.CustomRecordExtractor, NoneType] = None, **kwargs: Any) -> airbyte_cdk.OffsetIncrement:
2987 def create_offset_increment( 2988 self, 2989 model: OffsetIncrementModel, 2990 config: Config, 2991 decoder: Decoder, 2992 extractor_model: Optional[Union[CustomRecordExtractorModel, DpathExtractorModel]] = None, 2993 **kwargs: Any, 2994 ) -> OffsetIncrement: 2995 if isinstance(decoder, PaginationDecoderDecorator): 2996 inner_decoder = decoder.decoder 2997 else: 2998 inner_decoder = decoder 2999 decoder = PaginationDecoderDecorator(decoder=decoder) 3000 3001 if self._is_supported_decoder_for_pagination(inner_decoder): 3002 decoder_to_use = decoder 3003 else: 3004 raise ValueError( 3005 self._UNSUPPORTED_DECODER_ERROR.format(decoder_type=type(inner_decoder)) 3006 ) 3007 3008 # Ideally we would instantiate the runtime extractor from highest most level (in this case the SimpleRetriever) 3009 # so that it can be shared by OffSetIncrement and RecordSelector. However, due to how we instantiate the 3010 # decoder with various decorators here, but not in create_record_selector, it is simpler to retain existing 3011 # behavior by having two separate extractors with identical behavior since they use the same extractor model. 3012 # When we have more time to investigate we can look into reusing the same component. 3013 extractor = ( 3014 self._create_component_from_model( 3015 model=extractor_model, config=config, decoder=decoder_to_use 3016 ) 3017 if extractor_model 3018 else None 3019 ) 3020 3021 # Pydantic v1 Union type coercion can convert int to string depending on Union order. 3022 # If page_size is a string that represents an integer (not an interpolation), convert it back. 3023 page_size = model.page_size 3024 if isinstance(page_size, str) and page_size.isdigit(): 3025 page_size = int(page_size) 3026 3027 return OffsetIncrement( 3028 page_size=page_size, 3029 config=config, 3030 decoder=decoder_to_use, 3031 extractor=extractor, 3032 inject_on_first_request=model.inject_on_first_request or False, 3033 parameters=model.parameters or {}, 3034 )
def
create_page_increment( self, model: airbyte_cdk.sources.declarative.models.declarative_component_schema.PageIncrement, config: Mapping[str, Any], decoder: Optional[airbyte_cdk.Decoder] = None, extractor_model: Union[airbyte_cdk.sources.declarative.models.declarative_component_schema.DpathExtractor, airbyte_cdk.sources.declarative.models.declarative_component_schema.CustomRecordExtractor, NoneType] = None, **kwargs: Any) -> airbyte_cdk.PageIncrement:
3036 def create_page_increment( 3037 self, 3038 model: PageIncrementModel, 3039 config: Config, 3040 decoder: Optional[Decoder] = None, 3041 extractor_model: Optional[Union[CustomRecordExtractorModel, DpathExtractorModel]] = None, 3042 **kwargs: Any, 3043 ) -> PageIncrement: 3044 # Like OffsetIncrement, we instantiate a separate extractor with identical behavior to the 3045 # RecordSelector's so the strategy can count the raw records in the response. This ensures 3046 # pagination is driven by the API's page size, not the post-filter record count. 3047 extractor = ( 3048 self._create_component_from_model(model=extractor_model, config=config, decoder=decoder) 3049 if extractor_model 3050 else None 3051 ) 3052 3053 # Pydantic v1 Union type coercion can convert int to string depending on Union order. 3054 # If page_size is a string that represents an integer (not an interpolation), convert it back. 3055 page_size = model.page_size 3056 if isinstance(page_size, str) and page_size.isdigit(): 3057 page_size = int(page_size) 3058 3059 return PageIncrement( 3060 page_size=page_size, 3061 config=config, 3062 start_from_page=model.start_from_page or 0, 3063 inject_on_first_request=model.inject_on_first_request or False, 3064 extractor=extractor, 3065 parameters=model.parameters or {}, 3066 )
def
create_parent_stream_config( self, model: airbyte_cdk.sources.declarative.models.declarative_component_schema.ParentStreamConfig, config: Mapping[str, Any], *, stream_name: str, **kwargs: Any) -> airbyte_cdk.ParentStreamConfig:
3068 def create_parent_stream_config( 3069 self, model: ParentStreamConfigModel, config: Config, *, stream_name: str, **kwargs: Any 3070 ) -> ParentStreamConfig: 3071 declarative_stream = self._create_component_from_model( 3072 model.stream, 3073 config=config, 3074 is_parent=True, 3075 **kwargs, 3076 ) 3077 request_option = ( 3078 self._create_component_from_model(model.request_option, config=config) 3079 if model.request_option 3080 else None 3081 ) 3082 3083 if model.lazy_read_pointer and any("*" in pointer for pointer in model.lazy_read_pointer): 3084 raise ValueError( 3085 "The '*' wildcard in 'lazy_read_pointer' is not supported — only direct paths are allowed." 3086 ) 3087 3088 model_lazy_read_pointer: List[Union[InterpolatedString, str]] = ( 3089 [x for x in model.lazy_read_pointer] if model.lazy_read_pointer else [] 3090 ) 3091 3092 return ParentStreamConfig( 3093 parent_key=model.parent_key, 3094 request_option=request_option, 3095 stream=declarative_stream, 3096 partition_field=model.partition_field, 3097 config=config, 3098 incremental_dependency=model.incremental_dependency or False, 3099 parameters=model.parameters or {}, 3100 extra_fields=model.extra_fields, 3101 lazy_read_pointer=model_lazy_read_pointer, 3102 )
def
create_properties_from_endpoint( self, model: airbyte_cdk.sources.declarative.models.declarative_component_schema.PropertiesFromEndpoint, config: Mapping[str, Any], **kwargs: Any) -> airbyte_cdk.sources.declarative.requesters.query_properties.PropertiesFromEndpoint:
3104 def create_properties_from_endpoint( 3105 self, model: PropertiesFromEndpointModel, config: Config, **kwargs: Any 3106 ) -> PropertiesFromEndpoint: 3107 retriever = self._create_component_from_model( 3108 model=model.retriever, 3109 config=config, 3110 name="dynamic_properties", 3111 primary_key=None, 3112 stream_slicer=None, 3113 transformations=[], 3114 use_cache=True, # Enable caching on the HttpRequester/HttpClient because the properties endpoint will be called for every slice being processed, and it is highly unlikely for the response to different 3115 ) 3116 return PropertiesFromEndpoint( 3117 property_field_path=model.property_field_path, 3118 retriever=retriever, 3119 config=config, 3120 parameters=model.parameters or {}, 3121 )
def
create_property_chunking( self, model: airbyte_cdk.sources.declarative.models.declarative_component_schema.PropertyChunking, config: Mapping[str, Any], **kwargs: Any) -> airbyte_cdk.sources.declarative.requesters.query_properties.PropertyChunking:
3123 def create_property_chunking( 3124 self, model: PropertyChunkingModel, config: Config, **kwargs: Any 3125 ) -> PropertyChunking: 3126 record_merge_strategy = ( 3127 self._create_component_from_model( 3128 model=model.record_merge_strategy, config=config, **kwargs 3129 ) 3130 if model.record_merge_strategy 3131 else None 3132 ) 3133 3134 property_limit_type: PropertyLimitType 3135 match model.property_limit_type: 3136 case PropertyLimitTypeModel.property_count: 3137 property_limit_type = PropertyLimitType.property_count 3138 case PropertyLimitTypeModel.characters: 3139 property_limit_type = PropertyLimitType.characters 3140 case _: 3141 raise ValueError(f"Invalid PropertyLimitType {property_limit_type}") 3142 3143 return PropertyChunking( 3144 property_limit_type=property_limit_type, 3145 property_limit=model.property_limit, 3146 record_merge_strategy=record_merge_strategy, 3147 config=config, 3148 parameters=model.parameters or {}, 3149 )
def
create_query_properties( self, model: airbyte_cdk.sources.declarative.models.declarative_component_schema.QueryProperties, config: Mapping[str, Any], *, stream_name: str, **kwargs: Any) -> airbyte_cdk.sources.declarative.requesters.query_properties.QueryProperties:
3151 def create_query_properties( 3152 self, model: QueryPropertiesModel, config: Config, *, stream_name: str, **kwargs: Any 3153 ) -> QueryProperties: 3154 if isinstance(model.property_list, list): 3155 property_list = model.property_list 3156 else: 3157 property_list = self._create_component_from_model( 3158 model=model.property_list, config=config, **kwargs 3159 ) 3160 3161 property_chunking = ( 3162 self._create_component_from_model( 3163 model=model.property_chunking, config=config, **kwargs 3164 ) 3165 if model.property_chunking 3166 else None 3167 ) 3168 3169 property_selector = ( 3170 self._create_component_from_model( 3171 model=model.property_selector, config=config, stream_name=stream_name, **kwargs 3172 ) 3173 if model.property_selector 3174 else None 3175 ) 3176 3177 return QueryProperties( 3178 property_list=property_list, 3179 always_include_properties=model.always_include_properties, 3180 property_chunking=property_chunking, 3181 property_selector=property_selector, 3182 config=config, 3183 parameters=model.parameters or {}, 3184 )
def
create_json_schema_property_selector( self, model: airbyte_cdk.sources.declarative.models.declarative_component_schema.JsonSchemaPropertySelector, config: Mapping[str, Any], *, stream_name: str, **kwargs: Any) -> airbyte_cdk.sources.declarative.requesters.query_properties.property_selector.JsonSchemaPropertySelector:
3186 def create_json_schema_property_selector( 3187 self, 3188 model: JsonSchemaPropertySelectorModel, 3189 config: Config, 3190 *, 3191 stream_name: str, 3192 **kwargs: Any, 3193 ) -> JsonSchemaPropertySelector: 3194 configured_stream = self._stream_name_to_configured_stream.get(stream_name) 3195 3196 transformations = [] 3197 if model.transformations: 3198 for transformation_model in model.transformations: 3199 transformations.append( 3200 self._create_component_from_model(model=transformation_model, config=config) 3201 ) 3202 3203 return JsonSchemaPropertySelector( 3204 configured_stream=configured_stream, 3205 properties_transformations=transformations, 3206 config=config, 3207 parameters=model.parameters or {}, 3208 )
@staticmethod
def
create_record_filter( model: airbyte_cdk.sources.declarative.models.declarative_component_schema.RecordFilter, config: Mapping[str, Any], **kwargs: Any) -> airbyte_cdk.RecordFilter:
@staticmethod
def
create_request_path( model: airbyte_cdk.sources.declarative.models.declarative_component_schema.RequestPath, config: Mapping[str, Any], **kwargs: Any) -> airbyte_cdk.sources.declarative.requesters.request_path.RequestPath:
@staticmethod
def
create_request_option( model: airbyte_cdk.sources.declarative.models.declarative_component_schema.RequestOption, config: Mapping[str, Any], **kwargs: Any) -> airbyte_cdk.RequestOption:
3222 @staticmethod 3223 def create_request_option( 3224 model: RequestOptionModel, config: Config, **kwargs: Any 3225 ) -> RequestOption: 3226 inject_into = RequestOptionType(model.inject_into.value) 3227 field_path: Optional[List[Union[InterpolatedString, str]]] = ( 3228 [ 3229 InterpolatedString.create(segment, parameters=kwargs.get("parameters", {})) 3230 for segment in model.field_path 3231 ] 3232 if model.field_path 3233 else None 3234 ) 3235 field_name = ( 3236 InterpolatedString.create(model.field_name, parameters=kwargs.get("parameters", {})) 3237 if model.field_name 3238 else None 3239 ) 3240 return RequestOption( 3241 field_name=field_name, 3242 field_path=field_path, 3243 inject_into=inject_into, 3244 parameters=kwargs.get("parameters", {}), 3245 )
def
create_record_selector( self, model: airbyte_cdk.sources.declarative.models.declarative_component_schema.RecordSelector, config: Mapping[str, Any], *, name: str, transformations: Optional[List[airbyte_cdk.RecordTransformation]] = None, decoder: airbyte_cdk.Decoder | None = None, client_side_incremental_sync_cursor: Optional[airbyte_cdk.Cursor] = None, file_uploader: Optional[airbyte_cdk.sources.declarative.retrievers.file_uploader.DefaultFileUploader] = None, **kwargs: Any) -> airbyte_cdk.RecordSelector:
3247 def create_record_selector( 3248 self, 3249 model: RecordSelectorModel, 3250 config: Config, 3251 *, 3252 name: str, 3253 transformations: List[RecordTransformation] | None = None, 3254 decoder: Decoder | None = None, 3255 client_side_incremental_sync_cursor: Optional[Cursor] = None, 3256 file_uploader: Optional[DefaultFileUploader] = None, 3257 **kwargs: Any, 3258 ) -> RecordSelector: 3259 extractor = self._create_component_from_model( 3260 model=model.extractor, decoder=decoder, config=config 3261 ) 3262 record_filter = ( 3263 self._create_component_from_model(model.record_filter, config=config) 3264 if model.record_filter 3265 else None 3266 ) 3267 3268 transform_before_filtering = ( 3269 False if model.transform_before_filtering is None else model.transform_before_filtering 3270 ) 3271 if client_side_incremental_sync_cursor: 3272 record_filter = ClientSideIncrementalRecordFilterDecorator( 3273 config=config, 3274 parameters=model.parameters, 3275 condition=model.record_filter.condition 3276 if (model.record_filter and hasattr(model.record_filter, "condition")) 3277 else None, 3278 cursor=client_side_incremental_sync_cursor, 3279 ) 3280 transform_before_filtering = ( 3281 True 3282 if model.transform_before_filtering is None 3283 else model.transform_before_filtering 3284 ) 3285 3286 if model.schema_normalization is None: 3287 # default to no schema normalization if not set 3288 model.schema_normalization = SchemaNormalizationModel.None_ 3289 3290 schema_normalization = ( 3291 TypeTransformer(SCHEMA_TRANSFORMER_TYPE_MAPPING[model.schema_normalization]) 3292 if isinstance(model.schema_normalization, SchemaNormalizationModel) 3293 else self._create_component_from_model(model.schema_normalization, config=config) # type: ignore[arg-type] # custom normalization model expected here 3294 ) 3295 3296 return RecordSelector( 3297 extractor=extractor, 3298 name=name, 3299 config=config, 3300 record_filter=record_filter, 3301 transformations=transformations or [], 3302 file_uploader=file_uploader, 3303 schema_normalization=schema_normalization, 3304 parameters=model.parameters or {}, 3305 transform_before_filtering=transform_before_filtering, 3306 )
@staticmethod
def
create_remove_fields( model: airbyte_cdk.sources.declarative.models.declarative_component_schema.RemoveFields, config: Mapping[str, Any], **kwargs: Any) -> airbyte_cdk.sources.declarative.transformations.RemoveFields:
def
create_selective_authenticator( self, model: airbyte_cdk.sources.declarative.models.declarative_component_schema.SelectiveAuthenticator, config: Mapping[str, Any], **kwargs: Any) -> airbyte_cdk.DeclarativeAuthenticator:
3316 def create_selective_authenticator( 3317 self, model: SelectiveAuthenticatorModel, config: Config, **kwargs: Any 3318 ) -> DeclarativeAuthenticator: 3319 authenticators = { 3320 name: self._create_component_from_model(model=auth, config=config) 3321 for name, auth in model.authenticators.items() 3322 } 3323 # SelectiveAuthenticator will return instance of DeclarativeAuthenticator or raise ValueError error 3324 return SelectiveAuthenticator( # type: ignore[abstract] 3325 config=config, 3326 authenticators=authenticators, 3327 authenticator_selection_path=model.authenticator_selection_path, 3328 **kwargs, 3329 )
@staticmethod
def
create_legacy_session_token_authenticator( model: airbyte_cdk.sources.declarative.models.declarative_component_schema.LegacySessionTokenAuthenticator, config: Mapping[str, Any], *, url_base: str, **kwargs: Any) -> airbyte_cdk.sources.declarative.auth.token.LegacySessionTokenAuthenticator:
3331 @staticmethod 3332 def create_legacy_session_token_authenticator( 3333 model: LegacySessionTokenAuthenticatorModel, config: Config, *, url_base: str, **kwargs: Any 3334 ) -> LegacySessionTokenAuthenticator: 3335 return LegacySessionTokenAuthenticator( 3336 api_url=url_base, 3337 header=model.header, 3338 login_url=model.login_url, 3339 password=model.password or "", 3340 session_token=model.session_token or "", 3341 session_token_response_key=model.session_token_response_key or "", 3342 username=model.username or "", 3343 validate_session_url=model.validate_session_url, 3344 config=config, 3345 parameters=model.parameters or {}, 3346 )
def
create_simple_retriever( self, model: airbyte_cdk.sources.declarative.models.declarative_component_schema.SimpleRetriever, config: Mapping[str, Any], *, name: str, primary_key: Union[str, List[str], List[List[str]], NoneType], request_options_provider: Optional[airbyte_cdk.sources.declarative.requesters.request_options.RequestOptionsProvider] = None, cursor: Optional[airbyte_cdk.Cursor] = None, has_stop_condition_cursor: bool = False, is_client_side_incremental_sync: bool = False, transformations: List[airbyte_cdk.RecordTransformation], file_uploader: Optional[airbyte_cdk.sources.declarative.retrievers.file_uploader.DefaultFileUploader] = None, incremental_sync: Union[airbyte_cdk.sources.declarative.models.declarative_component_schema.DatetimeBasedCursor, airbyte_cdk.sources.declarative.models.declarative_component_schema.IncrementingCountCursor, NoneType] = None, use_cache: Optional[bool] = None, log_formatter: Optional[Callable[[requests.models.Response], Any]] = None, partition_router: Optional[airbyte_cdk.sources.declarative.partition_routers.PartitionRouter] = None, **kwargs: Any) -> airbyte_cdk.SimpleRetriever:
3348 def create_simple_retriever( 3349 self, 3350 model: SimpleRetrieverModel, 3351 config: Config, 3352 *, 3353 name: str, 3354 primary_key: Optional[Union[str, List[str], List[List[str]]]], 3355 request_options_provider: Optional[RequestOptionsProvider] = None, 3356 cursor: Optional[Cursor] = None, 3357 has_stop_condition_cursor: bool = False, 3358 is_client_side_incremental_sync: bool = False, 3359 transformations: List[RecordTransformation], 3360 file_uploader: Optional[DefaultFileUploader] = None, 3361 incremental_sync: Optional[ 3362 Union[IncrementingCountCursorModel, DatetimeBasedCursorModel] 3363 ] = None, 3364 use_cache: Optional[bool] = None, 3365 log_formatter: Optional[Callable[[Response], Any]] = None, 3366 partition_router: Optional[PartitionRouter] = None, 3367 **kwargs: Any, 3368 ) -> SimpleRetriever: 3369 def _get_url(req: Requester) -> str: 3370 """ 3371 Closure to get the URL from the requester. This is used to get the URL in the case of a lazy retriever. 3372 This is needed because the URL is not set until the requester is created. 3373 """ 3374 3375 _url: str = ( 3376 model.requester.url 3377 if hasattr(model.requester, "url") and model.requester.url is not None 3378 else req.get_url(stream_state=None, stream_slice=None, next_page_token=None) 3379 ) 3380 _url_base: str = ( 3381 model.requester.url_base 3382 if hasattr(model.requester, "url_base") and model.requester.url_base is not None 3383 else req.get_url_base(stream_state=None, stream_slice=None, next_page_token=None) 3384 ) 3385 3386 return _url or _url_base 3387 3388 if cursor is None: 3389 cursor = FinalStateCursor(name, None, self._message_repository) 3390 3391 decoder = ( 3392 self._create_component_from_model(model=model.decoder, config=config) 3393 if model.decoder 3394 else JsonDecoder(parameters={}) 3395 ) 3396 record_selector = self._create_component_from_model( 3397 model=model.record_selector, 3398 name=name, 3399 config=config, 3400 decoder=decoder, 3401 transformations=transformations, 3402 client_side_incremental_sync_cursor=cursor if is_client_side_incremental_sync else None, 3403 file_uploader=file_uploader, 3404 ) 3405 3406 query_properties: Optional[QueryProperties] = None 3407 query_properties_key: Optional[str] = None 3408 self._ensure_query_properties_to_model(model.requester) 3409 if self._has_query_properties_in_request_parameters(model.requester): 3410 # It is better to be explicit about an error if PropertiesFromEndpoint is defined in multiple 3411 # places instead of default to request_parameters which isn't clearly documented 3412 if ( 3413 hasattr(model.requester, "fetch_properties_from_endpoint") 3414 and model.requester.fetch_properties_from_endpoint 3415 ): 3416 raise ValueError( 3417 f"PropertiesFromEndpoint should only be specified once per stream, but found in {model.requester.type}.fetch_properties_from_endpoint and {model.requester.type}.request_parameters" 3418 ) 3419 3420 query_properties_definitions = [] 3421 for key, request_parameter in model.requester.request_parameters.items(): # type: ignore # request_parameters is already validated to be a Mapping using _has_query_properties_in_request_parameters() 3422 if isinstance(request_parameter, QueryPropertiesModel): 3423 query_properties_key = key 3424 query_properties_definitions.append(request_parameter) 3425 3426 if len(query_properties_definitions) > 1: 3427 raise ValueError( 3428 f"request_parameters only supports defining one QueryProperties field, but found {len(query_properties_definitions)} usages" 3429 ) 3430 3431 if len(query_properties_definitions) == 1: 3432 query_properties = self._create_component_from_model( 3433 model=query_properties_definitions[0], stream_name=name, config=config 3434 ) 3435 3436 # Removes QueryProperties components from the interpolated mappings because it has been designed 3437 # to be used by the SimpleRetriever and will be resolved from the provider from the slice directly 3438 # instead of through jinja interpolation 3439 if hasattr(model.requester, "request_parameters") and isinstance( 3440 model.requester.request_parameters, Mapping 3441 ): 3442 model.requester.request_parameters = self._remove_query_properties( 3443 model.requester.request_parameters 3444 ) 3445 elif ( 3446 hasattr(model.requester, "fetch_properties_from_endpoint") 3447 and model.requester.fetch_properties_from_endpoint 3448 ): 3449 # todo: Deprecate this condition once dependent connectors migrate to query_properties 3450 query_properties_definition = QueryPropertiesModel( 3451 type="QueryProperties", 3452 property_list=model.requester.fetch_properties_from_endpoint, 3453 always_include_properties=None, 3454 property_chunking=None, 3455 ) # type: ignore # $parameters has a default value 3456 3457 query_properties = self.create_query_properties( 3458 model=query_properties_definition, 3459 stream_name=name, 3460 config=config, 3461 ) 3462 elif hasattr(model.requester, "query_properties") and model.requester.query_properties: 3463 query_properties = self.create_query_properties( 3464 model=model.requester.query_properties, 3465 stream_name=name, 3466 config=config, 3467 ) 3468 3469 requester = self._create_component_from_model( 3470 model=model.requester, 3471 decoder=decoder, 3472 name=name, 3473 query_properties_key=query_properties_key, 3474 use_cache=use_cache, 3475 config=config, 3476 ) 3477 3478 if not request_options_provider: 3479 request_options_provider = DefaultRequestOptionsProvider(parameters={}) 3480 if isinstance(request_options_provider, DefaultRequestOptionsProvider) and isinstance( 3481 partition_router, PartitionRouter 3482 ): 3483 request_options_provider = partition_router 3484 3485 paginator = ( 3486 self._create_component_from_model( 3487 model=model.paginator, 3488 config=config, 3489 url_base=_get_url(requester), 3490 extractor_model=model.record_selector.extractor, 3491 decoder=decoder, 3492 cursor_used_for_stop_condition=cursor if has_stop_condition_cursor else None, 3493 ) 3494 if model.paginator 3495 else NoPagination(parameters={}) 3496 ) 3497 3498 ignore_stream_slicer_parameters_on_paginated_requests = ( 3499 model.ignore_stream_slicer_parameters_on_paginated_requests or False 3500 ) 3501 3502 if ( 3503 model.partition_router 3504 and isinstance(model.partition_router, SubstreamPartitionRouterModel) 3505 and not bool(self._connector_state_manager.get_stream_state(name, None)) 3506 and any( 3507 parent_stream_config.lazy_read_pointer 3508 for parent_stream_config in model.partition_router.parent_stream_configs 3509 ) 3510 ): 3511 if incremental_sync: 3512 if incremental_sync.type != "DatetimeBasedCursor": 3513 raise ValueError( 3514 f"LazySimpleRetriever only supports DatetimeBasedCursor. Found: {incremental_sync.type}." 3515 ) 3516 3517 elif incremental_sync.step or incremental_sync.cursor_granularity: 3518 raise ValueError( 3519 f"Found more that one slice per parent. LazySimpleRetriever only supports single slice read for stream - {name}." 3520 ) 3521 3522 if model.decoder and model.decoder.type != "JsonDecoder": 3523 raise ValueError( 3524 f"LazySimpleRetriever only supports JsonDecoder. Found: {model.decoder.type}." 3525 ) 3526 3527 return LazySimpleRetriever( 3528 name=name, 3529 paginator=paginator, 3530 primary_key=primary_key, 3531 requester=requester, 3532 record_selector=record_selector, 3533 stream_slicer=_NO_STREAM_SLICING, 3534 request_option_provider=request_options_provider, 3535 config=config, 3536 ignore_stream_slicer_parameters_on_paginated_requests=ignore_stream_slicer_parameters_on_paginated_requests, 3537 parameters=model.parameters or {}, 3538 ) 3539 3540 if ( 3541 model.record_selector.record_filter 3542 and model.pagination_reset 3543 and model.pagination_reset.limits 3544 ): 3545 raise ValueError("PaginationResetLimits are not supported while having record filter.") 3546 3547 return SimpleRetriever( 3548 name=name, 3549 paginator=paginator, 3550 primary_key=primary_key, 3551 requester=requester, 3552 record_selector=record_selector, 3553 stream_slicer=_NO_STREAM_SLICING, 3554 request_option_provider=request_options_provider, 3555 config=config, 3556 ignore_stream_slicer_parameters_on_paginated_requests=ignore_stream_slicer_parameters_on_paginated_requests, 3557 additional_query_properties=query_properties, 3558 log_formatter=self._get_log_formatter(log_formatter, name), 3559 pagination_tracker_factory=self._create_pagination_tracker_factory( 3560 model.pagination_reset, cursor 3561 ), 3562 parameters=model.parameters or {}, 3563 )
def
create_state_delegating_stream( self, model: airbyte_cdk.sources.declarative.models.declarative_component_schema.StateDelegatingStream, config: Mapping[str, Any], **kwargs: Any) -> airbyte_cdk.sources.streams.concurrent.default_stream.DefaultStream:
3641 def create_state_delegating_stream( 3642 self, 3643 model: StateDelegatingStreamModel, 3644 config: Config, 3645 **kwargs: Any, 3646 ) -> DefaultStream: 3647 if ( 3648 model.full_refresh_stream.name != model.name 3649 or model.name != model.incremental_stream.name 3650 ): 3651 raise ValueError( 3652 f"state_delegating_stream, full_refresh_stream name and incremental_stream must have equal names. Instead has {model.name}, {model.full_refresh_stream.name} and {model.incremental_stream.name}." 3653 ) 3654 3655 # Resolve api_retention_period with config context (supports Jinja2 interpolation) 3656 resolved_retention_period: Optional[str] = None 3657 if model.api_retention_period: 3658 interpolated_retention = InterpolatedString.create( 3659 model.api_retention_period, parameters=model.parameters or {} 3660 ) 3661 resolved_value = interpolated_retention.eval(config=config) 3662 if resolved_value: 3663 resolved_retention_period = str(resolved_value) 3664 3665 if resolved_retention_period: 3666 for stream_model in (model.full_refresh_stream, model.incremental_stream): 3667 if isinstance(stream_model.incremental_sync, IncrementingCountCursorModel): 3668 raise ValueError( 3669 f"Stream '{model.name}' uses IncrementingCountCursor which is not supported " 3670 f"with api_retention_period. IncrementingCountCursor does not use datetime-based " 3671 f"cursors, so cursor age validation cannot be performed." 3672 ) 3673 3674 stream_state = self._connector_state_manager.get_stream_state(model.name, None) 3675 3676 if not stream_state: 3677 return self._create_component_from_model( # type: ignore[no-any-return] 3678 model.full_refresh_stream, config=config, **kwargs 3679 ) 3680 3681 incremental_stream: DefaultStream = self._create_component_from_model( 3682 model.incremental_stream, config=config, **kwargs 3683 ) # type: ignore[assignment] 3684 3685 # Only run cursor age validation for streams that are in the configured 3686 # catalog (or when no catalog was provided, e.g. during discover / connector 3687 # builder). Streams not selected by the user but instantiated as parent-stream 3688 # dependencies must not go through this path because it emits state messages 3689 # that the destination does not know about, causing "Stream not found" crashes. 3690 stream_is_in_catalog = ( 3691 not self._stream_name_to_configured_stream # no catalog → validate by default 3692 or model.name in self._stream_name_to_configured_stream 3693 ) 3694 if resolved_retention_period and stream_is_in_catalog: 3695 full_refresh_stream: DefaultStream = self._create_component_from_model( 3696 model.full_refresh_stream, config=config, **kwargs 3697 ) # type: ignore[assignment] 3698 if self._is_cursor_older_than_retention_period( 3699 stream_state, 3700 full_refresh_stream.cursor, 3701 incremental_stream.cursor, 3702 resolved_retention_period, 3703 model.name, 3704 ): 3705 # Clear state BEFORE constructing the full_refresh_stream so that 3706 # its cursor starts from start_date instead of the stale cursor. 3707 self._connector_state_manager.update_state_for_stream(model.name, None, {}) 3708 state_message = self._connector_state_manager.create_state_message(model.name, None) 3709 self._message_repository.emit_message(state_message) 3710 return self._create_component_from_model( # type: ignore[no-any-return] 3711 model.full_refresh_stream, config=config, **kwargs 3712 ) 3713 3714 return incremental_stream
def
create_async_retriever( self, model: airbyte_cdk.sources.declarative.models.declarative_component_schema.AsyncRetriever, config: Mapping[str, Any], *, name: str, primary_key: Union[str, List[str], List[List[str]], NoneType], stream_slicer: Optional[airbyte_cdk.sources.declarative.stream_slicers.StreamSlicer], client_side_incremental_sync: Optional[Dict[str, Any]] = None, transformations: List[airbyte_cdk.RecordTransformation], **kwargs: Any) -> airbyte_cdk.sources.declarative.retrievers.AsyncRetriever:
3813 def create_async_retriever( 3814 self, 3815 model: AsyncRetrieverModel, 3816 config: Config, 3817 *, 3818 name: str, 3819 primary_key: Optional[ 3820 Union[str, List[str], List[List[str]]] 3821 ], # this seems to be needed to match create_simple_retriever 3822 stream_slicer: Optional[StreamSlicer], 3823 client_side_incremental_sync: Optional[Dict[str, Any]] = None, 3824 transformations: List[RecordTransformation], 3825 **kwargs: Any, 3826 ) -> AsyncRetriever: 3827 if model.download_target_requester and not model.download_target_extractor: 3828 raise ValueError( 3829 f"`download_target_extractor` required if using a `download_target_requester`" 3830 ) 3831 3832 def _get_download_retriever( 3833 requester: Requester, extractor: RecordExtractor, _decoder: Decoder 3834 ) -> SimpleRetriever: 3835 # We create a record selector for the download retriever 3836 # with no schema normalization and no transformations, neither record filter 3837 # as all this occurs in the record_selector of the AsyncRetriever 3838 record_selector = RecordSelector( 3839 extractor=extractor, 3840 name=name, 3841 record_filter=None, 3842 transformations=[], 3843 schema_normalization=TypeTransformer(TransformConfig.NoTransform), 3844 config=config, 3845 parameters={}, 3846 ) 3847 paginator = ( 3848 self._create_component_from_model( 3849 model=model.download_paginator, 3850 decoder=_decoder, 3851 config=config, 3852 url_base="", 3853 ) 3854 if model.download_paginator 3855 else NoPagination(parameters={}) 3856 ) 3857 3858 return SimpleRetriever( 3859 requester=requester, 3860 record_selector=record_selector, 3861 primary_key=None, 3862 name=name, 3863 paginator=paginator, 3864 config=config, 3865 parameters={}, 3866 log_formatter=self._get_log_formatter(None, name), 3867 ) 3868 3869 def _get_job_timeout() -> datetime.timedelta: 3870 user_defined_timeout: Optional[int] = ( 3871 int( 3872 InterpolatedString.create( 3873 str(model.polling_job_timeout), 3874 parameters={}, 3875 ).eval(config) 3876 ) 3877 if model.polling_job_timeout 3878 else None 3879 ) 3880 3881 # check for user defined timeout during the test read or 15 minutes 3882 test_read_timeout = datetime.timedelta(minutes=user_defined_timeout or 15) 3883 # default value for non-connector builder is 60 minutes. 3884 default_sync_timeout = datetime.timedelta(minutes=user_defined_timeout or 60) 3885 3886 return ( 3887 test_read_timeout if self._emit_connector_builder_messages else default_sync_timeout 3888 ) 3889 3890 decoder = ( 3891 self._create_component_from_model(model=model.decoder, config=config) 3892 if model.decoder 3893 else JsonDecoder(parameters={}) 3894 ) 3895 record_selector = self._create_component_from_model( 3896 model=model.record_selector, 3897 config=config, 3898 decoder=decoder, 3899 name=name, 3900 transformations=transformations, 3901 client_side_incremental_sync=client_side_incremental_sync, 3902 ) 3903 3904 stream_slicer = stream_slicer or SinglePartitionRouter(parameters={}) 3905 if self._should_limit_slices_fetched(): 3906 stream_slicer = cast( 3907 StreamSlicer, 3908 StreamSlicerTestReadDecorator( 3909 wrapped_slicer=stream_slicer, 3910 maximum_number_of_slices=self._limit_slices_fetched or 5, 3911 ), 3912 ) 3913 3914 creation_requester = self._create_component_from_model( 3915 model=model.creation_requester, 3916 decoder=decoder, 3917 config=config, 3918 name=f"job creation - {name}", 3919 ) 3920 polling_requester = self._create_component_from_model( 3921 model=model.polling_requester, 3922 decoder=decoder, 3923 config=config, 3924 name=f"job polling - {name}", 3925 ) 3926 job_download_components_name = f"job download - {name}" 3927 download_decoder = ( 3928 self._create_component_from_model(model=model.download_decoder, config=config) 3929 if model.download_decoder 3930 else JsonDecoder(parameters={}) 3931 ) 3932 download_extractor = ( 3933 self._create_component_from_model( 3934 model=model.download_extractor, 3935 config=config, 3936 decoder=download_decoder, 3937 parameters=model.parameters, 3938 ) 3939 if model.download_extractor 3940 else DpathExtractor( 3941 [], 3942 config=config, 3943 decoder=download_decoder, 3944 parameters=model.parameters or {}, 3945 ) 3946 ) 3947 download_requester = self._create_component_from_model( 3948 model=model.download_requester, 3949 decoder=download_decoder, 3950 config=config, 3951 name=job_download_components_name, 3952 ) 3953 download_retriever = _get_download_retriever( 3954 download_requester, download_extractor, download_decoder 3955 ) 3956 abort_requester = ( 3957 self._create_component_from_model( 3958 model=model.abort_requester, 3959 decoder=decoder, 3960 config=config, 3961 name=f"job abort - {name}", 3962 ) 3963 if model.abort_requester 3964 else None 3965 ) 3966 delete_requester = ( 3967 self._create_component_from_model( 3968 model=model.delete_requester, 3969 decoder=decoder, 3970 config=config, 3971 name=f"job delete - {name}", 3972 ) 3973 if model.delete_requester 3974 else None 3975 ) 3976 download_target_requester = ( 3977 self._create_component_from_model( 3978 model=model.download_target_requester, 3979 decoder=decoder, 3980 config=config, 3981 name=f"job extract_url - {name}", 3982 ) 3983 if model.download_target_requester 3984 else None 3985 ) 3986 status_extractor = self._create_component_from_model( 3987 model=model.status_extractor, decoder=decoder, config=config, name=name 3988 ) 3989 download_target_extractor = ( 3990 self._create_component_from_model( 3991 model=model.download_target_extractor, 3992 decoder=decoder, 3993 config=config, 3994 name=name, 3995 ) 3996 if model.download_target_extractor 3997 else None 3998 ) 3999 4000 job_repository: AsyncJobRepository = AsyncHttpJobRepository( 4001 creation_requester=creation_requester, 4002 polling_requester=polling_requester, 4003 download_retriever=download_retriever, 4004 download_target_requester=download_target_requester, 4005 abort_requester=abort_requester, 4006 delete_requester=delete_requester, 4007 status_extractor=status_extractor, 4008 status_mapping=self._create_async_job_status_mapping(model.status_mapping, config), 4009 download_target_extractor=download_target_extractor, 4010 job_timeout=_get_job_timeout(), 4011 ) 4012 4013 failed_retry_wait_time_in_seconds: Optional[int] = ( 4014 int( 4015 InterpolatedString.create( 4016 str(model.failed_retry_wait_time_in_seconds), 4017 parameters={}, 4018 ).eval(config) 4019 ) 4020 if model.failed_retry_wait_time_in_seconds 4021 else None 4022 ) 4023 4024 async_job_partition_router = AsyncJobPartitionRouter( 4025 job_orchestrator_factory=lambda stream_slices: AsyncJobOrchestrator( 4026 job_repository, 4027 stream_slices, 4028 self._job_tracker, 4029 self._message_repository, 4030 # FIXME work would need to be done here in order to detect if a stream as a parent stream that is bulk 4031 has_bulk_parent=False, 4032 # set the `job_max_retry` to 1 for the `Connector Builder`` use-case. 4033 # `None` == default retry is set to 3 attempts, under the hood. 4034 job_max_retry=1 if self._emit_connector_builder_messages else None, 4035 failed_retry_wait_time_in_seconds=failed_retry_wait_time_in_seconds, 4036 ), 4037 stream_slicer=stream_slicer, 4038 config=config, 4039 parameters=model.parameters or {}, 4040 ) 4041 4042 return AsyncRetriever( 4043 record_selector=record_selector, 4044 stream_slicer=async_job_partition_router, 4045 config=config, 4046 parameters=model.parameters or {}, 4047 )
def
create_spec( self, model: airbyte_cdk.sources.declarative.models.declarative_component_schema.Spec, config: Mapping[str, Any], **kwargs: Any) -> airbyte_cdk.sources.declarative.spec.Spec:
4049 def create_spec(self, model: SpecModel, config: Config, **kwargs: Any) -> Spec: 4050 config_migrations = [ 4051 self._create_component_from_model(migration, config) 4052 for migration in ( 4053 model.config_normalization_rules.config_migrations 4054 if ( 4055 model.config_normalization_rules 4056 and model.config_normalization_rules.config_migrations 4057 ) 4058 else [] 4059 ) 4060 ] 4061 config_transformations = [ 4062 self._create_component_from_model(transformation, config) 4063 for transformation in ( 4064 model.config_normalization_rules.transformations 4065 if ( 4066 model.config_normalization_rules 4067 and model.config_normalization_rules.transformations 4068 ) 4069 else [] 4070 ) 4071 ] 4072 config_validations = [ 4073 self._create_component_from_model(validation, config) 4074 for validation in ( 4075 model.config_normalization_rules.validations 4076 if ( 4077 model.config_normalization_rules 4078 and model.config_normalization_rules.validations 4079 ) 4080 else [] 4081 ) 4082 ] 4083 4084 return Spec( 4085 connection_specification=model.connection_specification, 4086 documentation_url=model.documentation_url, 4087 advanced_auth=model.advanced_auth, 4088 parameters={}, 4089 config_migrations=config_migrations, 4090 config_transformations=config_transformations, 4091 config_validations=config_validations, 4092 )
def
create_substream_partition_router( self, model: airbyte_cdk.sources.declarative.models.declarative_component_schema.SubstreamPartitionRouter, config: Mapping[str, Any], *, stream_name: str, **kwargs: Any) -> airbyte_cdk.SubstreamPartitionRouter:
4094 def create_substream_partition_router( 4095 self, 4096 model: SubstreamPartitionRouterModel, 4097 config: Config, 4098 *, 4099 stream_name: str, 4100 **kwargs: Any, 4101 ) -> SubstreamPartitionRouter: 4102 parent_stream_configs = [] 4103 if model.parent_stream_configs: 4104 parent_stream_configs.extend( 4105 [ 4106 self.create_parent_stream_config_with_substream_wrapper( 4107 model=parent_stream_config, config=config, stream_name=stream_name, **kwargs 4108 ) 4109 for parent_stream_config in model.parent_stream_configs 4110 ] 4111 ) 4112 4113 return SubstreamPartitionRouter( 4114 parent_stream_configs=parent_stream_configs, 4115 parameters=model.parameters or {}, 4116 config=config, 4117 )
def
create_parent_stream_config_with_substream_wrapper( self, model: airbyte_cdk.sources.declarative.models.declarative_component_schema.ParentStreamConfig, config: Mapping[str, Any], *, stream_name: str, **kwargs: Any) -> Any:
4119 def create_parent_stream_config_with_substream_wrapper( 4120 self, model: ParentStreamConfigModel, config: Config, *, stream_name: str, **kwargs: Any 4121 ) -> Any: 4122 child_state = self._connector_state_manager.get_stream_state(stream_name, None) 4123 4124 parent_state: Optional[Mapping[str, Any]] = ( 4125 child_state if model.incremental_dependency and child_state else None 4126 ) 4127 connector_state_manager = self._instantiate_parent_stream_state_manager( 4128 child_state, config, model, parent_state 4129 ) 4130 4131 substream_factory = ModelToComponentFactory( 4132 connector_state_manager=connector_state_manager, 4133 limit_pages_fetched_per_slice=self._limit_pages_fetched_per_slice, 4134 limit_slices_fetched=self._limit_slices_fetched, 4135 emit_connector_builder_messages=self._emit_connector_builder_messages, 4136 disable_retries=self._disable_retries, 4137 disable_cache=self._disable_cache, 4138 message_repository=StateFilteringMessageRepository( 4139 LogAppenderMessageRepositoryDecorator( 4140 { 4141 "airbyte_cdk": {"stream": {"is_substream": True}}, 4142 "http": {"is_auxiliary": True}, 4143 }, 4144 self._message_repository, 4145 self._evaluate_log_level(self._emit_connector_builder_messages), 4146 ), 4147 ), 4148 api_budget=self._api_budget, 4149 ) 4150 4151 return substream_factory.create_parent_stream_config( 4152 model=model, config=config, stream_name=stream_name, **kwargs 4153 )
@staticmethod
def
create_wait_time_from_header( model: airbyte_cdk.sources.declarative.models.declarative_component_schema.WaitTimeFromHeader, config: Mapping[str, Any], **kwargs: Any) -> airbyte_cdk.sources.declarative.requesters.error_handlers.backoff_strategies.WaitTimeFromHeaderBackoffStrategy:
4213 @staticmethod 4214 def create_wait_time_from_header( 4215 model: WaitTimeFromHeaderModel, config: Config, **kwargs: Any 4216 ) -> WaitTimeFromHeaderBackoffStrategy: 4217 return WaitTimeFromHeaderBackoffStrategy( 4218 header=model.header, 4219 parameters=model.parameters or {}, 4220 config=config, 4221 regex=model.regex, 4222 max_waiting_time_in_seconds=model.max_waiting_time_in_seconds 4223 if model.max_waiting_time_in_seconds is not None 4224 else None, 4225 )
@staticmethod
def
create_wait_until_time_from_header( model: airbyte_cdk.sources.declarative.models.declarative_component_schema.WaitUntilTimeFromHeader, config: Mapping[str, Any], **kwargs: Any) -> airbyte_cdk.sources.declarative.requesters.error_handlers.backoff_strategies.WaitUntilTimeFromHeaderBackoffStrategy:
4227 @staticmethod 4228 def create_wait_until_time_from_header( 4229 model: WaitUntilTimeFromHeaderModel, config: Config, **kwargs: Any 4230 ) -> WaitUntilTimeFromHeaderBackoffStrategy: 4231 return WaitUntilTimeFromHeaderBackoffStrategy( 4232 header=model.header, 4233 parameters=model.parameters or {}, 4234 config=config, 4235 min_wait=model.min_wait, 4236 regex=model.regex, 4237 )
@staticmethod
def
create_components_mapping_definition( model: airbyte_cdk.sources.declarative.models.declarative_component_schema.ComponentMappingDefinition, config: Mapping[str, Any], **kwargs: Any) -> airbyte_cdk.sources.declarative.resolvers.ComponentMappingDefinition:
4245 @staticmethod 4246 def create_components_mapping_definition( 4247 model: ComponentMappingDefinitionModel, config: Config, **kwargs: Any 4248 ) -> ComponentMappingDefinition: 4249 interpolated_value = InterpolatedString.create( 4250 model.value, parameters=model.parameters or {} 4251 ) 4252 field_path = [ 4253 InterpolatedString.create(path, parameters=model.parameters or {}) 4254 for path in model.field_path 4255 ] 4256 return ComponentMappingDefinition( 4257 field_path=field_path, # type: ignore[arg-type] # field_path can be str and InterpolatedString 4258 value=interpolated_value, 4259 value_type=ModelToComponentFactory._json_schema_type_name_to_type(model.value_type), 4260 create_or_update=model.create_or_update, 4261 condition=model.condition, 4262 parameters=model.parameters or {}, 4263 )
def
create_http_components_resolver( self, model: airbyte_cdk.sources.declarative.models.declarative_component_schema.HttpComponentsResolver, config: Mapping[str, Any], stream_name: Optional[str] = None) -> Any:
4265 def create_http_components_resolver( 4266 self, model: HttpComponentsResolverModel, config: Config, stream_name: Optional[str] = None 4267 ) -> Any: 4268 retriever = self._create_component_from_model( 4269 model=model.retriever, 4270 config=config, 4271 name=f"{stream_name if stream_name else '__http_components_resolver'}", 4272 primary_key=None, 4273 stream_slicer=self._build_stream_slicer_from_partition_router(model.retriever, config), 4274 transformations=[], 4275 ) 4276 4277 components_mapping = [] 4278 for component_mapping_definition_model in model.components_mapping: 4279 if component_mapping_definition_model.condition: 4280 raise ValueError("`condition` is only supported for `ConfigComponentsResolver`") 4281 components_mapping.append( 4282 self._create_component_from_model( 4283 model=component_mapping_definition_model, 4284 value_type=ModelToComponentFactory._json_schema_type_name_to_type( 4285 component_mapping_definition_model.value_type 4286 ), 4287 config=config, 4288 ) 4289 ) 4290 4291 return HttpComponentsResolver( 4292 retriever=retriever, 4293 stream_slicer=self._build_stream_slicer_from_partition_router(model.retriever, config), 4294 config=config, 4295 components_mapping=components_mapping, 4296 parameters=model.parameters or {}, 4297 )
@staticmethod
def
create_stream_config( model: airbyte_cdk.sources.declarative.models.declarative_component_schema.StreamConfig, config: Mapping[str, Any], **kwargs: Any) -> airbyte_cdk.sources.declarative.resolvers.StreamConfig:
4299 @staticmethod 4300 def create_stream_config( 4301 model: StreamConfigModel, config: Config, **kwargs: Any 4302 ) -> StreamConfig: 4303 model_configs_pointer: List[Union[InterpolatedString, str]] = ( 4304 [x for x in model.configs_pointer] if model.configs_pointer else [] 4305 ) 4306 4307 return StreamConfig( 4308 configs_pointer=model_configs_pointer, 4309 default_values=model.default_values, 4310 parameters=model.parameters or {}, 4311 )
def
create_config_components_resolver( self, model: airbyte_cdk.sources.declarative.models.declarative_component_schema.ConfigComponentsResolver, config: Mapping[str, Any]) -> Any:
4313 def create_config_components_resolver( 4314 self, 4315 model: ConfigComponentsResolverModel, 4316 config: Config, 4317 ) -> Any: 4318 model_stream_configs = ( 4319 model.stream_config if isinstance(model.stream_config, list) else [model.stream_config] 4320 ) 4321 4322 stream_configs = [ 4323 self._create_component_from_model( 4324 stream_config, config=config, parameters=model.parameters or {} 4325 ) 4326 for stream_config in model_stream_configs 4327 ] 4328 4329 components_mapping = [ 4330 self._create_component_from_model( 4331 model=components_mapping_definition_model, 4332 value_type=ModelToComponentFactory._json_schema_type_name_to_type( 4333 components_mapping_definition_model.value_type 4334 ), 4335 config=config, 4336 parameters=model.parameters, 4337 ) 4338 for components_mapping_definition_model in model.components_mapping 4339 ] 4340 4341 return ConfigComponentsResolver( 4342 stream_configs=stream_configs, 4343 config=config, 4344 components_mapping=components_mapping, 4345 parameters=model.parameters or {}, 4346 )
def
create_parametrized_components_resolver( self, model: airbyte_cdk.sources.declarative.models.declarative_component_schema.ParametrizedComponentsResolver, config: Mapping[str, Any]) -> airbyte_cdk.sources.declarative.resolvers.ParametrizedComponentsResolver:
4348 def create_parametrized_components_resolver( 4349 self, 4350 model: ParametrizedComponentsResolverModel, 4351 config: Config, 4352 ) -> ParametrizedComponentsResolver: 4353 stream_parameters = StreamParametersDefinition( 4354 list_of_parameters_for_stream=model.stream_parameters.list_of_parameters_for_stream 4355 ) 4356 4357 components_mapping = [] 4358 for components_mapping_definition_model in model.components_mapping: 4359 if components_mapping_definition_model.condition: 4360 raise ValueError("`condition` is only supported for `ConfigComponentsResolver`") 4361 components_mapping.append( 4362 self._create_component_from_model( 4363 model=components_mapping_definition_model, 4364 value_type=ModelToComponentFactory._json_schema_type_name_to_type( 4365 components_mapping_definition_model.value_type 4366 ), 4367 config=config, 4368 ) 4369 ) 4370 return ParametrizedComponentsResolver( 4371 stream_parameters=stream_parameters, 4372 config=config, 4373 components_mapping=components_mapping, 4374 parameters=model.parameters or {}, 4375 )
def
create_http_api_budget( self, model: airbyte_cdk.sources.declarative.models.declarative_component_schema.HTTPAPIBudget, config: Mapping[str, Any], **kwargs: Any) -> airbyte_cdk.HttpAPIBudget:
4399 def create_http_api_budget( 4400 self, model: HTTPAPIBudgetModel, config: Config, **kwargs: Any 4401 ) -> HttpAPIBudget: 4402 policies = [ 4403 self._create_component_from_model(model=policy, config=config) 4404 for policy in model.policies 4405 ] 4406 4407 return HttpAPIBudget( 4408 policies=policies, 4409 ratelimit_reset_header=model.ratelimit_reset_header or "ratelimit-reset", 4410 ratelimit_remaining_header=model.ratelimit_remaining_header or "ratelimit-remaining", 4411 status_codes_for_ratelimit_hit=model.status_codes_for_ratelimit_hit or [429], 4412 )
def
create_fixed_window_call_rate_policy( self, model: airbyte_cdk.sources.declarative.models.declarative_component_schema.FixedWindowCallRatePolicy, config: Mapping[str, Any], **kwargs: Any) -> airbyte_cdk.sources.streams.call_rate.FixedWindowCallRatePolicy:
4414 def create_fixed_window_call_rate_policy( 4415 self, model: FixedWindowCallRatePolicyModel, config: Config, **kwargs: Any 4416 ) -> FixedWindowCallRatePolicy: 4417 matchers = [ 4418 self._create_component_from_model(model=matcher, config=config) 4419 for matcher in model.matchers 4420 ] 4421 4422 # Set the initial reset timestamp to 10 days from now. 4423 # This value will be updated by the first request. 4424 return FixedWindowCallRatePolicy( 4425 next_reset_ts=datetime.datetime.now() + datetime.timedelta(days=10), 4426 period=parse_duration(model.period), 4427 call_limit=model.call_limit, 4428 matchers=matchers, 4429 )
def
create_file_uploader( self, model: airbyte_cdk.sources.declarative.models.declarative_component_schema.FileUploader, config: Mapping[str, Any], **kwargs: Any) -> airbyte_cdk.sources.declarative.retrievers.file_uploader.FileUploader:
4431 def create_file_uploader( 4432 self, model: FileUploaderModel, config: Config, **kwargs: Any 4433 ) -> FileUploader: 4434 name = "File Uploader" 4435 requester = self._create_component_from_model( 4436 model=model.requester, 4437 config=config, 4438 name=name, 4439 **kwargs, 4440 ) 4441 download_target_extractor = self._create_component_from_model( 4442 model=model.download_target_extractor, 4443 config=config, 4444 name=name, 4445 **kwargs, 4446 ) 4447 emit_connector_builder_messages = self._emit_connector_builder_messages 4448 file_uploader = DefaultFileUploader( 4449 requester=requester, 4450 download_target_extractor=download_target_extractor, 4451 config=config, 4452 file_writer=NoopFileWriter() 4453 if emit_connector_builder_messages 4454 else LocalFileSystemFileWriter(), 4455 parameters=model.parameters or {}, 4456 filename_extractor=model.filename_extractor if model.filename_extractor else None, 4457 ) 4458 4459 return ( 4460 ConnectorBuilderFileUploader(file_uploader) 4461 if emit_connector_builder_messages 4462 else file_uploader 4463 )
def
create_moving_window_call_rate_policy( self, model: airbyte_cdk.sources.declarative.models.declarative_component_schema.MovingWindowCallRatePolicy, config: Mapping[str, Any], **kwargs: Any) -> airbyte_cdk.MovingWindowCallRatePolicy:
4465 def create_moving_window_call_rate_policy( 4466 self, model: MovingWindowCallRatePolicyModel, config: Config, **kwargs: Any 4467 ) -> MovingWindowCallRatePolicy: 4468 rates = [ 4469 self._create_component_from_model(model=rate, config=config) for rate in model.rates 4470 ] 4471 matchers = [ 4472 self._create_component_from_model(model=matcher, config=config) 4473 for matcher in model.matchers 4474 ] 4475 return MovingWindowCallRatePolicy( 4476 rates=rates, 4477 matchers=matchers, 4478 )
def
create_unlimited_call_rate_policy( self, model: airbyte_cdk.sources.declarative.models.declarative_component_schema.UnlimitedCallRatePolicy, config: Mapping[str, Any], **kwargs: Any) -> airbyte_cdk.sources.streams.call_rate.UnlimitedCallRatePolicy:
4480 def create_unlimited_call_rate_policy( 4481 self, model: UnlimitedCallRatePolicyModel, config: Config, **kwargs: Any 4482 ) -> UnlimitedCallRatePolicy: 4483 matchers = [ 4484 self._create_component_from_model(model=matcher, config=config) 4485 for matcher in model.matchers 4486 ] 4487 4488 return UnlimitedCallRatePolicy( 4489 matchers=matchers, 4490 )
def
create_rate( self, model: airbyte_cdk.sources.declarative.models.declarative_component_schema.Rate, config: Mapping[str, Any], **kwargs: Any) -> airbyte_cdk.Rate:
def
create_http_request_matcher( self, model: airbyte_cdk.sources.declarative.models.declarative_component_schema.HttpRequestRegexMatcher, config: Mapping[str, Any], **kwargs: Any) -> airbyte_cdk.sources.streams.call_rate.HttpRequestRegexMatcher:
4499 def create_http_request_matcher( 4500 self, model: HttpRequestRegexMatcherModel, config: Config, **kwargs: Any 4501 ) -> HttpRequestRegexMatcher: 4502 weight = model.weight 4503 if weight is not None: 4504 if isinstance(weight, str): 4505 weight = int(InterpolatedString.create(weight, parameters={}).eval(config)) 4506 else: 4507 weight = int(weight) 4508 if weight < 1: 4509 raise ValueError(f"weight must be >= 1, got {weight}") 4510 return HttpRequestRegexMatcher( 4511 method=model.method, 4512 url_base=model.url_base, 4513 url_path_pattern=model.url_path_pattern, 4514 params=model.params, 4515 headers=model.headers, 4516 weight=weight, 4517 )
def
set_api_budget( self, component_definition: Mapping[str, Any], config: Mapping[str, Any]) -> None:
def
create_grouping_partition_router( self, model: airbyte_cdk.sources.declarative.models.declarative_component_schema.GroupingPartitionRouter, config: Mapping[str, Any], *, stream_name: str, **kwargs: Any) -> airbyte_cdk.sources.declarative.partition_routers.GroupingPartitionRouter:
4524 def create_grouping_partition_router( 4525 self, 4526 model: GroupingPartitionRouterModel, 4527 config: Config, 4528 *, 4529 stream_name: str, 4530 **kwargs: Any, 4531 ) -> GroupingPartitionRouter: 4532 underlying_router = self._create_component_from_model( 4533 model=model.underlying_partition_router, 4534 config=config, 4535 stream_name=stream_name, 4536 **kwargs, 4537 ) 4538 if model.group_size < 1: 4539 raise ValueError(f"Group size must be greater than 0, got {model.group_size}") 4540 4541 # Request options in underlying partition routers are not supported for GroupingPartitionRouter 4542 # because they are specific to individual partitions and cannot be aggregated or handled 4543 # when grouping, potentially leading to incorrect API calls. Any request customization 4544 # should be managed at the stream level through the requester's configuration. 4545 if isinstance(underlying_router, SubstreamPartitionRouter): 4546 if any( 4547 parent_config.request_option 4548 for parent_config in underlying_router.parent_stream_configs 4549 ): 4550 raise ValueError("Request options are not supported for GroupingPartitionRouter.") 4551 4552 if isinstance(underlying_router, ListPartitionRouter): 4553 if underlying_router.request_option: 4554 raise ValueError("Request options are not supported for GroupingPartitionRouter.") 4555 4556 return GroupingPartitionRouter( 4557 group_size=model.group_size, 4558 underlying_partition_router=underlying_router, 4559 deduplicate=model.deduplicate if model.deduplicate is not None else True, 4560 config=config, 4561 )