airbyte_cdk.sources.declarative.models.declarative_component_schema
1# Copyright (c) 2025 Airbyte, Inc., all rights reserved. 2 3# generated by datamodel-codegen: 4# filename: declarative_component_schema.yaml 5 6from __future__ import annotations 7 8from enum import Enum 9from typing import Any, Dict, List, Literal, Optional, Union 10 11from pydantic.v1 import BaseModel, Extra, Field 12 13from airbyte_cdk.sources.declarative.models.base_model_with_deprecations import ( 14 BaseModelWithDeprecations, 15) 16 17 18class AuthFlowType(Enum): 19 oauth2_0 = "oauth2.0" 20 oauth1_0 = "oauth1.0" 21 22 23class BasicHttpAuthenticator(BaseModel): 24 type: Literal["BasicHttpAuthenticator"] 25 username: str = Field( 26 ..., 27 description="The username that will be combined with the password, base64 encoded and used to make requests. Fill it in the user inputs.", 28 examples=["{{ config['username'] }}", "{{ config['api_key'] }}"], 29 title="Username", 30 ) 31 password: Optional[str] = Field( 32 "", 33 description="The password that will be combined with the username, base64 encoded and used to make requests. Fill it in the user inputs.", 34 examples=["{{ config['password'] }}", ""], 35 title="Password", 36 ) 37 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters") 38 39 40class BearerAuthenticator(BaseModel): 41 type: Literal["BearerAuthenticator"] 42 api_token: str = Field( 43 ..., 44 description="Token to inject as request header for authenticating with the API.", 45 examples=["{{ config['api_key'] }}", "{{ config['token'] }}"], 46 title="Bearer Token", 47 ) 48 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters") 49 50 51class DynamicStreamCheckConfig(BaseModel): 52 type: Literal["DynamicStreamCheckConfig"] 53 dynamic_stream_name: str = Field( 54 ..., description="The dynamic stream name.", title="Dynamic Stream Name" 55 ) 56 stream_count: Optional[int] = Field( 57 0, 58 description="The number of streams to attempt reading from during a check operation. If `stream_count` exceeds the total number of available streams, the minimum of the two values will be used.", 59 title="Stream Count", 60 ) 61 62 63class CheckDynamicStream(BaseModel): 64 type: Literal["CheckDynamicStream"] 65 stream_count: int = Field( 66 ..., 67 description="Numbers of the streams to try reading from when running a check operation.", 68 title="Stream Count", 69 ) 70 use_check_availability: Optional[bool] = Field( 71 True, 72 description="Enables stream check availability. This field is automatically set by the CDK.", 73 title="Use Check Availability", 74 ) 75 76 77class ConcurrencyLevel(BaseModel): 78 type: Optional[Literal["ConcurrencyLevel"]] = None 79 default_concurrency: Union[int, str] = Field( 80 ..., 81 description="The amount of concurrency that will applied during a sync. This value can be hardcoded or user-defined in the config if different users have varying volume thresholds in the target API.", 82 examples=[10, "{{ config['num_workers'] or 10 }}"], 83 title="Default Concurrency", 84 ) 85 max_concurrency: Optional[int] = Field( 86 None, 87 description="The maximum level of concurrency that will be used during a sync. This becomes a required field when the default_concurrency derives from the config, because it serves as a safeguard against a user-defined threshold that is too high.", 88 examples=[20, 100], 89 title="Max Concurrency", 90 ) 91 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters") 92 93 94class ConstantBackoffStrategy(BaseModel): 95 type: Literal["ConstantBackoffStrategy"] 96 backoff_time_in_seconds: Union[float, str] = Field( 97 ..., 98 description="Backoff time in seconds.", 99 examples=[30, 30.5, "{{ config['backoff_time'] }}"], 100 title="Backoff Time", 101 ) 102 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters") 103 104 105class CursorPagination(BaseModel): 106 type: Literal["CursorPagination"] 107 cursor_value: str = Field( 108 ..., 109 description="Value of the cursor defining the next page to fetch.", 110 examples=[ 111 "{{ headers.link.next.cursor }}", 112 "{{ last_record['key'] }}", 113 "{{ response['nextPage'] }}", 114 ], 115 title="Cursor Value", 116 ) 117 page_size: Optional[int] = Field( 118 None, 119 description="The number of records to include in each pages.", 120 examples=[100], 121 title="Page Size", 122 ) 123 stop_condition: Optional[str] = Field( 124 None, 125 description="Template string evaluating when to stop paginating.", 126 examples=[ 127 "{{ response.data.has_more is false }}", 128 "{{ 'next' not in headers['link'] }}", 129 ], 130 title="Stop Condition", 131 ) 132 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters") 133 134 135class CustomAuthenticator(BaseModel): 136 class Config: 137 extra = Extra.allow 138 139 type: Literal["CustomAuthenticator"] 140 class_name: str = Field( 141 ..., 142 description="Fully-qualified name of the class that will be implementing the custom authentication strategy. Has to be a sub class of DeclarativeAuthenticator. The format is `source_<name>.<package>.<class_name>`.", 143 examples=["source_railz.components.ShortLivedTokenAuthenticator"], 144 title="Class Name", 145 ) 146 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters") 147 148 149class CustomBackoffStrategy(BaseModel): 150 class Config: 151 extra = Extra.allow 152 153 type: Literal["CustomBackoffStrategy"] 154 class_name: str = Field( 155 ..., 156 description="Fully-qualified name of the class that will be implementing the custom backoff strategy. The format is `source_<name>.<package>.<class_name>`.", 157 examples=["source_railz.components.MyCustomBackoffStrategy"], 158 title="Class Name", 159 ) 160 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters") 161 162 163class CustomErrorHandler(BaseModel): 164 class Config: 165 extra = Extra.allow 166 167 type: Literal["CustomErrorHandler"] 168 class_name: str = Field( 169 ..., 170 description="Fully-qualified name of the class that will be implementing the custom error handler. The format is `source_<name>.<package>.<class_name>`.", 171 examples=["source_railz.components.MyCustomErrorHandler"], 172 title="Class Name", 173 ) 174 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters") 175 176 177class CustomIncrementalSync(BaseModel): 178 class Config: 179 extra = Extra.allow 180 181 type: Literal["CustomIncrementalSync"] 182 class_name: str = Field( 183 ..., 184 description="Fully-qualified name of the class that will be implementing the custom incremental sync. The format is `source_<name>.<package>.<class_name>`.", 185 examples=["source_railz.components.MyCustomIncrementalSync"], 186 title="Class Name", 187 ) 188 cursor_field: str = Field( 189 ..., 190 description="The location of the value on a record that will be used as a bookmark during sync.", 191 ) 192 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters") 193 194 195class CustomPaginationStrategy(BaseModel): 196 class Config: 197 extra = Extra.allow 198 199 type: Literal["CustomPaginationStrategy"] 200 class_name: str = Field( 201 ..., 202 description="Fully-qualified name of the class that will be implementing the custom pagination strategy. The format is `source_<name>.<package>.<class_name>`.", 203 examples=["source_railz.components.MyCustomPaginationStrategy"], 204 title="Class Name", 205 ) 206 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters") 207 208 209class CustomRecordExtractor(BaseModel): 210 class Config: 211 extra = Extra.allow 212 213 type: Literal["CustomRecordExtractor"] 214 class_name: str = Field( 215 ..., 216 description="Fully-qualified name of the class that will be implementing the custom record extraction strategy. The format is `source_<name>.<package>.<class_name>`.", 217 examples=["source_railz.components.MyCustomRecordExtractor"], 218 title="Class Name", 219 ) 220 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters") 221 222 223class CustomRecordFilter(BaseModel): 224 class Config: 225 extra = Extra.allow 226 227 type: Literal["CustomRecordFilter"] 228 class_name: str = Field( 229 ..., 230 description="Fully-qualified name of the class that will be implementing the custom record filter strategy. The format is `source_<name>.<package>.<class_name>`.", 231 examples=["source_railz.components.MyCustomCustomRecordFilter"], 232 title="Class Name", 233 ) 234 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters") 235 236 237class CustomRequester(BaseModel): 238 class Config: 239 extra = Extra.allow 240 241 type: Literal["CustomRequester"] 242 class_name: str = Field( 243 ..., 244 description="Fully-qualified name of the class that will be implementing the custom requester strategy. The format is `source_<name>.<package>.<class_name>`.", 245 examples=["source_railz.components.MyCustomRecordExtractor"], 246 title="Class Name", 247 ) 248 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters") 249 250 251class CustomRetriever(BaseModel): 252 class Config: 253 extra = Extra.allow 254 255 type: Literal["CustomRetriever"] 256 class_name: str = Field( 257 ..., 258 description="Fully-qualified name of the class that will be implementing the custom retriever strategy. The format is `source_<name>.<package>.<class_name>`.", 259 examples=["source_railz.components.MyCustomRetriever"], 260 title="Class Name", 261 ) 262 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters") 263 264 265class CustomPartitionRouter(BaseModel): 266 class Config: 267 extra = Extra.allow 268 269 type: Literal["CustomPartitionRouter"] 270 class_name: str = Field( 271 ..., 272 description="Fully-qualified name of the class that will be implementing the custom partition router. The format is `source_<name>.<package>.<class_name>`.", 273 examples=["source_railz.components.MyCustomPartitionRouter"], 274 title="Class Name", 275 ) 276 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters") 277 278 279class CustomSchemaLoader(BaseModel): 280 class Config: 281 extra = Extra.allow 282 283 type: Literal["CustomSchemaLoader"] 284 class_name: str = Field( 285 ..., 286 description="Fully-qualified name of the class that will be implementing the custom schema loader. The format is `source_<name>.<package>.<class_name>`.", 287 examples=["source_railz.components.MyCustomSchemaLoader"], 288 title="Class Name", 289 ) 290 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters") 291 292 293class CustomSchemaNormalization(BaseModel): 294 class Config: 295 extra = Extra.allow 296 297 type: Literal["CustomSchemaNormalization"] 298 class_name: str = Field( 299 ..., 300 description="Fully-qualified name of the class that will be implementing the custom normalization. The format is `source_<name>.<package>.<class_name>`.", 301 examples=[ 302 "source_amazon_seller_partner.components.LedgerDetailedViewReportsTypeTransformer" 303 ], 304 title="Class Name", 305 ) 306 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters") 307 308 309class CustomStateMigration(BaseModel): 310 class Config: 311 extra = Extra.allow 312 313 type: Literal["CustomStateMigration"] 314 class_name: str = Field( 315 ..., 316 description="Fully-qualified name of the class that will be implementing the custom state migration. The format is `source_<name>.<package>.<class_name>`.", 317 examples=["source_railz.components.MyCustomStateMigration"], 318 title="Class Name", 319 ) 320 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters") 321 322 323class CustomTransformation(BaseModel): 324 class Config: 325 extra = Extra.allow 326 327 type: Literal["CustomTransformation"] 328 class_name: str = Field( 329 ..., 330 description="Fully-qualified name of the class that will be implementing the custom transformation. The format is `source_<name>.<package>.<class_name>`.", 331 examples=["source_railz.components.MyCustomTransformation"], 332 title="Class Name", 333 ) 334 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters") 335 336 337class LegacyToPerPartitionStateMigration(BaseModel): 338 class Config: 339 extra = Extra.allow 340 341 type: Optional[Literal["LegacyToPerPartitionStateMigration"]] = None 342 343 344class Clamping(BaseModel): 345 target: str = Field( 346 ..., 347 description="The period of time that datetime windows will be clamped by", 348 examples=["DAY", "WEEK", "MONTH", "{{ config['target'] }}"], 349 title="Target", 350 ) 351 target_details: Optional[Dict[str, Any]] = None 352 353 354class Algorithm(Enum): 355 HS256 = "HS256" 356 HS384 = "HS384" 357 HS512 = "HS512" 358 ES256 = "ES256" 359 ES256K = "ES256K" 360 ES384 = "ES384" 361 ES512 = "ES512" 362 RS256 = "RS256" 363 RS384 = "RS384" 364 RS512 = "RS512" 365 PS256 = "PS256" 366 PS384 = "PS384" 367 PS512 = "PS512" 368 EdDSA = "EdDSA" 369 370 371class JwtHeaders(BaseModel): 372 class Config: 373 extra = Extra.forbid 374 375 kid: Optional[str] = Field( 376 None, 377 description="Private key ID for user account.", 378 examples=["{{ config['kid'] }}"], 379 title="Key Identifier", 380 ) 381 typ: Optional[str] = Field( 382 "JWT", 383 description="The media type of the complete JWT.", 384 examples=["JWT"], 385 title="Type", 386 ) 387 cty: Optional[str] = Field( 388 None, 389 description="Content type of JWT header.", 390 examples=["JWT"], 391 title="Content Type", 392 ) 393 394 395class JwtPayload(BaseModel): 396 class Config: 397 extra = Extra.forbid 398 399 iss: Optional[str] = Field( 400 None, 401 description="The user/principal that issued the JWT. Commonly a value unique to the user.", 402 examples=["{{ config['iss'] }}"], 403 title="Issuer", 404 ) 405 sub: Optional[str] = Field( 406 None, 407 description="The subject of the JWT. Commonly defined by the API.", 408 title="Subject", 409 ) 410 aud: Optional[str] = Field( 411 None, 412 description="The recipient that the JWT is intended for. Commonly defined by the API.", 413 examples=["appstoreconnect-v1"], 414 title="Audience", 415 ) 416 417 418class JwtAuthenticator(BaseModel): 419 type: Literal["JwtAuthenticator"] 420 secret_key: str = Field( 421 ..., 422 description="Secret used to sign the JSON web token.", 423 examples=["{{ config['secret_key'] }}"], 424 title="Secret Key", 425 ) 426 base64_encode_secret_key: Optional[bool] = Field( 427 False, 428 description='When set to true, the secret key will be base64 encoded prior to being encoded as part of the JWT. Only set to "true" when required by the API.', 429 title="Base64-encode Secret Key", 430 ) 431 algorithm: Algorithm = Field( 432 ..., 433 description="Algorithm used to sign the JSON web token.", 434 examples=["ES256", "HS256", "RS256", "{{ config['algorithm'] }}"], 435 title="Algorithm", 436 ) 437 token_duration: Optional[int] = Field( 438 1200, 439 description="The amount of time in seconds a JWT token can be valid after being issued.", 440 examples=[1200, 3600], 441 title="Token Duration", 442 ) 443 header_prefix: Optional[str] = Field( 444 None, 445 description="The prefix to be used within the Authentication header.", 446 examples=["Bearer", "Basic"], 447 title="Header Prefix", 448 ) 449 jwt_headers: Optional[JwtHeaders] = Field( 450 None, 451 description="JWT headers used when signing JSON web token.", 452 title="JWT Headers", 453 ) 454 additional_jwt_headers: Optional[Dict[str, Any]] = Field( 455 None, 456 description="Additional headers to be included with the JWT headers object.", 457 title="Additional JWT Headers", 458 ) 459 jwt_payload: Optional[JwtPayload] = Field( 460 None, 461 description="JWT Payload used when signing JSON web token.", 462 title="JWT Payload", 463 ) 464 additional_jwt_payload: Optional[Dict[str, Any]] = Field( 465 None, 466 description="Additional properties to be added to the JWT payload.", 467 title="Additional JWT Payload Properties", 468 ) 469 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters") 470 471 472class RefreshTokenUpdater(BaseModel): 473 refresh_token_name: Optional[str] = Field( 474 "refresh_token", 475 description="The name of the property which contains the updated refresh token in the response from the token refresh endpoint.", 476 examples=["refresh_token"], 477 title="Refresh Token Property Name", 478 ) 479 access_token_config_path: Optional[List[str]] = Field( 480 ["credentials", "access_token"], 481 description="Config path to the access token. Make sure the field actually exists in the config.", 482 examples=[["credentials", "access_token"], ["access_token"]], 483 title="Config Path To Access Token", 484 ) 485 refresh_token_config_path: Optional[List[str]] = Field( 486 ["credentials", "refresh_token"], 487 description="Config path to the access token. Make sure the field actually exists in the config.", 488 examples=[["credentials", "refresh_token"], ["refresh_token"]], 489 title="Config Path To Refresh Token", 490 ) 491 token_expiry_date_config_path: Optional[List[str]] = Field( 492 ["credentials", "token_expiry_date"], 493 description="Config path to the expiry date. Make sure actually exists in the config.", 494 examples=[["credentials", "token_expiry_date"]], 495 title="Config Path To Expiry Date", 496 ) 497 refresh_token_error_status_codes: Optional[List[int]] = Field( 498 [], 499 description="Status Codes to Identify refresh token error in response (Refresh Token Error Key and Refresh Token Error Values should be also specified). Responses with one of the error status code and containing an error value will be flagged as a config error", 500 examples=[[400, 500]], 501 title="Refresh Token Error Status Codes", 502 ) 503 refresh_token_error_key: Optional[str] = Field( 504 "", 505 description="Key to Identify refresh token error in response (Refresh Token Error Status Codes and Refresh Token Error Values should be also specified).", 506 examples=["error"], 507 title="Refresh Token Error Key", 508 ) 509 refresh_token_error_values: Optional[List[str]] = Field( 510 [], 511 description='List of values to check for exception during token refresh process. Used to check if the error found in the response matches the key from the Refresh Token Error Key field (e.g. response={"error": "invalid_grant"}). Only responses with one of the error status code and containing an error value will be flagged as a config error', 512 examples=[["invalid_grant", "invalid_permissions"]], 513 title="Refresh Token Error Values", 514 ) 515 516 517class OAuthAuthenticator(BaseModel): 518 type: Literal["OAuthAuthenticator"] 519 client_id_name: Optional[str] = Field( 520 "client_id", 521 description="The name of the property to use to refresh the `access_token`.", 522 examples=["custom_app_id"], 523 title="Client ID Property Name", 524 ) 525 client_id: Optional[str] = Field( 526 None, 527 description="The OAuth client ID. Fill it in the user inputs.", 528 examples=[ 529 "{{ config['client_id'] }}", 530 "{{ config['credentials']['client_id }}", 531 ], 532 title="Client ID", 533 ) 534 client_secret_name: Optional[str] = Field( 535 "client_secret", 536 description="The name of the property to use to refresh the `access_token`.", 537 examples=["custom_app_secret"], 538 title="Client Secret Property Name", 539 ) 540 client_secret: Optional[str] = Field( 541 None, 542 description="The OAuth client secret. Fill it in the user inputs.", 543 examples=[ 544 "{{ config['client_secret'] }}", 545 "{{ config['credentials']['client_secret }}", 546 ], 547 title="Client Secret", 548 ) 549 refresh_token_name: Optional[str] = Field( 550 "refresh_token", 551 description="The name of the property to use to refresh the `access_token`.", 552 examples=["custom_app_refresh_value"], 553 title="Refresh Token Property Name", 554 ) 555 refresh_token: Optional[str] = Field( 556 None, 557 description="Credential artifact used to get a new access token.", 558 examples=[ 559 "{{ config['refresh_token'] }}", 560 "{{ config['credentials]['refresh_token'] }}", 561 ], 562 title="Refresh Token", 563 ) 564 token_refresh_endpoint: Optional[str] = Field( 565 None, 566 description="The full URL to call to obtain a new access token.", 567 examples=["https://connect.squareup.com/oauth2/token"], 568 title="Token Refresh Endpoint", 569 ) 570 access_token_name: Optional[str] = Field( 571 "access_token", 572 description="The name of the property which contains the access token in the response from the token refresh endpoint.", 573 examples=["access_token"], 574 title="Access Token Property Name", 575 ) 576 access_token_value: Optional[str] = Field( 577 None, 578 description="The value of the access_token to bypass the token refreshing using `refresh_token`.", 579 examples=["secret_access_token_value"], 580 title="Access Token Value", 581 ) 582 expires_in_name: Optional[str] = Field( 583 "expires_in", 584 description="The name of the property which contains the expiry date in the response from the token refresh endpoint.", 585 examples=["expires_in"], 586 title="Token Expiry Property Name", 587 ) 588 grant_type_name: Optional[str] = Field( 589 "grant_type", 590 description="The name of the property to use to refresh the `access_token`.", 591 examples=["custom_grant_type"], 592 title="Grant Type Property Name", 593 ) 594 grant_type: Optional[str] = Field( 595 "refresh_token", 596 description="Specifies the OAuth2 grant type. If set to refresh_token, the refresh_token needs to be provided as well. For client_credentials, only client id and secret are required. Other grant types are not officially supported.", 597 examples=["refresh_token", "client_credentials"], 598 title="Grant Type", 599 ) 600 refresh_request_body: Optional[Dict[str, Any]] = Field( 601 None, 602 description="Body of the request sent to get a new access token.", 603 examples=[ 604 { 605 "applicationId": "{{ config['application_id'] }}", 606 "applicationSecret": "{{ config['application_secret'] }}", 607 "token": "{{ config['token'] }}", 608 } 609 ], 610 title="Refresh Request Body", 611 ) 612 refresh_request_headers: Optional[Dict[str, Any]] = Field( 613 None, 614 description="Headers of the request sent to get a new access token.", 615 examples=[ 616 { 617 "Authorization": "<AUTH_TOKEN>", 618 "Content-Type": "application/x-www-form-urlencoded", 619 } 620 ], 621 title="Refresh Request Headers", 622 ) 623 scopes: Optional[List[str]] = Field( 624 None, 625 description="List of scopes that should be granted to the access token.", 626 examples=[["crm.list.read", "crm.objects.contacts.read", "crm.schema.contacts.read"]], 627 title="Scopes", 628 ) 629 token_expiry_date: Optional[str] = Field( 630 None, 631 description="The access token expiry date.", 632 examples=["2023-04-06T07:12:10.421833+00:00", 1680842386], 633 title="Token Expiry Date", 634 ) 635 token_expiry_date_format: Optional[str] = Field( 636 None, 637 description="The format of the time to expiration datetime. Provide it if the time is returned as a date-time string instead of seconds.", 638 examples=["%Y-%m-%d %H:%M:%S.%f+00:00"], 639 title="Token Expiry Date Format", 640 ) 641 refresh_token_updater: Optional[RefreshTokenUpdater] = Field( 642 None, 643 description="When the refresh token updater is defined, new refresh tokens, access tokens and the access token expiry date are written back from the authentication response to the config object. This is important if the refresh token can only used once.", 644 title="Refresh Token Updater", 645 ) 646 profile_assertion: Optional[JwtAuthenticator] = Field( 647 None, 648 description="The authenticator being used to authenticate the client authenticator.", 649 title="Profile Assertion", 650 ) 651 use_profile_assertion: Optional[bool] = Field( 652 False, 653 description="Enable using profile assertion as a flow for OAuth authorization.", 654 title="Use Profile Assertion", 655 ) 656 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters") 657 658 659class Rate(BaseModel): 660 class Config: 661 extra = Extra.allow 662 663 limit: Union[int, str] = Field( 664 ..., 665 description="The maximum number of calls allowed within the interval.", 666 title="Limit", 667 ) 668 interval: str = Field( 669 ..., 670 description="The time interval for the rate limit.", 671 examples=["PT1H", "P1D"], 672 title="Interval", 673 ) 674 675 676class HttpRequestRegexMatcher(BaseModel): 677 class Config: 678 extra = Extra.allow 679 680 method: Optional[str] = Field( 681 None, description="The HTTP method to match (e.g., GET, POST).", title="Method" 682 ) 683 url_base: Optional[str] = Field( 684 None, 685 description='The base URL (scheme and host, e.g. "https://api.example.com") to match.', 686 title="URL Base", 687 ) 688 url_path_pattern: Optional[str] = Field( 689 None, 690 description="A regular expression pattern to match the URL path.", 691 title="URL Path Pattern", 692 ) 693 params: Optional[Dict[str, Any]] = Field( 694 None, description="The query parameters to match.", title="Parameters" 695 ) 696 headers: Optional[Dict[str, Any]] = Field( 697 None, description="The headers to match.", title="Headers" 698 ) 699 700 701class DpathExtractor(BaseModel): 702 type: Literal["DpathExtractor"] 703 field_path: List[str] = Field( 704 ..., 705 description='List of potentially nested fields describing the full path of the field to extract. Use "*" to extract all values from an array. See more info in the [docs](https://docs.airbyte.com/connector-development/config-based/understanding-the-yaml-file/record-selector).', 706 examples=[ 707 ["data"], 708 ["data", "records"], 709 ["data", "{{ parameters.name }}"], 710 ["data", "*", "record"], 711 ], 712 title="Field Path", 713 ) 714 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters") 715 716 717class ResponseToFileExtractor(BaseModel): 718 type: Literal["ResponseToFileExtractor"] 719 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters") 720 721 722class ExponentialBackoffStrategy(BaseModel): 723 type: Literal["ExponentialBackoffStrategy"] 724 factor: Optional[Union[float, str]] = Field( 725 5, 726 description="Multiplicative constant applied on each retry.", 727 examples=[5, 5.5, "10"], 728 title="Factor", 729 ) 730 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters") 731 732 733class GroupByKeyMergeStrategy(BaseModel): 734 type: Literal["GroupByKeyMergeStrategy"] 735 key: Union[str, List[str]] = Field( 736 ..., 737 description="The name of the field on the record whose value will be used to group properties that were retrieved through multiple API requests.", 738 examples=["id", ["parent_id", "end_date"]], 739 title="Key", 740 ) 741 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters") 742 743 744class SessionTokenRequestBearerAuthenticator(BaseModel): 745 type: Literal["Bearer"] 746 747 748class HttpMethod(Enum): 749 GET = "GET" 750 POST = "POST" 751 752 753class Action(Enum): 754 SUCCESS = "SUCCESS" 755 FAIL = "FAIL" 756 RETRY = "RETRY" 757 IGNORE = "IGNORE" 758 RATE_LIMITED = "RATE_LIMITED" 759 760 761class FailureType(Enum): 762 system_error = "system_error" 763 config_error = "config_error" 764 transient_error = "transient_error" 765 766 767class HttpResponseFilter(BaseModel): 768 type: Literal["HttpResponseFilter"] 769 action: Optional[Action] = Field( 770 None, 771 description="Action to execute if a response matches the filter.", 772 examples=["SUCCESS", "FAIL", "RETRY", "IGNORE", "RATE_LIMITED"], 773 title="Action", 774 ) 775 failure_type: Optional[FailureType] = Field( 776 None, 777 description="Failure type of traced exception if a response matches the filter.", 778 examples=["system_error", "config_error", "transient_error"], 779 title="Failure Type", 780 ) 781 error_message: Optional[str] = Field( 782 None, 783 description="Error Message to display if the response matches the filter.", 784 title="Error Message", 785 ) 786 error_message_contains: Optional[str] = Field( 787 None, 788 description="Match the response if its error message contains the substring.", 789 example=["This API operation is not enabled for this site"], 790 title="Error Message Substring", 791 ) 792 http_codes: Optional[List[int]] = Field( 793 None, 794 description="Match the response if its HTTP code is included in this list.", 795 examples=[[420, 429], [500]], 796 title="HTTP Codes", 797 unique_items=True, 798 ) 799 predicate: Optional[str] = Field( 800 None, 801 description="Match the response if the predicate evaluates to true.", 802 examples=[ 803 "{{ 'Too much requests' in response }}", 804 "{{ 'error_code' in response and response['error_code'] == 'ComplexityException' }}", 805 ], 806 title="Predicate", 807 ) 808 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters") 809 810 811class ComplexFieldType(BaseModel): 812 field_type: str 813 items: Optional[Union[str, ComplexFieldType]] = None 814 815 816class TypesMap(BaseModel): 817 target_type: Union[str, List[str], ComplexFieldType] 818 current_type: Union[str, List[str]] 819 condition: Optional[str] = None 820 821 822class SchemaTypeIdentifier(BaseModel): 823 type: Optional[Literal["SchemaTypeIdentifier"]] = None 824 schema_pointer: Optional[List[str]] = Field( 825 [], 826 description="List of nested fields defining the schema field path to extract. Defaults to [].", 827 title="Schema Path", 828 ) 829 key_pointer: List[str] = Field( 830 ..., 831 description="List of potentially nested fields describing the full path of the field key to extract.", 832 title="Key Path", 833 ) 834 type_pointer: Optional[List[str]] = Field( 835 None, 836 description="List of potentially nested fields describing the full path of the field type to extract.", 837 title="Type Path", 838 ) 839 types_mapping: Optional[List[TypesMap]] = None 840 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters") 841 842 843class InlineSchemaLoader(BaseModel): 844 type: Literal["InlineSchemaLoader"] 845 schema_: Optional[Dict[str, Any]] = Field( 846 None, 847 alias="schema", 848 description='Describes a streams\' schema. Refer to the <a href="https://docs.airbyte.com/understanding-airbyte/supported-data-types/">Data Types documentation</a> for more details on which types are valid.', 849 title="Schema", 850 ) 851 852 853class JsonFileSchemaLoader(BaseModel): 854 type: Literal["JsonFileSchemaLoader"] 855 file_path: Optional[str] = Field( 856 None, 857 description="Path to the JSON file defining the schema. The path is relative to the connector module's root.", 858 example=["./schemas/users.json"], 859 title="File Path", 860 ) 861 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters") 862 863 864class JsonDecoder(BaseModel): 865 type: Literal["JsonDecoder"] 866 867 868class JsonlDecoder(BaseModel): 869 type: Literal["JsonlDecoder"] 870 871 872class KeysToLower(BaseModel): 873 type: Literal["KeysToLower"] 874 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters") 875 876 877class KeysToSnakeCase(BaseModel): 878 type: Literal["KeysToSnakeCase"] 879 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters") 880 881 882class FlattenFields(BaseModel): 883 type: Literal["FlattenFields"] 884 flatten_lists: Optional[bool] = Field( 885 True, 886 description="Whether to flatten lists or leave it as is. Default is True.", 887 title="Flatten Lists", 888 ) 889 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters") 890 891 892class KeyTransformation(BaseModel): 893 type: Literal["KeyTransformation"] 894 prefix: Optional[str] = Field( 895 None, 896 description="Prefix to add for object keys. If not provided original keys remain unchanged.", 897 examples=["flattened_"], 898 title="Key Prefix", 899 ) 900 suffix: Optional[str] = Field( 901 None, 902 description="Suffix to add for object keys. If not provided original keys remain unchanged.", 903 examples=["_flattened"], 904 title="Key Suffix", 905 ) 906 907 908class DpathFlattenFields(BaseModel): 909 type: Literal["DpathFlattenFields"] 910 field_path: List[str] = Field( 911 ..., 912 description="A path to field that needs to be flattened.", 913 examples=[["data"], ["data", "*", "field"]], 914 title="Field Path", 915 ) 916 delete_origin_value: Optional[bool] = Field( 917 None, 918 description="Whether to delete the origin value or keep it. Default is False.", 919 title="Delete Origin Value", 920 ) 921 replace_record: Optional[bool] = Field( 922 None, 923 description="Whether to replace the origin record or not. Default is False.", 924 title="Replace Origin Record", 925 ) 926 key_transformation: Optional[KeyTransformation] = Field( 927 None, 928 description="Transformation for object keys. If not provided, original key will be used.", 929 title="Key transformation", 930 ) 931 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters") 932 933 934class KeysReplace(BaseModel): 935 type: Literal["KeysReplace"] 936 old: str = Field( 937 ..., 938 description="Old value to replace.", 939 examples=[ 940 " ", 941 "{{ record.id }}", 942 "{{ config['id'] }}", 943 "{{ stream_slice['id'] }}", 944 ], 945 title="Old value", 946 ) 947 new: str = Field( 948 ..., 949 description="New value to set.", 950 examples=[ 951 "_", 952 "{{ record.id }}", 953 "{{ config['id'] }}", 954 "{{ stream_slice['id'] }}", 955 ], 956 title="New value", 957 ) 958 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters") 959 960 961class IterableDecoder(BaseModel): 962 type: Literal["IterableDecoder"] 963 964 965class XmlDecoder(BaseModel): 966 type: Literal["XmlDecoder"] 967 968 969class CustomDecoder(BaseModel): 970 class Config: 971 extra = Extra.allow 972 973 type: Literal["CustomDecoder"] 974 class_name: str = Field( 975 ..., 976 description="Fully-qualified name of the class that will be implementing the custom decoding. Has to be a sub class of Decoder. The format is `source_<name>.<package>.<class_name>`.", 977 examples=["source_amazon_ads.components.GzipJsonlDecoder"], 978 title="Class Name", 979 ) 980 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters") 981 982 983class MinMaxDatetime(BaseModel): 984 type: Literal["MinMaxDatetime"] 985 datetime: str = Field( 986 ..., 987 description="Datetime value.", 988 examples=[ 989 "2021-01-01", 990 "2021-01-01T00:00:00Z", 991 "{{ config['start_time'] }}", 992 "{{ now_utc().strftime('%Y-%m-%dT%H:%M:%SZ') }}", 993 ], 994 title="Datetime", 995 ) 996 datetime_format: Optional[str] = Field( 997 "", 998 description='Format of the datetime value. Defaults to "%Y-%m-%dT%H:%M:%S.%f%z" if left empty. Use placeholders starting with "%" to describe the format the API is using. The following placeholders are available:\n * **%s**: Epoch unix timestamp - `1686218963`\n * **%s_as_float**: Epoch unix timestamp in seconds as float with microsecond precision - `1686218963.123456`\n * **%ms**: Epoch unix timestamp - `1686218963123`\n * **%a**: Weekday (abbreviated) - `Sun`\n * **%A**: Weekday (full) - `Sunday`\n * **%w**: Weekday (decimal) - `0` (Sunday), `6` (Saturday)\n * **%d**: Day of the month (zero-padded) - `01`, `02`, ..., `31`\n * **%b**: Month (abbreviated) - `Jan`\n * **%B**: Month (full) - `January`\n * **%m**: Month (zero-padded) - `01`, `02`, ..., `12`\n * **%y**: Year (without century, zero-padded) - `00`, `01`, ..., `99`\n * **%Y**: Year (with century) - `0001`, `0002`, ..., `9999`\n * **%H**: Hour (24-hour, zero-padded) - `00`, `01`, ..., `23`\n * **%I**: Hour (12-hour, zero-padded) - `01`, `02`, ..., `12`\n * **%p**: AM/PM indicator\n * **%M**: Minute (zero-padded) - `00`, `01`, ..., `59`\n * **%S**: Second (zero-padded) - `00`, `01`, ..., `59`\n * **%f**: Microsecond (zero-padded to 6 digits) - `000000`, `000001`, ..., `999999`\n * **%_ms**: Millisecond (zero-padded to 3 digits) - `000`, `001`, ..., `999`\n * **%z**: UTC offset - `(empty)`, `+0000`, `-04:00`\n * **%Z**: Time zone name - `(empty)`, `UTC`, `GMT`\n * **%j**: Day of the year (zero-padded) - `001`, `002`, ..., `366`\n * **%U**: Week number of the year (Sunday as first day) - `00`, `01`, ..., `53`\n * **%W**: Week number of the year (Monday as first day) - `00`, `01`, ..., `53`\n * **%c**: Date and time representation - `Tue Aug 16 21:30:00 1988`\n * **%x**: Date representation - `08/16/1988`\n * **%X**: Time representation - `21:30:00`\n * **%%**: Literal \'%\' character\n\n Some placeholders depend on the locale of the underlying system - in most cases this locale is configured as en/US. For more information see the [Python documentation](https://docs.python.org/3/library/datetime.html#strftime-and-strptime-format-codes).\n', 999 examples=["%Y-%m-%dT%H:%M:%S.%f%z", "%Y-%m-%d", "%s"], 1000 title="Datetime Format", 1001 ) 1002 max_datetime: Optional[str] = Field( 1003 None, 1004 description="Ceiling applied on the datetime value. Must be formatted with the datetime_format field.", 1005 examples=["2021-01-01T00:00:00Z", "2021-01-01"], 1006 title="Max Datetime", 1007 ) 1008 min_datetime: Optional[str] = Field( 1009 None, 1010 description="Floor applied on the datetime value. Must be formatted with the datetime_format field.", 1011 examples=["2010-01-01T00:00:00Z", "2010-01-01"], 1012 title="Min Datetime", 1013 ) 1014 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters") 1015 1016 1017class NoAuth(BaseModel): 1018 type: Literal["NoAuth"] 1019 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters") 1020 1021 1022class NoPagination(BaseModel): 1023 type: Literal["NoPagination"] 1024 1025 1026class State(BaseModel): 1027 class Config: 1028 extra = Extra.allow 1029 1030 min: int 1031 max: int 1032 1033 1034class OauthConnectorInputSpecification(BaseModel): 1035 class Config: 1036 extra = Extra.allow 1037 1038 consent_url: str = Field( 1039 ..., 1040 description="The DeclarativeOAuth Specific string URL string template to initiate the authentication.\nThe placeholders are replaced during the processing to provide neccessary values.", 1041 examples=[ 1042 "https://domain.host.com/marketing_api/auth?{{client_id_key}}={{client_id_value}}&{{redirect_uri_key}}={{{{redirect_uri_value}} | urlEncoder}}&{{state_key}}={{state_value}}", 1043 "https://endpoint.host.com/oauth2/authorize?{{client_id_key}}={{client_id_value}}&{{redirect_uri_key}}={{{{redirect_uri_value}} | urlEncoder}}&{{scope_key}}={{{{scope_value}} | urlEncoder}}&{{state_key}}={{state_value}}&subdomain={{subdomain}}", 1044 ], 1045 title="Consent URL", 1046 ) 1047 scope: Optional[str] = Field( 1048 None, 1049 description="The DeclarativeOAuth Specific string of the scopes needed to be grant for authenticated user.", 1050 examples=["user:read user:read_orders workspaces:read"], 1051 title="Scopes", 1052 ) 1053 access_token_url: str = Field( 1054 ..., 1055 description="The DeclarativeOAuth Specific URL templated string to obtain the `access_token`, `refresh_token` etc.\nThe placeholders are replaced during the processing to provide neccessary values.", 1056 examples=[ 1057 "https://auth.host.com/oauth2/token?{{client_id_key}}={{client_id_value}}&{{client_secret_key}}={{client_secret_value}}&{{auth_code_key}}={{auth_code_value}}&{{redirect_uri_key}}={{{{redirect_uri_value}} | urlEncoder}}" 1058 ], 1059 title="Access Token URL", 1060 ) 1061 access_token_headers: Optional[Dict[str, Any]] = Field( 1062 None, 1063 description="The DeclarativeOAuth Specific optional headers to inject while exchanging the `auth_code` to `access_token` during `completeOAuthFlow` step.", 1064 examples=[ 1065 { 1066 "Authorization": "Basic {{ {{ client_id_value }}:{{ client_secret_value }} | base64Encoder }}" 1067 } 1068 ], 1069 title="Access Token Headers", 1070 ) 1071 access_token_params: Optional[Dict[str, Any]] = Field( 1072 None, 1073 description="The DeclarativeOAuth Specific optional query parameters to inject while exchanging the `auth_code` to `access_token` during `completeOAuthFlow` step.\nWhen this property is provided, the query params will be encoded as `Json` and included in the outgoing API request.", 1074 examples=[ 1075 { 1076 "{{ auth_code_key }}": "{{ auth_code_value }}", 1077 "{{ client_id_key }}": "{{ client_id_value }}", 1078 "{{ client_secret_key }}": "{{ client_secret_value }}", 1079 } 1080 ], 1081 title="Access Token Query Params (Json Encoded)", 1082 ) 1083 extract_output: Optional[List[str]] = Field( 1084 None, 1085 description="The DeclarativeOAuth Specific list of strings to indicate which keys should be extracted and returned back to the input config.", 1086 examples=[["access_token", "refresh_token", "other_field"]], 1087 title="Extract Output", 1088 ) 1089 state: Optional[State] = Field( 1090 None, 1091 description="The DeclarativeOAuth Specific object to provide the criteria of how the `state` query param should be constructed,\nincluding length and complexity.", 1092 examples=[{"min": 7, "max": 128}], 1093 title="Configurable State Query Param", 1094 ) 1095 client_id_key: Optional[str] = Field( 1096 None, 1097 description="The DeclarativeOAuth Specific optional override to provide the custom `client_id` key name, if required by data-provider.", 1098 examples=["my_custom_client_id_key_name"], 1099 title="Client ID Key Override", 1100 ) 1101 client_secret_key: Optional[str] = Field( 1102 None, 1103 description="The DeclarativeOAuth Specific optional override to provide the custom `client_secret` key name, if required by data-provider.", 1104 examples=["my_custom_client_secret_key_name"], 1105 title="Client Secret Key Override", 1106 ) 1107 scope_key: Optional[str] = Field( 1108 None, 1109 description="The DeclarativeOAuth Specific optional override to provide the custom `scope` key name, if required by data-provider.", 1110 examples=["my_custom_scope_key_key_name"], 1111 title="Scopes Key Override", 1112 ) 1113 state_key: Optional[str] = Field( 1114 None, 1115 description="The DeclarativeOAuth Specific optional override to provide the custom `state` key name, if required by data-provider.", 1116 examples=["my_custom_state_key_key_name"], 1117 title="State Key Override", 1118 ) 1119 auth_code_key: Optional[str] = Field( 1120 None, 1121 description="The DeclarativeOAuth Specific optional override to provide the custom `code` key name to something like `auth_code` or `custom_auth_code`, if required by data-provider.", 1122 examples=["my_custom_auth_code_key_name"], 1123 title="Auth Code Key Override", 1124 ) 1125 redirect_uri_key: Optional[str] = Field( 1126 None, 1127 description="The DeclarativeOAuth Specific optional override to provide the custom `redirect_uri` key name to something like `callback_uri`, if required by data-provider.", 1128 examples=["my_custom_redirect_uri_key_name"], 1129 title="Redirect URI Key Override", 1130 ) 1131 1132 1133class OAuthConfigSpecification(BaseModel): 1134 class Config: 1135 extra = Extra.allow 1136 1137 oauth_user_input_from_connector_config_specification: Optional[Dict[str, Any]] = Field( 1138 None, 1139 description="OAuth specific blob. This is a Json Schema used to validate Json configurations used as input to OAuth.\nMust be a valid non-nested JSON that refers to properties from ConnectorSpecification.connectionSpecification\nusing special annotation 'path_in_connector_config'.\nThese are input values the user is entering through the UI to authenticate to the connector, that might also shared\nas inputs for syncing data via the connector.\nExamples:\nif no connector values is shared during oauth flow, oauth_user_input_from_connector_config_specification=[]\nif connector values such as 'app_id' inside the top level are used to generate the API url for the oauth flow,\n oauth_user_input_from_connector_config_specification={\n app_id: {\n type: string\n path_in_connector_config: ['app_id']\n }\n }\nif connector values such as 'info.app_id' nested inside another object are used to generate the API url for the oauth flow,\n oauth_user_input_from_connector_config_specification={\n app_id: {\n type: string\n path_in_connector_config: ['info', 'app_id']\n }\n }", 1140 examples=[ 1141 {"app_id": {"type": "string", "path_in_connector_config": ["app_id"]}}, 1142 { 1143 "app_id": { 1144 "type": "string", 1145 "path_in_connector_config": ["info", "app_id"], 1146 } 1147 }, 1148 ], 1149 title="OAuth user input", 1150 ) 1151 oauth_connector_input_specification: Optional[OauthConnectorInputSpecification] = Field( 1152 None, 1153 description='The DeclarativeOAuth specific blob.\nPertains to the fields defined by the connector relating to the OAuth flow.\n\nInterpolation capabilities:\n- The variables placeholders are declared as `{{my_var}}`.\n- The nested resolution variables like `{{ {{my_nested_var}} }}` is allowed as well.\n\n- The allowed interpolation context is:\n + base64Encoder - encode to `base64`, {{ {{my_var_a}}:{{my_var_b}} | base64Encoder }}\n + base64Decorer - decode from `base64` encoded string, {{ {{my_string_variable_or_string_value}} | base64Decoder }}\n + urlEncoder - encode the input string to URL-like format, {{ https://test.host.com/endpoint | urlEncoder}}\n + urlDecorer - decode the input url-encoded string into text format, {{ urlDecoder:https%3A%2F%2Fairbyte.io | urlDecoder}}\n + codeChallengeS256 - get the `codeChallenge` encoded value to provide additional data-provider specific authorisation values, {{ {{state_value}} | codeChallengeS256 }}\n\nExamples:\n - The TikTok Marketing DeclarativeOAuth spec:\n {\n "oauth_connector_input_specification": {\n "type": "object",\n "additionalProperties": false,\n "properties": {\n "consent_url": "https://ads.tiktok.com/marketing_api/auth?{{client_id_key}}={{client_id_value}}&{{redirect_uri_key}}={{ {{redirect_uri_value}} | urlEncoder}}&{{state_key}}={{state_value}}",\n "access_token_url": "https://business-api.tiktok.com/open_api/v1.3/oauth2/access_token/",\n "access_token_params": {\n "{{ auth_code_key }}": "{{ auth_code_value }}",\n "{{ client_id_key }}": "{{ client_id_value }}",\n "{{ client_secret_key }}": "{{ client_secret_value }}"\n },\n "access_token_headers": {\n "Content-Type": "application/json",\n "Accept": "application/json"\n },\n "extract_output": ["data.access_token"],\n "client_id_key": "app_id",\n "client_secret_key": "secret",\n "auth_code_key": "auth_code"\n }\n }\n }', 1154 title="DeclarativeOAuth Connector Specification", 1155 ) 1156 complete_oauth_output_specification: Optional[Dict[str, Any]] = Field( 1157 None, 1158 description="OAuth specific blob. This is a Json Schema used to validate Json configurations produced by the OAuth flows as they are\nreturned by the distant OAuth APIs.\nMust be a valid JSON describing the fields to merge back to `ConnectorSpecification.connectionSpecification`.\nFor each field, a special annotation `path_in_connector_config` can be specified to determine where to merge it,\nExamples:\n complete_oauth_output_specification={\n refresh_token: {\n type: string,\n path_in_connector_config: ['credentials', 'refresh_token']\n }\n }", 1159 examples=[ 1160 { 1161 "refresh_token": { 1162 "type": "string,", 1163 "path_in_connector_config": ["credentials", "refresh_token"], 1164 } 1165 } 1166 ], 1167 title="OAuth output specification", 1168 ) 1169 complete_oauth_server_input_specification: Optional[Dict[str, Any]] = Field( 1170 None, 1171 description="OAuth specific blob. This is a Json Schema used to validate Json configurations persisted as Airbyte Server configurations.\nMust be a valid non-nested JSON describing additional fields configured by the Airbyte Instance or Workspace Admins to be used by the\nserver when completing an OAuth flow (typically exchanging an auth code for refresh token).\nExamples:\n complete_oauth_server_input_specification={\n client_id: {\n type: string\n },\n client_secret: {\n type: string\n }\n }", 1172 examples=[{"client_id": {"type": "string"}, "client_secret": {"type": "string"}}], 1173 title="OAuth input specification", 1174 ) 1175 complete_oauth_server_output_specification: Optional[Dict[str, Any]] = Field( 1176 None, 1177 description="OAuth specific blob. This is a Json Schema used to validate Json configurations persisted as Airbyte Server configurations that\nalso need to be merged back into the connector configuration at runtime.\nThis is a subset configuration of `complete_oauth_server_input_specification` that filters fields out to retain only the ones that\nare necessary for the connector to function with OAuth. (some fields could be used during oauth flows but not needed afterwards, therefore\nthey would be listed in the `complete_oauth_server_input_specification` but not `complete_oauth_server_output_specification`)\nMust be a valid non-nested JSON describing additional fields configured by the Airbyte Instance or Workspace Admins to be used by the\nconnector when using OAuth flow APIs.\nThese fields are to be merged back to `ConnectorSpecification.connectionSpecification`.\nFor each field, a special annotation `path_in_connector_config` can be specified to determine where to merge it,\nExamples:\n complete_oauth_server_output_specification={\n client_id: {\n type: string,\n path_in_connector_config: ['credentials', 'client_id']\n },\n client_secret: {\n type: string,\n path_in_connector_config: ['credentials', 'client_secret']\n }\n }", 1178 examples=[ 1179 { 1180 "client_id": { 1181 "type": "string,", 1182 "path_in_connector_config": ["credentials", "client_id"], 1183 }, 1184 "client_secret": { 1185 "type": "string,", 1186 "path_in_connector_config": ["credentials", "client_secret"], 1187 }, 1188 } 1189 ], 1190 title="OAuth server output specification", 1191 ) 1192 1193 1194class OffsetIncrement(BaseModel): 1195 type: Literal["OffsetIncrement"] 1196 page_size: Optional[Union[int, str]] = Field( 1197 None, 1198 description="The number of records to include in each pages.", 1199 examples=[100, "{{ config['page_size'] }}"], 1200 title="Limit", 1201 ) 1202 inject_on_first_request: Optional[bool] = Field( 1203 False, 1204 description="Using the `offset` with value `0` during the first request", 1205 title="Inject Offset on First Request", 1206 ) 1207 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters") 1208 1209 1210class PageIncrement(BaseModel): 1211 type: Literal["PageIncrement"] 1212 page_size: Optional[Union[int, str]] = Field( 1213 None, 1214 description="The number of records to include in each pages.", 1215 examples=[100, "100", "{{ config['page_size'] }}"], 1216 title="Page Size", 1217 ) 1218 start_from_page: Optional[int] = Field( 1219 0, 1220 description="Index of the first page to request.", 1221 examples=[0, 1], 1222 title="Start From Page", 1223 ) 1224 inject_on_first_request: Optional[bool] = Field( 1225 False, 1226 description="Using the `page number` with value defined by `start_from_page` during the first request", 1227 title="Inject Page Number on First Request", 1228 ) 1229 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters") 1230 1231 1232class PrimaryKey(BaseModel): 1233 __root__: Union[str, List[str], List[List[str]]] = Field( 1234 ..., 1235 description="The stream field to be used to distinguish unique records. Can either be a single field, an array of fields representing a composite key, or an array of arrays representing a composite key where the fields are nested fields.", 1236 examples=["id", ["code", "type"]], 1237 title="Primary Key", 1238 ) 1239 1240 1241class PropertyLimitType(Enum): 1242 characters = "characters" 1243 property_count = "property_count" 1244 1245 1246class PropertyChunking(BaseModel): 1247 type: Literal["PropertyChunking"] 1248 property_limit_type: PropertyLimitType = Field( 1249 ..., 1250 description="The type used to determine the maximum number of properties per chunk", 1251 title="Property Limit Type", 1252 ) 1253 property_limit: Optional[int] = Field( 1254 None, 1255 description="The maximum amount of properties that can be retrieved per request according to the limit type.", 1256 title="Property Limit", 1257 ) 1258 record_merge_strategy: Optional[GroupByKeyMergeStrategy] = Field( 1259 None, 1260 description="Dictates how to records that require multiple requests to get all properties should be emitted to the destination", 1261 title="Record Merge Strategy", 1262 ) 1263 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters") 1264 1265 1266class RecordFilter(BaseModel): 1267 type: Literal["RecordFilter"] 1268 condition: Optional[str] = Field( 1269 "", 1270 description="The predicate to filter a record. Records will be removed if evaluated to False.", 1271 examples=[ 1272 "{{ record['created_at'] >= stream_interval['start_time'] }}", 1273 "{{ record.status in ['active', 'expired'] }}", 1274 ], 1275 title="Condition", 1276 ) 1277 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters") 1278 1279 1280class SchemaNormalization(Enum): 1281 Default = "Default" 1282 None_ = "None" 1283 1284 1285class RemoveFields(BaseModel): 1286 type: Literal["RemoveFields"] 1287 condition: Optional[str] = Field( 1288 "", 1289 description="The predicate to filter a property by a property value. Property will be removed if it is empty OR expression is evaluated to True.,", 1290 examples=[ 1291 "{{ property|string == '' }}", 1292 "{{ property is integer }}", 1293 "{{ property|length > 5 }}", 1294 "{{ property == 'some_string_to_match' }}", 1295 ], 1296 ) 1297 field_pointers: List[List[str]] = Field( 1298 ..., 1299 description="Array of paths defining the field to remove. Each item is an array whose field describe the path of a field to remove.", 1300 examples=[["tags"], [["content", "html"], ["content", "plain_text"]]], 1301 title="Field Paths", 1302 ) 1303 1304 1305class RequestPath(BaseModel): 1306 type: Literal["RequestPath"] 1307 1308 1309class InjectInto(Enum): 1310 request_parameter = "request_parameter" 1311 header = "header" 1312 body_data = "body_data" 1313 body_json = "body_json" 1314 1315 1316class RequestOption(BaseModel): 1317 type: Literal["RequestOption"] 1318 inject_into: InjectInto = Field( 1319 ..., 1320 description="Configures where the descriptor should be set on the HTTP requests. Note that request parameters that are already encoded in the URL path will not be duplicated.", 1321 examples=["request_parameter", "header", "body_data", "body_json"], 1322 title="Inject Into", 1323 ) 1324 field_name: Optional[str] = Field( 1325 None, 1326 description="Configures which key should be used in the location that the descriptor is being injected into. We hope to eventually deprecate this field in favor of `field_path` for all request_options, but must currently maintain it for backwards compatibility in the Builder.", 1327 examples=["segment_id"], 1328 title="Field Name", 1329 ) 1330 field_path: Optional[List[str]] = Field( 1331 None, 1332 description="Configures a path to be used for nested structures in JSON body requests (e.g. GraphQL queries)", 1333 examples=[["data", "viewer", "id"]], 1334 title="Field Path", 1335 ) 1336 1337 1338class Schemas(BaseModel): 1339 pass 1340 1341 class Config: 1342 extra = Extra.allow 1343 1344 1345class LegacySessionTokenAuthenticator(BaseModel): 1346 type: Literal["LegacySessionTokenAuthenticator"] 1347 header: str = Field( 1348 ..., 1349 description="The name of the session token header that will be injected in the request", 1350 examples=["X-Session"], 1351 title="Session Request Header", 1352 ) 1353 login_url: str = Field( 1354 ..., 1355 description="Path of the login URL (do not include the base URL)", 1356 examples=["session"], 1357 title="Login Path", 1358 ) 1359 session_token: Optional[str] = Field( 1360 None, 1361 description="Session token to use if using a pre-defined token. Not needed if authenticating with username + password pair", 1362 example=["{{ config['session_token'] }}"], 1363 title="Session Token", 1364 ) 1365 session_token_response_key: str = Field( 1366 ..., 1367 description="Name of the key of the session token to be extracted from the response", 1368 examples=["id"], 1369 title="Response Token Response Key", 1370 ) 1371 username: Optional[str] = Field( 1372 None, 1373 description="Username used to authenticate and obtain a session token", 1374 examples=[" {{ config['username'] }}"], 1375 title="Username", 1376 ) 1377 password: Optional[str] = Field( 1378 "", 1379 description="Password used to authenticate and obtain a session token", 1380 examples=["{{ config['password'] }}", ""], 1381 title="Password", 1382 ) 1383 validate_session_url: str = Field( 1384 ..., 1385 description="Path of the URL to use to validate that the session token is valid (do not include the base URL)", 1386 examples=["user/current"], 1387 title="Validate Session Path", 1388 ) 1389 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters") 1390 1391 1392class CsvDecoder(BaseModel): 1393 type: Literal["CsvDecoder"] 1394 encoding: Optional[str] = "utf-8" 1395 delimiter: Optional[str] = "," 1396 set_values_to_none: Optional[List[str]] = None 1397 1398 1399class AsyncJobStatusMap(BaseModel): 1400 type: Optional[Literal["AsyncJobStatusMap"]] = None 1401 running: List[str] 1402 completed: List[str] 1403 failed: List[str] 1404 timeout: List[str] 1405 1406 1407class ValueType(Enum): 1408 string = "string" 1409 number = "number" 1410 integer = "integer" 1411 boolean = "boolean" 1412 1413 1414class WaitTimeFromHeader(BaseModel): 1415 type: Literal["WaitTimeFromHeader"] 1416 header: str = Field( 1417 ..., 1418 description="The name of the response header defining how long to wait before retrying.", 1419 examples=["Retry-After"], 1420 title="Response Header Name", 1421 ) 1422 regex: Optional[str] = Field( 1423 None, 1424 description="Optional regex to apply on the header to extract its value. The regex should define a capture group defining the wait time.", 1425 examples=["([-+]?\\d+)"], 1426 title="Extraction Regex", 1427 ) 1428 max_waiting_time_in_seconds: Optional[float] = Field( 1429 None, 1430 description="Given the value extracted from the header is greater than this value, stop the stream.", 1431 examples=[3600], 1432 title="Max Waiting Time in Seconds", 1433 ) 1434 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters") 1435 1436 1437class WaitUntilTimeFromHeader(BaseModel): 1438 type: Literal["WaitUntilTimeFromHeader"] 1439 header: str = Field( 1440 ..., 1441 description="The name of the response header defining how long to wait before retrying.", 1442 examples=["wait_time"], 1443 title="Response Header", 1444 ) 1445 min_wait: Optional[Union[float, str]] = Field( 1446 None, 1447 description="Minimum time to wait before retrying.", 1448 examples=[10, "60"], 1449 title="Minimum Wait Time", 1450 ) 1451 regex: Optional[str] = Field( 1452 None, 1453 description="Optional regex to apply on the header to extract its value. The regex should define a capture group defining the wait time.", 1454 examples=["([-+]?\\d+)"], 1455 title="Extraction Regex", 1456 ) 1457 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters") 1458 1459 1460class ComponentMappingDefinition(BaseModel): 1461 type: Literal["ComponentMappingDefinition"] 1462 field_path: List[str] = Field( 1463 ..., 1464 description="A list of potentially nested fields indicating the full path where value will be added or updated.", 1465 examples=[ 1466 ["data"], 1467 ["data", "records"], 1468 ["data", 1, "name"], 1469 ["data", "{{ components_values.name }}"], 1470 ["data", "*", "record"], 1471 ["*", "**", "name"], 1472 ], 1473 title="Field Path", 1474 ) 1475 value: str = Field( 1476 ..., 1477 description="The dynamic or static value to assign to the key. Interpolated values can be used to dynamically determine the value during runtime.", 1478 examples=[ 1479 "{{ components_values['updates'] }}", 1480 "{{ components_values['MetaData']['LastUpdatedTime'] }}", 1481 "{{ config['segment_id'] }}", 1482 "{{ stream_slice['parent_id'] }}", 1483 "{{ stream_slice['extra_fields']['name'] }}", 1484 ], 1485 title="Value", 1486 ) 1487 value_type: Optional[ValueType] = Field( 1488 None, 1489 description="The expected data type of the value. If omitted, the type will be inferred from the value provided.", 1490 title="Value Type", 1491 ) 1492 create_or_update: Optional[bool] = Field( 1493 False, 1494 description="Determines whether to create a new path if it doesn't exist (true) or only update existing paths (false). When set to true, the resolver will create new paths in the stream template if they don't exist. When false (default), it will only update existing paths.", 1495 title="Create or Update", 1496 ) 1497 condition: Optional[str] = Field( 1498 None, 1499 description="A condition that must be met for the mapping to be applied. This property is only supported for `ConfigComponentsResolver`.", 1500 examples=[ 1501 "{{ components_values.get('cursor_field', None) }}", 1502 "{{ '_incremental' in components_values.get('stream_name', '') }}", 1503 ], 1504 title="Condition", 1505 ) 1506 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters") 1507 1508 1509class StreamConfig(BaseModel): 1510 type: Literal["StreamConfig"] 1511 configs_pointer: List[str] = Field( 1512 ..., 1513 description="A list of potentially nested fields indicating the full path in source config file where streams configs located.", 1514 examples=[["data"], ["data", "streams"], ["data", "{{ parameters.name }}"]], 1515 title="Configs Pointer", 1516 ) 1517 default_values: Optional[List[Dict[str, Any]]] = Field( 1518 None, 1519 description="A list of default values, each matching the structure expected from the parsed component value.", 1520 title="Default Values", 1521 ) 1522 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters") 1523 1524 1525class ConfigComponentsResolver(BaseModel): 1526 type: Literal["ConfigComponentsResolver"] 1527 stream_config: Union[List[StreamConfig], StreamConfig] 1528 components_mapping: List[ComponentMappingDefinition] 1529 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters") 1530 1531 1532class StreamParametersDefinition(BaseModel): 1533 type: Literal["StreamParametersDefinition"] 1534 list_of_parameters_for_stream: List[Dict[str, Any]] = Field( 1535 ..., 1536 description="A list of object of parameters for stream, each object in the list represents params for one stream.", 1537 examples=[ 1538 [ 1539 { 1540 "name": "test stream", 1541 "$parameters": {"entity": "test entity"}, 1542 "primary_key": "test key", 1543 } 1544 ] 1545 ], 1546 title="Stream Parameters", 1547 ) 1548 1549 1550class ParametrizedComponentsResolver(BaseModel): 1551 type: Literal["ParametrizedComponentsResolver"] 1552 stream_parameters: StreamParametersDefinition 1553 components_mapping: List[ComponentMappingDefinition] 1554 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters") 1555 1556 1557class RequestBodyPlainText(BaseModel): 1558 type: Literal["RequestBodyPlainText"] 1559 value: str 1560 1561 1562class RequestBodyUrlEncodedForm(BaseModel): 1563 type: Literal["RequestBodyUrlEncodedForm"] 1564 value: Dict[str, str] 1565 1566 1567class RequestBodyJsonObject(BaseModel): 1568 type: Literal["RequestBodyJsonObject"] 1569 value: Dict[str, Any] 1570 1571 1572class RequestBodyGraphQlQuery(BaseModel): 1573 class Config: 1574 extra = Extra.allow 1575 1576 query: str = Field(..., description="The GraphQL query to be executed") 1577 1578 1579class ValidateAdheresToSchema(BaseModel): 1580 type: Literal["ValidateAdheresToSchema"] 1581 base_schema: Union[str, Dict[str, Any]] = Field( 1582 ..., 1583 description="The base JSON schema against which the user-provided schema will be validated.", 1584 examples=[ 1585 "{{ config['report_validation_schema'] }}", 1586 '\'{\n "$schema": "http://json-schema.org/draft-07/schema#",\n "title": "Person",\n "type": "object",\n "properties": {\n "name": {\n "type": "string",\n "description": "The person\'s name"\n },\n "age": {\n "type": "integer",\n "minimum": 0,\n "description": "The person\'s age"\n }\n },\n "required": ["name", "age"]\n}\'\n', 1587 { 1588 "$schema": "http://json-schema.org/draft-07/schema#", 1589 "title": "Person", 1590 "type": "object", 1591 "properties": { 1592 "name": {"type": "string", "description": "The person's name"}, 1593 "age": { 1594 "type": "integer", 1595 "minimum": 0, 1596 "description": "The person's age", 1597 }, 1598 }, 1599 "required": ["name", "age"], 1600 }, 1601 ], 1602 title="Base JSON Schema", 1603 ) 1604 1605 1606class CustomValidationStrategy(BaseModel): 1607 class Config: 1608 extra = Extra.allow 1609 1610 type: Literal["CustomValidationStrategy"] 1611 class_name: str = Field( 1612 ..., 1613 description="Fully-qualified name of the class that will be implementing the custom validation strategy. Has to be a sub class of ValidationStrategy. The format is `source_<name>.<package>.<class_name>`.", 1614 examples=["source_declarative_manifest.components.MyCustomValidationStrategy"], 1615 title="Class Name", 1616 ) 1617 1618 1619class ConfigRemapField(BaseModel): 1620 type: Literal["ConfigRemapField"] 1621 map: Union[Dict[str, Any], str] = Field( 1622 ..., 1623 description="A mapping of original values to new values. When a field value matches a key in this map, it will be replaced with the corresponding value.", 1624 examples=[ 1625 {"pending": "in_progress", "done": "completed", "cancelled": "terminated"}, 1626 "{{ config['status_mapping'] }}", 1627 ], 1628 title="Value Mapping", 1629 ) 1630 field_path: List[str] = Field( 1631 ..., 1632 description="The path to the field whose value should be remapped. Specified as a list of path components to navigate through nested objects.", 1633 examples=[ 1634 ["status"], 1635 ["data", "status"], 1636 ["data", "{{ config.name }}", "status"], 1637 ["data", "*", "status"], 1638 ], 1639 title="Field Path", 1640 ) 1641 1642 1643class ConfigRemoveFields(BaseModel): 1644 type: Literal["ConfigRemoveFields"] 1645 field_pointers: List[List[str]] = Field( 1646 ..., 1647 description="A list of field pointers to be removed from the config.", 1648 examples=[["tags"], [["content", "html"], ["content", "plain_text"]]], 1649 title="Field Pointers", 1650 ) 1651 condition: Optional[str] = Field( 1652 "", 1653 description="Fields will be removed if expression is evaluated to True.", 1654 examples=[ 1655 "{{ config['environemnt'] == 'sandbox' }}", 1656 "{{ property is integer }}", 1657 "{{ property|length > 5 }}", 1658 "{{ property == 'some_string_to_match' }}", 1659 ], 1660 ) 1661 1662 1663class CustomConfigTransformation(BaseModel): 1664 type: Literal["CustomConfigTransformation"] 1665 class_name: str = Field( 1666 ..., 1667 description="Fully-qualified name of the class that will be implementing the custom config transformation. The format is `source_<name>.<package>.<class_name>`.", 1668 examples=["source_declarative_manifest.components.MyCustomConfigTransformation"], 1669 ) 1670 parameters: Optional[Dict[str, Any]] = Field( 1671 None, 1672 alias="$parameters", 1673 description="Additional parameters to be passed to the custom config transformation.", 1674 ) 1675 1676 1677class AddedFieldDefinition(BaseModel): 1678 type: Literal["AddedFieldDefinition"] 1679 path: List[str] = Field( 1680 ..., 1681 description="List of strings defining the path where to add the value on the record.", 1682 examples=[["segment_id"], ["metadata", "segment_id"]], 1683 title="Path", 1684 ) 1685 value: str = Field( 1686 ..., 1687 description="Value of the new field. Use {{ record['existing_field'] }} syntax to refer to other fields in the record.", 1688 examples=[ 1689 "{{ record['updates'] }}", 1690 "{{ record['MetaData']['LastUpdatedTime'] }}", 1691 "{{ stream_partition['segment_id'] }}", 1692 ], 1693 title="Value", 1694 ) 1695 value_type: Optional[ValueType] = Field( 1696 None, 1697 description="Type of the value. If not specified, the type will be inferred from the value.", 1698 title="Value Type", 1699 ) 1700 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters") 1701 1702 1703class AddFields(BaseModel): 1704 type: Literal["AddFields"] 1705 fields: List[AddedFieldDefinition] = Field( 1706 ..., 1707 description="List of transformations (path and corresponding value) that will be added to the record.", 1708 title="Fields", 1709 ) 1710 condition: Optional[str] = Field( 1711 "", 1712 description="Fields will be added if expression is evaluated to True.", 1713 examples=[ 1714 "{{ property|string == '' }}", 1715 "{{ property is integer }}", 1716 "{{ property|length > 5 }}", 1717 "{{ property == 'some_string_to_match' }}", 1718 ], 1719 ) 1720 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters") 1721 1722 1723class ApiKeyAuthenticator(BaseModel): 1724 type: Literal["ApiKeyAuthenticator"] 1725 api_token: Optional[str] = Field( 1726 None, 1727 description="The API key to inject in the request. Fill it in the user inputs.", 1728 examples=["{{ config['api_key'] }}", "Token token={{ config['api_key'] }}"], 1729 title="API Key", 1730 ) 1731 header: Optional[str] = Field( 1732 None, 1733 description="The name of the HTTP header that will be set to the API key. This setting is deprecated, use inject_into instead. Header and inject_into can not be defined at the same time.", 1734 examples=["Authorization", "Api-Token", "X-Auth-Token"], 1735 title="Header Name", 1736 ) 1737 inject_into: Optional[RequestOption] = Field( 1738 None, 1739 description="Configure how the API Key will be sent in requests to the source API. Either inject_into or header has to be defined.", 1740 examples=[ 1741 {"inject_into": "header", "field_name": "Authorization"}, 1742 {"inject_into": "request_parameter", "field_name": "authKey"}, 1743 ], 1744 title="Inject API Key Into Outgoing HTTP Request", 1745 ) 1746 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters") 1747 1748 1749class AuthFlow(BaseModel): 1750 auth_flow_type: Optional[AuthFlowType] = Field( 1751 None, description="The type of auth to use", title="Auth flow type" 1752 ) 1753 predicate_key: Optional[List[str]] = Field( 1754 None, 1755 description="JSON path to a field in the connectorSpecification that should exist for the advanced auth to be applicable.", 1756 examples=[["credentials", "auth_type"]], 1757 title="Predicate key", 1758 ) 1759 predicate_value: Optional[str] = Field( 1760 None, 1761 description="Value of the predicate_key fields for the advanced auth to be applicable.", 1762 examples=["Oauth"], 1763 title="Predicate value", 1764 ) 1765 oauth_config_specification: Optional[OAuthConfigSpecification] = None 1766 1767 1768class CheckStream(BaseModel): 1769 type: Literal["CheckStream"] 1770 stream_names: Optional[List[str]] = Field( 1771 None, 1772 description="Names of the streams to try reading from when running a check operation.", 1773 examples=[["users"], ["users", "contacts"]], 1774 title="Stream Names", 1775 ) 1776 dynamic_streams_check_configs: Optional[List[DynamicStreamCheckConfig]] = None 1777 1778 1779class IncrementingCountCursor(BaseModel): 1780 type: Literal["IncrementingCountCursor"] 1781 cursor_field: str = Field( 1782 ..., 1783 description="The location of the value on a record that will be used as a bookmark during sync. To ensure no data loss, the API must return records in ascending order based on the cursor field. Nested fields are not supported, so the field must be at the top level of the record. You can use a combination of Add Field and Remove Field transformations to move the nested field to the top.", 1784 examples=["created_at", "{{ config['record_cursor'] }}"], 1785 title="Cursor Field", 1786 ) 1787 start_value: Optional[Union[str, int]] = Field( 1788 None, 1789 description="The value that determines the earliest record that should be synced.", 1790 examples=[0, "{{ config['start_value'] }}"], 1791 title="Start Value", 1792 ) 1793 start_value_option: Optional[RequestOption] = Field( 1794 None, 1795 description="Optionally configures how the start value will be sent in requests to the source API.", 1796 title="Inject Start Value Into Outgoing HTTP Request", 1797 ) 1798 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters") 1799 1800 1801class DatetimeBasedCursor(BaseModel): 1802 type: Literal["DatetimeBasedCursor"] 1803 clamping: Optional[Clamping] = Field( 1804 None, 1805 description="This option is used to adjust the upper and lower boundaries of each datetime window to beginning and end of the provided target period (day, week, month)", 1806 title="Date Range Clamping", 1807 ) 1808 cursor_field: str = Field( 1809 ..., 1810 description="The location of the value on a record that will be used as a bookmark during sync. To ensure no data loss, the API must return records in ascending order based on the cursor field. Nested fields are not supported, so the field must be at the top level of the record. You can use a combination of Add Field and Remove Field transformations to move the nested field to the top.", 1811 examples=["created_at", "{{ config['record_cursor'] }}"], 1812 title="Cursor Field", 1813 ) 1814 cursor_datetime_formats: Optional[List[str]] = Field( 1815 None, 1816 description="The possible formats for the cursor field, in order of preference. The first format that matches the cursor field value will be used to parse it. If not provided, the Outgoing Datetime Format will be used.\nUse placeholders starting with \"%\" to describe the format the API is using. The following placeholders are available:\n * **%s**: Epoch unix timestamp - `1686218963`\n * **%s_as_float**: Epoch unix timestamp in seconds as float with microsecond precision - `1686218963.123456`\n * **%ms**: Epoch unix timestamp - `1686218963123`\n * **%a**: Weekday (abbreviated) - `Sun`\n * **%A**: Weekday (full) - `Sunday`\n * **%w**: Weekday (decimal) - `0` (Sunday), `6` (Saturday)\n * **%d**: Day of the month (zero-padded) - `01`, `02`, ..., `31`\n * **%b**: Month (abbreviated) - `Jan`\n * **%B**: Month (full) - `January`\n * **%m**: Month (zero-padded) - `01`, `02`, ..., `12`\n * **%y**: Year (without century, zero-padded) - `00`, `01`, ..., `99`\n * **%Y**: Year (with century) - `0001`, `0002`, ..., `9999`\n * **%H**: Hour (24-hour, zero-padded) - `00`, `01`, ..., `23`\n * **%I**: Hour (12-hour, zero-padded) - `01`, `02`, ..., `12`\n * **%p**: AM/PM indicator\n * **%M**: Minute (zero-padded) - `00`, `01`, ..., `59`\n * **%S**: Second (zero-padded) - `00`, `01`, ..., `59`\n * **%f**: Microsecond (zero-padded to 6 digits) - `000000`, `000001`, ..., `999999`\n * **%_ms**: Millisecond (zero-padded to 3 digits) - `000`, `001`, ..., `999`\n * **%z**: UTC offset - `(empty)`, `+0000`, `-04:00`\n * **%Z**: Time zone name - `(empty)`, `UTC`, `GMT`\n * **%j**: Day of the year (zero-padded) - `001`, `002`, ..., `366`\n * **%U**: Week number of the year (Sunday as first day) - `00`, `01`, ..., `53`\n * **%W**: Week number of the year (Monday as first day) - `00`, `01`, ..., `53`\n * **%c**: Date and time representation - `Tue Aug 16 21:30:00 1988`\n * **%x**: Date representation - `08/16/1988`\n * **%X**: Time representation - `21:30:00`\n * **%%**: Literal '%' character\n\n Some placeholders depend on the locale of the underlying system - in most cases this locale is configured as en/US. For more information see the [Python documentation](https://docs.python.org/3/library/datetime.html#strftime-and-strptime-format-codes).\n", 1817 examples=[ 1818 "%Y-%m-%d", 1819 "%Y-%m-%d %H:%M:%S", 1820 "%Y-%m-%dT%H:%M:%S", 1821 "%Y-%m-%dT%H:%M:%SZ", 1822 "%Y-%m-%dT%H:%M:%S%z", 1823 "%Y-%m-%dT%H:%M:%S.%fZ", 1824 "%Y-%m-%dT%H:%M:%S.%f%z", 1825 "%Y-%m-%d %H:%M:%S.%f+00:00", 1826 "%s", 1827 "%ms", 1828 ], 1829 title="Cursor Datetime Formats", 1830 ) 1831 start_datetime: Union[MinMaxDatetime, str] = Field( 1832 ..., 1833 description="The datetime that determines the earliest record that should be synced.", 1834 examples=["2020-01-1T00:00:00Z", "{{ config['start_time'] }}"], 1835 title="Start Datetime", 1836 ) 1837 start_time_option: Optional[RequestOption] = Field( 1838 None, 1839 description="Optionally configures how the start datetime will be sent in requests to the source API.", 1840 title="Inject Start Time Into Outgoing HTTP Request", 1841 ) 1842 end_datetime: Optional[Union[MinMaxDatetime, str]] = Field( 1843 None, 1844 description="The datetime that determines the last record that should be synced. If not provided, `{{ now_utc() }}` will be used.", 1845 examples=["2021-01-1T00:00:00Z", "{{ now_utc() }}", "{{ day_delta(-1) }}"], 1846 title="End Datetime", 1847 ) 1848 end_time_option: Optional[RequestOption] = Field( 1849 None, 1850 description="Optionally configures how the end datetime will be sent in requests to the source API.", 1851 title="Inject End Time Into Outgoing HTTP Request", 1852 ) 1853 datetime_format: str = Field( 1854 ..., 1855 description="The datetime format used to format the datetime values that are sent in outgoing requests to the API. Use placeholders starting with \"%\" to describe the format the API is using. The following placeholders are available:\n * **%s**: Epoch unix timestamp - `1686218963`\n * **%s_as_float**: Epoch unix timestamp in seconds as float with microsecond precision - `1686218963.123456`\n * **%ms**: Epoch unix timestamp (milliseconds) - `1686218963123`\n * **%a**: Weekday (abbreviated) - `Sun`\n * **%A**: Weekday (full) - `Sunday`\n * **%w**: Weekday (decimal) - `0` (Sunday), `6` (Saturday)\n * **%d**: Day of the month (zero-padded) - `01`, `02`, ..., `31`\n * **%b**: Month (abbreviated) - `Jan`\n * **%B**: Month (full) - `January`\n * **%m**: Month (zero-padded) - `01`, `02`, ..., `12`\n * **%y**: Year (without century, zero-padded) - `00`, `01`, ..., `99`\n * **%Y**: Year (with century) - `0001`, `0002`, ..., `9999`\n * **%H**: Hour (24-hour, zero-padded) - `00`, `01`, ..., `23`\n * **%I**: Hour (12-hour, zero-padded) - `01`, `02`, ..., `12`\n * **%p**: AM/PM indicator\n * **%M**: Minute (zero-padded) - `00`, `01`, ..., `59`\n * **%S**: Second (zero-padded) - `00`, `01`, ..., `59`\n * **%f**: Microsecond (zero-padded to 6 digits) - `000000`\n * **%_ms**: Millisecond (zero-padded to 3 digits) - `000`\n * **%z**: UTC offset - `(empty)`, `+0000`, `-04:00`\n * **%Z**: Time zone name - `(empty)`, `UTC`, `GMT`\n * **%j**: Day of the year (zero-padded) - `001`, `002`, ..., `366`\n * **%U**: Week number of the year (starting Sunday) - `00`, ..., `53`\n * **%W**: Week number of the year (starting Monday) - `00`, ..., `53`\n * **%c**: Date and time - `Tue Aug 16 21:30:00 1988`\n * **%x**: Date standard format - `08/16/1988`\n * **%X**: Time standard format - `21:30:00`\n * **%%**: Literal '%' character\n\n Some placeholders depend on the locale of the underlying system - in most cases this locale is configured as en/US. For more information see the [Python documentation](https://docs.python.org/3/library/datetime.html#strftime-and-strptime-format-codes).\n", 1856 examples=["%Y-%m-%dT%H:%M:%S.%f%z", "%Y-%m-%d", "%s", "%ms", "%s_as_float"], 1857 title="Outgoing Datetime Format", 1858 ) 1859 cursor_granularity: Optional[str] = Field( 1860 None, 1861 description="Smallest increment the datetime_format has (ISO 8601 duration) that is used to ensure the start of a slice does not overlap with the end of the previous one, e.g. for %Y-%m-%d the granularity should\nbe P1D, for %Y-%m-%dT%H:%M:%SZ the granularity should be PT1S. Given this field is provided, `step` needs to be provided as well.\n * **PT0.000001S**: 1 microsecond\n * **PT0.001S**: 1 millisecond\n * **PT1S**: 1 second\n * **PT1M**: 1 minute\n * **PT1H**: 1 hour\n * **P1D**: 1 day\n", 1862 examples=["PT1S"], 1863 title="Cursor Granularity", 1864 ) 1865 is_data_feed: Optional[bool] = Field( 1866 None, 1867 description="A data feed API is an API that does not allow filtering and paginates the content from the most recent to the least recent. Given this, the CDK needs to know when to stop paginating and this field will generate a stop condition for pagination.", 1868 title="Data Feed API", 1869 ) 1870 is_client_side_incremental: Optional[bool] = Field( 1871 None, 1872 description="Set to True if the target API endpoint does not take cursor values to filter records and returns all records anyway. This will cause the connector to filter out records locally, and only emit new records from the last sync, hence incremental. This means that all records would be read from the API, but only new records will be emitted to the destination.", 1873 title="Client-side Incremental Filtering", 1874 ) 1875 is_compare_strictly: Optional[bool] = Field( 1876 False, 1877 description="Set to True if the target API does not accept queries where the start time equal the end time. This will cause those requests to be skipped.", 1878 title="Strict Start-End Time Comparison", 1879 ) 1880 global_substream_cursor: Optional[bool] = Field( 1881 False, 1882 description="Setting to True causes the connector to store the cursor as one value, instead of per-partition. This setting optimizes performance when the parent stream has thousands of partitions. Notably, the substream state is updated only at the end of the sync, which helps prevent data loss in case of a sync failure. See more info in the [docs](https://docs.airbyte.com/connector-development/config-based/understanding-the-yaml-file/incremental-syncs).", 1883 title="Global Substream Cursor", 1884 ) 1885 lookback_window: Optional[str] = Field( 1886 None, 1887 description="Time interval (ISO8601 duration) before the start_datetime to read data for, e.g. P1M for looking back one month.\n * **PT1H**: 1 hour\n * **P1D**: 1 day\n * **P1W**: 1 week\n * **P1M**: 1 month\n * **P1Y**: 1 year\n", 1888 examples=["P1D", "P{{ config['lookback_days'] }}D"], 1889 title="Lookback Window", 1890 ) 1891 partition_field_end: Optional[str] = Field( 1892 None, 1893 description="Name of the partition start time field.", 1894 examples=["ending_time"], 1895 title="Partition Field End", 1896 ) 1897 partition_field_start: Optional[str] = Field( 1898 None, 1899 description="Name of the partition end time field.", 1900 examples=["starting_time"], 1901 title="Partition Field Start", 1902 ) 1903 step: Optional[str] = Field( 1904 None, 1905 description="The size of the time window (ISO8601 duration). Given this field is provided, `cursor_granularity` needs to be provided as well.\n * **PT1H**: 1 hour\n * **P1D**: 1 day\n * **P1W**: 1 week\n * **P1M**: 1 month\n * **P1Y**: 1 year\n", 1906 examples=["P1W", "{{ config['step_increment'] }}"], 1907 title="Step", 1908 ) 1909 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters") 1910 1911 1912class FixedWindowCallRatePolicy(BaseModel): 1913 class Config: 1914 extra = Extra.allow 1915 1916 type: Literal["FixedWindowCallRatePolicy"] 1917 period: str = Field( 1918 ..., description="The time interval for the rate limit window.", title="Period" 1919 ) 1920 call_limit: int = Field( 1921 ..., 1922 description="The maximum number of calls allowed within the period.", 1923 title="Call Limit", 1924 ) 1925 matchers: List[HttpRequestRegexMatcher] = Field( 1926 ..., 1927 description="List of matchers that define which requests this policy applies to.", 1928 title="Matchers", 1929 ) 1930 1931 1932class MovingWindowCallRatePolicy(BaseModel): 1933 class Config: 1934 extra = Extra.allow 1935 1936 type: Literal["MovingWindowCallRatePolicy"] 1937 rates: List[Rate] = Field( 1938 ..., 1939 description="List of rates that define the call limits for different time intervals.", 1940 title="Rates", 1941 ) 1942 matchers: List[HttpRequestRegexMatcher] = Field( 1943 ..., 1944 description="List of matchers that define which requests this policy applies to.", 1945 title="Matchers", 1946 ) 1947 1948 1949class UnlimitedCallRatePolicy(BaseModel): 1950 class Config: 1951 extra = Extra.allow 1952 1953 type: Literal["UnlimitedCallRatePolicy"] 1954 matchers: List[HttpRequestRegexMatcher] = Field( 1955 ..., 1956 description="List of matchers that define which requests this policy applies to.", 1957 title="Matchers", 1958 ) 1959 1960 1961class DefaultErrorHandler(BaseModel): 1962 type: Literal["DefaultErrorHandler"] 1963 backoff_strategies: Optional[ 1964 List[ 1965 Union[ 1966 ConstantBackoffStrategy, 1967 ExponentialBackoffStrategy, 1968 WaitTimeFromHeader, 1969 WaitUntilTimeFromHeader, 1970 CustomBackoffStrategy, 1971 ] 1972 ] 1973 ] = Field( 1974 None, 1975 description="List of backoff strategies to use to determine how long to wait before retrying a retryable request.", 1976 title="Backoff Strategies", 1977 ) 1978 max_retries: Optional[int] = Field( 1979 5, 1980 description="The maximum number of time to retry a retryable request before giving up and failing.", 1981 examples=[5, 0, 10], 1982 title="Max Retry Count", 1983 ) 1984 response_filters: Optional[List[HttpResponseFilter]] = Field( 1985 None, 1986 description="List of response filters to iterate on when deciding how to handle an error. When using an array of multiple filters, the filters will be applied sequentially and the response will be selected if it matches any of the filter's predicate.", 1987 title="Response Filters", 1988 ) 1989 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters") 1990 1991 1992class DefaultPaginator(BaseModel): 1993 type: Literal["DefaultPaginator"] 1994 pagination_strategy: Union[ 1995 PageIncrement, OffsetIncrement, CursorPagination, CustomPaginationStrategy 1996 ] = Field( 1997 ..., 1998 description="Strategy defining how records are paginated.", 1999 title="Pagination Strategy", 2000 ) 2001 page_size_option: Optional[RequestOption] = Field( 2002 None, title="Inject Page Size Into Outgoing HTTP Request" 2003 ) 2004 page_token_option: Optional[Union[RequestOption, RequestPath]] = Field( 2005 None, 2006 description="Inject the page token into the outgoing HTTP requests by inserting it into either the request URL path or a field on the request.", 2007 title="Inject Page Token Into Outgoing HTTP Request", 2008 ) 2009 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters") 2010 2011 2012class SessionTokenRequestApiKeyAuthenticator(BaseModel): 2013 type: Literal["ApiKey"] 2014 inject_into: RequestOption = Field( 2015 ..., 2016 description="Configure how the API Key will be sent in requests to the source API.", 2017 examples=[ 2018 {"inject_into": "header", "field_name": "Authorization"}, 2019 {"inject_into": "request_parameter", "field_name": "authKey"}, 2020 ], 2021 title="Inject API Key Into Outgoing HTTP Request", 2022 ) 2023 2024 2025class ListPartitionRouter(BaseModel): 2026 type: Literal["ListPartitionRouter"] 2027 cursor_field: str = Field( 2028 ..., 2029 description='While iterating over list values, the name of field used to reference a list value. The partition value can be accessed with string interpolation. e.g. "{{ stream_partition[\'my_key\'] }}" where "my_key" is the value of the cursor_field.', 2030 examples=["section", "{{ config['section_key'] }}"], 2031 title="Current Partition Value Identifier", 2032 ) 2033 values: Union[str, List[str]] = Field( 2034 ..., 2035 description="The list of attributes being iterated over and used as input for the requests made to the source API.", 2036 examples=[["section_a", "section_b", "section_c"], "{{ config['sections'] }}"], 2037 title="Partition Values", 2038 ) 2039 request_option: Optional[RequestOption] = Field( 2040 None, 2041 description="A request option describing where the list value should be injected into and under what field name if applicable.", 2042 title="Inject Partition Value Into Outgoing HTTP Request", 2043 ) 2044 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters") 2045 2046 2047class RecordSelector(BaseModel): 2048 type: Literal["RecordSelector"] 2049 extractor: Union[DpathExtractor, CustomRecordExtractor] 2050 record_filter: Optional[Union[RecordFilter, CustomRecordFilter]] = Field( 2051 None, 2052 description="Responsible for filtering records to be emitted by the Source.", 2053 title="Record Filter", 2054 ) 2055 schema_normalization: Optional[Union[SchemaNormalization, CustomSchemaNormalization]] = Field( 2056 None, 2057 description="Responsible for normalization according to the schema.", 2058 title="Schema Normalization", 2059 ) 2060 transform_before_filtering: Optional[bool] = Field( 2061 None, 2062 description="If true, transformation will be applied before record filtering.", 2063 title="Transform Before Filtering", 2064 ) 2065 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters") 2066 2067 2068class GzipDecoder(BaseModel): 2069 type: Literal["GzipDecoder"] 2070 decoder: Union[CsvDecoder, GzipDecoder, JsonDecoder, JsonlDecoder] 2071 2072 2073class RequestBodyGraphQL(BaseModel): 2074 type: Literal["RequestBodyGraphQL"] 2075 value: RequestBodyGraphQlQuery 2076 2077 2078class DpathValidator(BaseModel): 2079 type: Literal["DpathValidator"] 2080 field_path: List[str] = Field( 2081 ..., 2082 description='List of potentially nested fields describing the full path of the field to validate. Use "*" to validate all values from an array.', 2083 examples=[ 2084 ["data"], 2085 ["data", "records"], 2086 ["data", "{{ parameters.name }}"], 2087 ["data", "*", "record"], 2088 ], 2089 title="Field Path", 2090 ) 2091 validation_strategy: Union[ValidateAdheresToSchema, CustomValidationStrategy] = Field( 2092 ..., 2093 description="The condition that the specified config value will be evaluated against", 2094 title="Validation Strategy", 2095 ) 2096 2097 2098class PredicateValidator(BaseModel): 2099 type: Literal["PredicateValidator"] 2100 value: Optional[Union[str, float, Dict[str, Any], List[Any], bool]] = Field( 2101 ..., 2102 description="The value to be validated. Can be a literal value or interpolated from configuration.", 2103 examples=[ 2104 "test-value", 2105 "{{ config['api_version'] }}", 2106 "{{ config['tenant_id'] }}", 2107 123, 2108 ], 2109 title="Value", 2110 ) 2111 validation_strategy: Union[ValidateAdheresToSchema, CustomValidationStrategy] = Field( 2112 ..., 2113 description="The validation strategy to apply to the value.", 2114 title="Validation Strategy", 2115 ) 2116 2117 2118class ConfigAddFields(BaseModel): 2119 type: Literal["ConfigAddFields"] 2120 fields: List[AddedFieldDefinition] = Field( 2121 ..., 2122 description="A list of transformations (path and corresponding value) that will be added to the config.", 2123 title="Fields", 2124 ) 2125 condition: Optional[str] = Field( 2126 "", 2127 description="Fields will be added if expression is evaluated to True.", 2128 examples=[ 2129 "{{ config['environemnt'] == 'sandbox' }}", 2130 "{{ property is integer }}", 2131 "{{ property|length > 5 }}", 2132 "{{ property == 'some_string_to_match' }}", 2133 ], 2134 ) 2135 2136 2137class CompositeErrorHandler(BaseModel): 2138 type: Literal["CompositeErrorHandler"] 2139 error_handlers: List[Union[CompositeErrorHandler, DefaultErrorHandler]] = Field( 2140 ..., 2141 description="List of error handlers to iterate on to determine how to handle a failed response.", 2142 title="Error Handlers", 2143 ) 2144 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters") 2145 2146 2147class HTTPAPIBudget(BaseModel): 2148 class Config: 2149 extra = Extra.allow 2150 2151 type: Literal["HTTPAPIBudget"] 2152 policies: List[ 2153 Union[ 2154 FixedWindowCallRatePolicy, 2155 MovingWindowCallRatePolicy, 2156 UnlimitedCallRatePolicy, 2157 ] 2158 ] = Field( 2159 ..., 2160 description="List of call rate policies that define how many calls are allowed.", 2161 title="Policies", 2162 ) 2163 ratelimit_reset_header: Optional[str] = Field( 2164 "ratelimit-reset", 2165 description="The HTTP response header name that indicates when the rate limit resets.", 2166 title="Rate Limit Reset Header", 2167 ) 2168 ratelimit_remaining_header: Optional[str] = Field( 2169 "ratelimit-remaining", 2170 description="The HTTP response header name that indicates the number of remaining allowed calls.", 2171 title="Rate Limit Remaining Header", 2172 ) 2173 status_codes_for_ratelimit_hit: Optional[List[int]] = Field( 2174 [429], 2175 description="List of HTTP status codes that indicate a rate limit has been hit.", 2176 title="Status Codes for Rate Limit Hit", 2177 ) 2178 2179 2180class ZipfileDecoder(BaseModel): 2181 class Config: 2182 extra = Extra.allow 2183 2184 type: Literal["ZipfileDecoder"] 2185 decoder: Union[CsvDecoder, GzipDecoder, JsonDecoder, JsonlDecoder] = Field( 2186 ..., 2187 description="Parser to parse the decompressed data from the zipfile(s).", 2188 title="Parser", 2189 ) 2190 2191 2192class ConfigMigration(BaseModel): 2193 type: Literal["ConfigMigration"] 2194 description: Optional[str] = Field( 2195 None, description="The description/purpose of the config migration." 2196 ) 2197 transformations: List[ 2198 Union[ 2199 ConfigRemapField, 2200 ConfigAddFields, 2201 ConfigRemoveFields, 2202 CustomConfigTransformation, 2203 ] 2204 ] = Field( 2205 ..., 2206 description="The list of transformations that will attempt to be applied on an incoming unmigrated config. The transformations will be applied in the order they are defined.", 2207 title="Transformations", 2208 ) 2209 2210 2211class ConfigNormalizationRules(BaseModel): 2212 class Config: 2213 extra = Extra.forbid 2214 2215 type: Literal["ConfigNormalizationRules"] 2216 config_migrations: Optional[List[ConfigMigration]] = Field( 2217 [], 2218 description="The discrete migrations that will be applied on the incoming config. Each migration will be applied in the order they are defined.", 2219 title="Config Migrations", 2220 ) 2221 transformations: Optional[ 2222 List[ 2223 Union[ 2224 ConfigRemapField, 2225 ConfigAddFields, 2226 ConfigRemoveFields, 2227 CustomConfigTransformation, 2228 ] 2229 ] 2230 ] = Field( 2231 [], 2232 description="The list of transformations that will be applied on the incoming config at the start of each sync. The transformations will be applied in the order they are defined.", 2233 title="Transformations", 2234 ) 2235 validations: Optional[List[Union[DpathValidator, PredicateValidator]]] = Field( 2236 [], 2237 description="The list of validations that will be performed on the incoming config at the start of each sync.", 2238 title="Validations", 2239 ) 2240 2241 2242class Spec(BaseModel): 2243 type: Literal["Spec"] 2244 connection_specification: Dict[str, Any] = Field( 2245 ..., 2246 description="A connection specification describing how a the connector can be configured.", 2247 title="Connection Specification", 2248 ) 2249 documentation_url: Optional[str] = Field( 2250 None, 2251 description="URL of the connector's documentation page.", 2252 examples=["https://docs.airbyte.com/integrations/sources/dremio"], 2253 title="Documentation URL", 2254 ) 2255 advanced_auth: Optional[AuthFlow] = Field( 2256 None, 2257 description="Advanced specification for configuring the authentication flow.", 2258 title="Advanced Auth", 2259 ) 2260 config_normalization_rules: Optional[ConfigNormalizationRules] = Field( 2261 None, title="Config Normalization Rules" 2262 ) 2263 2264 2265class DeclarativeSource1(BaseModel): 2266 class Config: 2267 extra = Extra.forbid 2268 2269 type: Literal["DeclarativeSource"] 2270 check: Union[CheckStream, CheckDynamicStream] 2271 streams: List[Union[ConditionalStreams, DeclarativeStream, StateDelegatingStream]] 2272 dynamic_streams: Optional[List[DynamicDeclarativeStream]] = None 2273 version: str = Field( 2274 ..., 2275 description="The version of the Airbyte CDK used to build and test the source.", 2276 ) 2277 schemas: Optional[Schemas] = None 2278 definitions: Optional[Dict[str, Any]] = None 2279 spec: Optional[Spec] = None 2280 concurrency_level: Optional[ConcurrencyLevel] = None 2281 api_budget: Optional[HTTPAPIBudget] = None 2282 max_concurrent_async_job_count: Optional[Union[int, str]] = Field( 2283 None, 2284 description="Maximum number of concurrent asynchronous jobs to run. This property is only relevant for sources/streams that support asynchronous job execution through the AsyncRetriever (e.g. a report-based stream that initiates a job, polls the job status, and then fetches the job results). This is often set by the API's maximum number of concurrent jobs on the account level. Refer to the API's documentation for this information.", 2285 examples=[3, "{{ config['max_concurrent_async_job_count'] }}"], 2286 title="Maximum Concurrent Asynchronous Jobs", 2287 ) 2288 metadata: Optional[Dict[str, Any]] = Field( 2289 None, 2290 description="For internal Airbyte use only - DO NOT modify manually. Used by consumers of declarative manifests for storing related metadata.", 2291 ) 2292 description: Optional[str] = Field( 2293 None, 2294 description="A description of the connector. It will be presented on the Source documentation page.", 2295 ) 2296 2297 2298class DeclarativeSource2(BaseModel): 2299 class Config: 2300 extra = Extra.forbid 2301 2302 type: Literal["DeclarativeSource"] 2303 check: Union[CheckStream, CheckDynamicStream] 2304 streams: Optional[List[Union[ConditionalStreams, DeclarativeStream, StateDelegatingStream]]] = ( 2305 None 2306 ) 2307 dynamic_streams: List[DynamicDeclarativeStream] 2308 version: str = Field( 2309 ..., 2310 description="The version of the Airbyte CDK used to build and test the source.", 2311 ) 2312 schemas: Optional[Schemas] = None 2313 definitions: Optional[Dict[str, Any]] = None 2314 spec: Optional[Spec] = None 2315 concurrency_level: Optional[ConcurrencyLevel] = None 2316 api_budget: Optional[HTTPAPIBudget] = None 2317 max_concurrent_async_job_count: Optional[Union[int, str]] = Field( 2318 None, 2319 description="Maximum number of concurrent asynchronous jobs to run. This property is only relevant for sources/streams that support asynchronous job execution through the AsyncRetriever (e.g. a report-based stream that initiates a job, polls the job status, and then fetches the job results). This is often set by the API's maximum number of concurrent jobs on the account level. Refer to the API's documentation for this information.", 2320 examples=[3, "{{ config['max_concurrent_async_job_count'] }}"], 2321 title="Maximum Concurrent Asynchronous Jobs", 2322 ) 2323 metadata: Optional[Dict[str, Any]] = Field( 2324 None, 2325 description="For internal Airbyte use only - DO NOT modify manually. Used by consumers of declarative manifests for storing related metadata.", 2326 ) 2327 description: Optional[str] = Field( 2328 None, 2329 description="A description of the connector. It will be presented on the Source documentation page.", 2330 ) 2331 2332 2333class DeclarativeSource(BaseModel): 2334 class Config: 2335 extra = Extra.forbid 2336 2337 __root__: Union[DeclarativeSource1, DeclarativeSource2] = Field( 2338 ..., 2339 description="An API source that extracts data according to its declarative components.", 2340 title="DeclarativeSource", 2341 ) 2342 2343 2344class SelectiveAuthenticator(BaseModel): 2345 class Config: 2346 extra = Extra.allow 2347 2348 type: Literal["SelectiveAuthenticator"] 2349 authenticator_selection_path: List[str] = Field( 2350 ..., 2351 description="Path of the field in config with selected authenticator name", 2352 examples=[["auth"], ["auth", "type"]], 2353 title="Authenticator Selection Path", 2354 ) 2355 authenticators: Dict[ 2356 str, 2357 Union[ 2358 ApiKeyAuthenticator, 2359 BasicHttpAuthenticator, 2360 BearerAuthenticator, 2361 OAuthAuthenticator, 2362 JwtAuthenticator, 2363 SessionTokenAuthenticator, 2364 LegacySessionTokenAuthenticator, 2365 CustomAuthenticator, 2366 NoAuth, 2367 ], 2368 ] = Field( 2369 ..., 2370 description="Authenticators to select from.", 2371 examples=[ 2372 { 2373 "authenticators": { 2374 "token": "#/definitions/ApiKeyAuthenticator", 2375 "oauth": "#/definitions/OAuthAuthenticator", 2376 "jwt": "#/definitions/JwtAuthenticator", 2377 } 2378 } 2379 ], 2380 title="Authenticators", 2381 ) 2382 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters") 2383 2384 2385class ConditionalStreams(BaseModel): 2386 type: Literal["ConditionalStreams"] 2387 condition: str = Field( 2388 ..., 2389 description="Condition that will be evaluated to determine if a set of streams should be available.", 2390 examples=["{{ config['is_sandbox'] }}"], 2391 title="Condition", 2392 ) 2393 streams: List[DeclarativeStream] = Field( 2394 ..., 2395 description="Streams that will be used during an operation based on the condition.", 2396 title="Streams", 2397 ) 2398 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters") 2399 2400 2401class FileUploader(BaseModel): 2402 type: Literal["FileUploader"] 2403 requester: Union[HttpRequester, CustomRequester] = Field( 2404 ..., 2405 description="Requester component that describes how to prepare HTTP requests to send to the source API.", 2406 ) 2407 download_target_extractor: Union[DpathExtractor, CustomRecordExtractor] = Field( 2408 ..., 2409 description="Responsible for fetching the url where the file is located. This is applied on each records and not on the HTTP response", 2410 ) 2411 file_extractor: Optional[Union[DpathExtractor, CustomRecordExtractor]] = Field( 2412 None, 2413 description="Responsible for fetching the content of the file. If not defined, the assumption is that the whole response body is the file content", 2414 ) 2415 filename_extractor: Optional[str] = Field( 2416 None, 2417 description="Defines the name to store the file. Stream name is automatically added to the file path. File unique ID can be used to avoid overwriting files. Random UUID will be used if the extractor is not provided.", 2418 examples=[ 2419 "{{ record.id }}/{{ record.file_name }}/", 2420 "{{ record.id }}_{{ record.file_name }}/", 2421 ], 2422 ) 2423 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters") 2424 2425 2426class DeclarativeStream(BaseModel): 2427 class Config: 2428 extra = Extra.allow 2429 2430 type: Literal["DeclarativeStream"] 2431 name: Optional[str] = Field("", description="The stream name.", example=["Users"], title="Name") 2432 retriever: Union[SimpleRetriever, AsyncRetriever, CustomRetriever] = Field( 2433 ..., 2434 description="Component used to coordinate how records are extracted across stream slices and request pages.", 2435 title="Retriever", 2436 ) 2437 incremental_sync: Optional[ 2438 Union[DatetimeBasedCursor, IncrementingCountCursor, CustomIncrementalSync] 2439 ] = Field( 2440 None, 2441 description="Component used to fetch data incrementally based on a time field in the data.", 2442 title="Incremental Sync", 2443 ) 2444 primary_key: Optional[PrimaryKey] = Field("", title="Primary Key") 2445 schema_loader: Optional[ 2446 Union[ 2447 InlineSchemaLoader, 2448 DynamicSchemaLoader, 2449 JsonFileSchemaLoader, 2450 List[ 2451 Union[ 2452 InlineSchemaLoader, 2453 DynamicSchemaLoader, 2454 JsonFileSchemaLoader, 2455 CustomSchemaLoader, 2456 ] 2457 ], 2458 CustomSchemaLoader, 2459 ] 2460 ] = Field( 2461 None, 2462 description="One or many schema loaders can be used to retrieve the schema for the current stream. When multiple schema loaders are defined, schema properties will be merged together. Schema loaders defined first taking precedence in the event of a conflict.", 2463 title="Schema Loader", 2464 ) 2465 transformations: Optional[ 2466 List[ 2467 Union[ 2468 AddFields, 2469 RemoveFields, 2470 KeysToLower, 2471 KeysToSnakeCase, 2472 FlattenFields, 2473 DpathFlattenFields, 2474 KeysReplace, 2475 CustomTransformation, 2476 ] 2477 ] 2478 ] = Field( 2479 None, 2480 description="A list of transformations to be applied to each output record.", 2481 title="Transformations", 2482 ) 2483 state_migrations: Optional[ 2484 List[Union[LegacyToPerPartitionStateMigration, CustomStateMigration]] 2485 ] = Field( 2486 [], 2487 description="Array of state migrations to be applied on the input state", 2488 title="State Migrations", 2489 ) 2490 file_uploader: Optional[FileUploader] = Field( 2491 None, 2492 description="(experimental) Describes how to fetch a file", 2493 title="File Uploader", 2494 ) 2495 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters") 2496 2497 2498class SessionTokenAuthenticator(BaseModel): 2499 type: Literal["SessionTokenAuthenticator"] 2500 login_requester: HttpRequester = Field( 2501 ..., 2502 description="Description of the request to perform to obtain a session token to perform data requests. The response body is expected to be a JSON object with a session token property.", 2503 examples=[ 2504 { 2505 "type": "HttpRequester", 2506 "url_base": "https://my_api.com", 2507 "path": "/login", 2508 "authenticator": { 2509 "type": "BasicHttpAuthenticator", 2510 "username": "{{ config.username }}", 2511 "password": "{{ config.password }}", 2512 }, 2513 } 2514 ], 2515 title="Login Requester", 2516 ) 2517 session_token_path: List[str] = Field( 2518 ..., 2519 description="The path in the response body returned from the login requester to the session token.", 2520 examples=[["access_token"], ["result", "token"]], 2521 title="Session Token Path", 2522 ) 2523 expiration_duration: Optional[str] = Field( 2524 None, 2525 description="The duration in ISO 8601 duration notation after which the session token expires, starting from the time it was obtained. Omitting it will result in the session token being refreshed for every request.\n * **PT1H**: 1 hour\n * **P1D**: 1 day\n * **P1W**: 1 week\n * **P1M**: 1 month\n * **P1Y**: 1 year\n", 2526 examples=["PT1H", "P1D"], 2527 title="Expiration Duration", 2528 ) 2529 request_authentication: Union[ 2530 SessionTokenRequestApiKeyAuthenticator, SessionTokenRequestBearerAuthenticator 2531 ] = Field( 2532 ..., 2533 description="Authentication method to use for requests sent to the API, specifying how to inject the session token.", 2534 title="Data Request Authentication", 2535 ) 2536 decoder: Optional[Union[JsonDecoder, XmlDecoder]] = Field( 2537 None, description="Component used to decode the response.", title="Decoder" 2538 ) 2539 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters") 2540 2541 2542class HttpRequester(BaseModelWithDeprecations): 2543 type: Literal["HttpRequester"] 2544 url_base: Optional[str] = Field( 2545 None, 2546 deprecated=True, 2547 deprecation_message="Use `url` field instead.", 2548 description="Deprecated, use the `url` instead. Base URL of the API source. Do not put sensitive information (e.g. API tokens) into this field - Use the Authenticator component for this.", 2549 examples=[ 2550 "https://connect.squareup.com/v2", 2551 "{{ config['base_url'] or 'https://app.posthog.com'}}/api", 2552 "https://connect.squareup.com/v2/quotes/{{ stream_partition['id'] }}/quote_line_groups", 2553 "https://example.com/api/v1/resource/{{ next_page_token['id'] }}", 2554 ], 2555 title="API Base URL", 2556 ) 2557 url: Optional[str] = Field( 2558 None, 2559 description="The URL of the source API endpoint. Do not put sensitive information (e.g. API tokens) into this field - Use the Authenticator component for this.", 2560 examples=[ 2561 "https://connect.squareup.com/v2", 2562 "{{ config['url'] or 'https://app.posthog.com'}}/api", 2563 "https://connect.squareup.com/v2/quotes/{{ stream_partition['id'] }}/quote_line_groups", 2564 "https://example.com/api/v1/resource/{{ next_page_token['id'] }}", 2565 ], 2566 title="API Endpoint URL", 2567 ) 2568 path: Optional[str] = Field( 2569 None, 2570 deprecated=True, 2571 deprecation_message="Use `url` field instead.", 2572 description="Deprecated, use the `url` instead. Path the specific API endpoint that this stream represents. Do not put sensitive information (e.g. API tokens) into this field - Use the Authenticator component for this.", 2573 examples=[ 2574 "/products", 2575 "/quotes/{{ stream_partition['id'] }}/quote_line_groups", 2576 "/trades/{{ config['symbol_id'] }}/history", 2577 ], 2578 title="URL Path", 2579 ) 2580 http_method: Optional[HttpMethod] = Field( 2581 HttpMethod.GET, 2582 description="The HTTP method used to fetch data from the source (can be GET or POST).", 2583 examples=["GET", "POST"], 2584 title="HTTP Method", 2585 ) 2586 authenticator: Optional[ 2587 Union[ 2588 ApiKeyAuthenticator, 2589 BasicHttpAuthenticator, 2590 BearerAuthenticator, 2591 OAuthAuthenticator, 2592 JwtAuthenticator, 2593 SessionTokenAuthenticator, 2594 SelectiveAuthenticator, 2595 CustomAuthenticator, 2596 NoAuth, 2597 LegacySessionTokenAuthenticator, 2598 ] 2599 ] = Field( 2600 None, 2601 description="Authentication method to use for requests sent to the API.", 2602 title="Authenticator", 2603 ) 2604 fetch_properties_from_endpoint: Optional[PropertiesFromEndpoint] = Field( 2605 None, 2606 deprecated=True, 2607 deprecation_message="Use `query_properties` field instead.", 2608 description="Allows for retrieving a dynamic set of properties from an API endpoint which can be injected into outbound request using the stream_partition.extra_fields.", 2609 title="Fetch Properties from Endpoint", 2610 ) 2611 query_properties: Optional[QueryProperties] = Field( 2612 None, 2613 description="For APIs that require explicit specification of the properties to query for, this component will take a static or dynamic set of properties (which can be optionally split into chunks) and allow them to be injected into an outbound request by accessing stream_partition.extra_fields.", 2614 title="Query Properties", 2615 ) 2616 request_parameters: Optional[Union[Dict[str, Union[str, QueryProperties]], str]] = Field( 2617 None, 2618 description="Specifies the query parameters that should be set on an outgoing HTTP request given the inputs.", 2619 examples=[ 2620 {"unit": "day"}, 2621 { 2622 "query": 'last_event_time BETWEEN TIMESTAMP "{{ stream_interval.start_time }}" AND TIMESTAMP "{{ stream_interval.end_time }}"' 2623 }, 2624 {"searchIn": "{{ ','.join(config.get('search_in', [])) }}"}, 2625 {"sort_by[asc]": "updated_at"}, 2626 ], 2627 title="Query Parameters", 2628 ) 2629 request_headers: Optional[Union[Dict[str, str], str]] = Field( 2630 None, 2631 description="Return any non-auth headers. Authentication headers will overwrite any overlapping headers returned from this method.", 2632 examples=[{"Output-Format": "JSON"}, {"Version": "{{ config['version'] }}"}], 2633 title="Request Headers", 2634 ) 2635 request_body_data: Optional[Union[Dict[str, str], str]] = Field( 2636 None, 2637 deprecated=True, 2638 deprecation_message="Use `request_body` field instead.", 2639 description="Specifies how to populate the body of the request with a non-JSON payload. Plain text will be sent as is, whereas objects will be converted to a urlencoded form.", 2640 examples=[ 2641 '[{"clause": {"type": "timestamp", "operator": 10, "parameters":\n [{"value": {{ stream_interval[\'start_time\'] | int * 1000 }} }]\n }, "orderBy": 1, "columnName": "Timestamp"}]/\n' 2642 ], 2643 title="Request Body Payload (Non-JSON)", 2644 ) 2645 request_body_json: Optional[Union[Dict[str, Any], str]] = Field( 2646 None, 2647 deprecated=True, 2648 deprecation_message="Use `request_body` field instead.", 2649 description="Specifies how to populate the body of the request with a JSON payload. Can contain nested objects.", 2650 examples=[ 2651 {"sort_order": "ASC", "sort_field": "CREATED_AT"}, 2652 {"key": "{{ config['value'] }}"}, 2653 {"sort": {"field": "updated_at", "order": "ascending"}}, 2654 ], 2655 title="Request Body JSON Payload", 2656 ) 2657 request_body: Optional[ 2658 Union[ 2659 RequestBodyPlainText, 2660 RequestBodyUrlEncodedForm, 2661 RequestBodyJsonObject, 2662 RequestBodyGraphQL, 2663 ] 2664 ] = Field( 2665 None, 2666 description="Specifies how to populate the body of the request with a payload. Can contain nested objects.", 2667 title="Request Body", 2668 ) 2669 error_handler: Optional[ 2670 Union[DefaultErrorHandler, CompositeErrorHandler, CustomErrorHandler] 2671 ] = Field( 2672 None, 2673 description="Error handler component that defines how to handle errors.", 2674 title="Error Handler", 2675 ) 2676 use_cache: Optional[bool] = Field( 2677 False, 2678 description="Enables stream requests caching. This field is automatically set by the CDK.", 2679 title="Use Cache", 2680 ) 2681 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters") 2682 2683 2684class DynamicSchemaLoader(BaseModel): 2685 type: Literal["DynamicSchemaLoader"] 2686 retriever: Union[SimpleRetriever, AsyncRetriever, CustomRetriever] = Field( 2687 ..., 2688 description="Component used to coordinate how records are extracted across stream slices and request pages.", 2689 title="Retriever", 2690 ) 2691 schema_filter: Optional[Union[RecordFilter, CustomRecordFilter]] = Field( 2692 None, 2693 description="Responsible for filtering fields to be added to json schema.", 2694 title="Schema Filter", 2695 ) 2696 schema_transformations: Optional[ 2697 List[ 2698 Union[ 2699 AddFields, 2700 RemoveFields, 2701 KeysToLower, 2702 KeysToSnakeCase, 2703 FlattenFields, 2704 DpathFlattenFields, 2705 KeysReplace, 2706 CustomTransformation, 2707 ] 2708 ] 2709 ] = Field( 2710 None, 2711 description="A list of transformations to be applied to the schema.", 2712 title="Schema Transformations", 2713 ) 2714 schema_type_identifier: SchemaTypeIdentifier 2715 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters") 2716 2717 2718class ParentStreamConfig(BaseModel): 2719 type: Literal["ParentStreamConfig"] 2720 stream: Union[DeclarativeStream, StateDelegatingStream] = Field( 2721 ..., description="Reference to the parent stream.", title="Parent Stream" 2722 ) 2723 parent_key: str = Field( 2724 ..., 2725 description="The primary key of records from the parent stream that will be used during the retrieval of records for the current substream. This parent identifier field is typically a characteristic of the child records being extracted from the source API.", 2726 examples=["id", "{{ config['parent_record_id'] }}"], 2727 title="Parent Key", 2728 ) 2729 partition_field: str = Field( 2730 ..., 2731 description="While iterating over parent records during a sync, the parent_key value can be referenced by using this field.", 2732 examples=["parent_id", "{{ config['parent_partition_field'] }}"], 2733 title="Current Parent Key Value Identifier", 2734 ) 2735 request_option: Optional[RequestOption] = Field( 2736 None, 2737 description="A request option describing where the parent key value should be injected into and under what field name if applicable.", 2738 title="Request Option", 2739 ) 2740 incremental_dependency: Optional[bool] = Field( 2741 False, 2742 description="Indicates whether the parent stream should be read incrementally based on updates in the child stream.", 2743 title="Incremental Dependency", 2744 ) 2745 lazy_read_pointer: Optional[List[str]] = Field( 2746 [], 2747 description="If set, this will enable lazy reading, using the initial read of parent records to extract child records.", 2748 title="Lazy Read Pointer", 2749 ) 2750 extra_fields: Optional[List[List[str]]] = Field( 2751 None, 2752 description="Array of field paths to include as additional fields in the stream slice. Each path is an array of strings representing keys to access fields in the respective parent record. Accessible via `stream_slice.extra_fields`. Missing fields are set to `None`.", 2753 title="Extra Fields", 2754 ) 2755 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters") 2756 2757 2758class PropertiesFromEndpoint(BaseModel): 2759 type: Literal["PropertiesFromEndpoint"] 2760 property_field_path: List[str] = Field( 2761 ..., 2762 description="Describes the path to the field that should be extracted", 2763 examples=[["name"]], 2764 ) 2765 retriever: Union[SimpleRetriever, CustomRetriever] = Field( 2766 ..., 2767 description="Requester component that describes how to fetch the properties to query from a remote API endpoint.", 2768 ) 2769 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters") 2770 2771 2772class QueryProperties(BaseModel): 2773 type: Literal["QueryProperties"] 2774 property_list: Union[List[str], PropertiesFromEndpoint] = Field( 2775 ..., 2776 description="The set of properties that will be queried for in the outbound request. This can either be statically defined or dynamic based on an API endpoint", 2777 title="Property List", 2778 ) 2779 always_include_properties: Optional[List[str]] = Field( 2780 None, 2781 description="The list of properties that should be included in every set of properties when multiple chunks of properties are being requested.", 2782 title="Always Include Properties", 2783 ) 2784 property_chunking: Optional[PropertyChunking] = Field( 2785 None, 2786 description="Defines how query properties will be grouped into smaller sets for APIs with limitations on the number of properties fetched per API request.", 2787 title="Property Chunking", 2788 ) 2789 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters") 2790 2791 2792class StateDelegatingStream(BaseModel): 2793 type: Literal["StateDelegatingStream"] 2794 name: str = Field(..., description="The stream name.", example=["Users"], title="Name") 2795 full_refresh_stream: DeclarativeStream = Field( 2796 ..., 2797 description="Component used to coordinate how records are extracted across stream slices and request pages when the state is empty or not provided.", 2798 title="Full Refresh Stream", 2799 ) 2800 incremental_stream: DeclarativeStream = Field( 2801 ..., 2802 description="Component used to coordinate how records are extracted across stream slices and request pages when the state provided.", 2803 title="Incremental Stream", 2804 ) 2805 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters") 2806 2807 2808class SimpleRetriever(BaseModel): 2809 type: Literal["SimpleRetriever"] 2810 requester: Union[HttpRequester, CustomRequester] = Field( 2811 ..., 2812 description="Requester component that describes how to prepare HTTP requests to send to the source API.", 2813 ) 2814 decoder: Optional[ 2815 Union[ 2816 JsonDecoder, 2817 XmlDecoder, 2818 CsvDecoder, 2819 JsonlDecoder, 2820 GzipDecoder, 2821 IterableDecoder, 2822 ZipfileDecoder, 2823 CustomDecoder, 2824 ] 2825 ] = Field( 2826 None, 2827 description="Component decoding the response so records can be extracted.", 2828 title="HTTP Response Format", 2829 ) 2830 record_selector: RecordSelector = Field( 2831 ..., 2832 description="Component that describes how to extract records from a HTTP response.", 2833 ) 2834 paginator: Optional[Union[DefaultPaginator, NoPagination]] = Field( 2835 None, 2836 description="Paginator component that describes how to navigate through the API's pages.", 2837 ) 2838 ignore_stream_slicer_parameters_on_paginated_requests: Optional[bool] = Field( 2839 False, 2840 description="If true, the partition router and incremental request options will be ignored when paginating requests. Request options set directly on the requester will not be ignored.", 2841 ) 2842 partition_router: Optional[ 2843 Union[ 2844 SubstreamPartitionRouter, 2845 ListPartitionRouter, 2846 GroupingPartitionRouter, 2847 CustomPartitionRouter, 2848 List[ 2849 Union[ 2850 SubstreamPartitionRouter, 2851 ListPartitionRouter, 2852 GroupingPartitionRouter, 2853 CustomPartitionRouter, 2854 ] 2855 ], 2856 ] 2857 ] = Field( 2858 None, 2859 description="Used to iteratively execute requests over a set of values, such as a parent stream's records or a list of constant values.", 2860 title="Partition Router", 2861 ) 2862 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters") 2863 2864 2865class AsyncRetriever(BaseModel): 2866 type: Literal["AsyncRetriever"] 2867 record_selector: RecordSelector = Field( 2868 ..., 2869 description="Component that describes how to extract records from a HTTP response.", 2870 ) 2871 status_mapping: AsyncJobStatusMap = Field( 2872 ..., description="Async Job Status to Airbyte CDK Async Job Status mapping." 2873 ) 2874 status_extractor: Union[DpathExtractor, CustomRecordExtractor] = Field( 2875 ..., description="Responsible for fetching the actual status of the async job." 2876 ) 2877 download_target_extractor: Union[DpathExtractor, CustomRecordExtractor] = Field( 2878 ..., 2879 description="Responsible for fetching the final result `urls` provided by the completed / finished / ready async job.", 2880 ) 2881 download_extractor: Optional[ 2882 Union[DpathExtractor, CustomRecordExtractor, ResponseToFileExtractor] 2883 ] = Field(None, description="Responsible for fetching the records from provided urls.") 2884 creation_requester: Union[HttpRequester, CustomRequester] = Field( 2885 ..., 2886 description="Requester component that describes how to prepare HTTP requests to send to the source API to create the async server-side job.", 2887 ) 2888 polling_requester: Union[HttpRequester, CustomRequester] = Field( 2889 ..., 2890 description="Requester component that describes how to prepare HTTP requests to send to the source API to fetch the status of the running async job.", 2891 ) 2892 polling_job_timeout: Optional[Union[int, str]] = Field( 2893 None, 2894 description="The time in minutes after which the single Async Job should be considered as Timed Out.", 2895 ) 2896 download_target_requester: Optional[Union[HttpRequester, CustomRequester]] = Field( 2897 None, 2898 description="Requester component that describes how to prepare HTTP requests to send to the source API to extract the url from polling response by the completed async job.", 2899 ) 2900 download_requester: Union[HttpRequester, CustomRequester] = Field( 2901 ..., 2902 description="Requester component that describes how to prepare HTTP requests to send to the source API to download the data provided by the completed async job.", 2903 ) 2904 download_paginator: Optional[Union[DefaultPaginator, NoPagination]] = Field( 2905 None, 2906 description="Paginator component that describes how to navigate through the API's pages during download.", 2907 ) 2908 abort_requester: Optional[Union[HttpRequester, CustomRequester]] = Field( 2909 None, 2910 description="Requester component that describes how to prepare HTTP requests to send to the source API to abort a job once it is timed out from the source's perspective.", 2911 ) 2912 delete_requester: Optional[Union[HttpRequester, CustomRequester]] = Field( 2913 None, 2914 description="Requester component that describes how to prepare HTTP requests to send to the source API to delete a job once the records are extracted.", 2915 ) 2916 partition_router: Optional[ 2917 Union[ 2918 ListPartitionRouter, 2919 SubstreamPartitionRouter, 2920 GroupingPartitionRouter, 2921 CustomPartitionRouter, 2922 List[ 2923 Union[ 2924 ListPartitionRouter, 2925 SubstreamPartitionRouter, 2926 GroupingPartitionRouter, 2927 CustomPartitionRouter, 2928 ] 2929 ], 2930 ] 2931 ] = Field( 2932 [], 2933 description="PartitionRouter component that describes how to partition the stream, enabling incremental syncs and checkpointing.", 2934 title="Partition Router", 2935 ) 2936 decoder: Optional[ 2937 Union[ 2938 CsvDecoder, 2939 GzipDecoder, 2940 JsonDecoder, 2941 JsonlDecoder, 2942 IterableDecoder, 2943 XmlDecoder, 2944 ZipfileDecoder, 2945 CustomDecoder, 2946 ] 2947 ] = Field( 2948 None, 2949 description="Component decoding the response so records can be extracted.", 2950 title="HTTP Response Format", 2951 ) 2952 download_decoder: Optional[ 2953 Union[ 2954 CsvDecoder, 2955 GzipDecoder, 2956 JsonDecoder, 2957 JsonlDecoder, 2958 IterableDecoder, 2959 XmlDecoder, 2960 ZipfileDecoder, 2961 CustomDecoder, 2962 ] 2963 ] = Field( 2964 None, 2965 description="Component decoding the download response so records can be extracted.", 2966 title="Download HTTP Response Format", 2967 ) 2968 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters") 2969 2970 2971class SubstreamPartitionRouter(BaseModel): 2972 type: Literal["SubstreamPartitionRouter"] 2973 parent_stream_configs: List[ParentStreamConfig] = Field( 2974 ..., 2975 description="Specifies which parent streams are being iterated over and how parent records should be used to partition the child stream data set.", 2976 title="Parent Stream Configs", 2977 ) 2978 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters") 2979 2980 2981class GroupingPartitionRouter(BaseModel): 2982 type: Literal["GroupingPartitionRouter"] 2983 group_size: int = Field( 2984 ..., 2985 description="The number of partitions to include in each group. This determines how many partition values are batched together in a single slice.", 2986 examples=[10, 50], 2987 title="Group Size", 2988 ) 2989 underlying_partition_router: Union[ 2990 ListPartitionRouter, SubstreamPartitionRouter, CustomPartitionRouter 2991 ] = Field( 2992 ..., 2993 description="The partition router whose output will be grouped. This can be any valid partition router component.", 2994 title="Underlying Partition Router", 2995 ) 2996 deduplicate: Optional[bool] = Field( 2997 True, 2998 description="If true, ensures that partitions are unique within each group by removing duplicates based on the partition key.", 2999 title="Deduplicate Partitions", 3000 ) 3001 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters") 3002 3003 3004class HttpComponentsResolver(BaseModel): 3005 type: Literal["HttpComponentsResolver"] 3006 retriever: Union[SimpleRetriever, AsyncRetriever, CustomRetriever] = Field( 3007 ..., 3008 description="Component used to coordinate how records are extracted across stream slices and request pages.", 3009 title="Retriever", 3010 ) 3011 components_mapping: List[ComponentMappingDefinition] 3012 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters") 3013 3014 3015class DynamicDeclarativeStream(BaseModel): 3016 type: Literal["DynamicDeclarativeStream"] 3017 name: Optional[str] = Field( 3018 "", description="The dynamic stream name.", example=["Tables"], title="Name" 3019 ) 3020 stream_template: Union[DeclarativeStream, StateDelegatingStream] = Field( 3021 ..., description="Reference to the stream template.", title="Stream Template" 3022 ) 3023 components_resolver: Union[ 3024 HttpComponentsResolver, ConfigComponentsResolver, ParametrizedComponentsResolver 3025 ] = Field( 3026 ..., 3027 description="Component resolve and populates stream templates with components values.", 3028 title="Components Resolver", 3029 ) 3030 use_parent_parameters: Optional[bool] = Field( 3031 True, 3032 description="Whether or not to prioritize parent parameters over component parameters when constructing dynamic streams. Defaults to true for backward compatibility.", 3033 title="Use Parent Parameters", 3034 ) 3035 3036 3037ComplexFieldType.update_forward_refs() 3038GzipDecoder.update_forward_refs() 3039CompositeErrorHandler.update_forward_refs() 3040DeclarativeSource1.update_forward_refs() 3041DeclarativeSource2.update_forward_refs() 3042SelectiveAuthenticator.update_forward_refs() 3043ConditionalStreams.update_forward_refs() 3044FileUploader.update_forward_refs() 3045DeclarativeStream.update_forward_refs() 3046SessionTokenAuthenticator.update_forward_refs() 3047HttpRequester.update_forward_refs() 3048DynamicSchemaLoader.update_forward_refs() 3049ParentStreamConfig.update_forward_refs() 3050PropertiesFromEndpoint.update_forward_refs() 3051SimpleRetriever.update_forward_refs() 3052AsyncRetriever.update_forward_refs()
class
AuthFlowType(enum.Enum):
An enumeration.
oauth2_0 =
<AuthFlowType.oauth2_0: 'oauth2.0'>
oauth1_0 =
<AuthFlowType.oauth1_0: 'oauth1.0'>
class
BasicHttpAuthenticator(pydantic.v1.main.BaseModel):
24class BasicHttpAuthenticator(BaseModel): 25 type: Literal["BasicHttpAuthenticator"] 26 username: str = Field( 27 ..., 28 description="The username that will be combined with the password, base64 encoded and used to make requests. Fill it in the user inputs.", 29 examples=["{{ config['username'] }}", "{{ config['api_key'] }}"], 30 title="Username", 31 ) 32 password: Optional[str] = Field( 33 "", 34 description="The password that will be combined with the username, base64 encoded and used to make requests. Fill it in the user inputs.", 35 examples=["{{ config['password'] }}", ""], 36 title="Password", 37 ) 38 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters")
class
BearerAuthenticator(pydantic.v1.main.BaseModel):
41class BearerAuthenticator(BaseModel): 42 type: Literal["BearerAuthenticator"] 43 api_token: str = Field( 44 ..., 45 description="Token to inject as request header for authenticating with the API.", 46 examples=["{{ config['api_key'] }}", "{{ config['token'] }}"], 47 title="Bearer Token", 48 ) 49 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters")
class
DynamicStreamCheckConfig(pydantic.v1.main.BaseModel):
52class DynamicStreamCheckConfig(BaseModel): 53 type: Literal["DynamicStreamCheckConfig"] 54 dynamic_stream_name: str = Field( 55 ..., description="The dynamic stream name.", title="Dynamic Stream Name" 56 ) 57 stream_count: Optional[int] = Field( 58 0, 59 description="The number of streams to attempt reading from during a check operation. If `stream_count` exceeds the total number of available streams, the minimum of the two values will be used.", 60 title="Stream Count", 61 )
class
CheckDynamicStream(pydantic.v1.main.BaseModel):
64class CheckDynamicStream(BaseModel): 65 type: Literal["CheckDynamicStream"] 66 stream_count: int = Field( 67 ..., 68 description="Numbers of the streams to try reading from when running a check operation.", 69 title="Stream Count", 70 ) 71 use_check_availability: Optional[bool] = Field( 72 True, 73 description="Enables stream check availability. This field is automatically set by the CDK.", 74 title="Use Check Availability", 75 )
class
ConcurrencyLevel(pydantic.v1.main.BaseModel):
78class ConcurrencyLevel(BaseModel): 79 type: Optional[Literal["ConcurrencyLevel"]] = None 80 default_concurrency: Union[int, str] = Field( 81 ..., 82 description="The amount of concurrency that will applied during a sync. This value can be hardcoded or user-defined in the config if different users have varying volume thresholds in the target API.", 83 examples=[10, "{{ config['num_workers'] or 10 }}"], 84 title="Default Concurrency", 85 ) 86 max_concurrency: Optional[int] = Field( 87 None, 88 description="The maximum level of concurrency that will be used during a sync. This becomes a required field when the default_concurrency derives from the config, because it serves as a safeguard against a user-defined threshold that is too high.", 89 examples=[20, 100], 90 title="Max Concurrency", 91 ) 92 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters")
class
ConstantBackoffStrategy(pydantic.v1.main.BaseModel):
95class ConstantBackoffStrategy(BaseModel): 96 type: Literal["ConstantBackoffStrategy"] 97 backoff_time_in_seconds: Union[float, str] = Field( 98 ..., 99 description="Backoff time in seconds.", 100 examples=[30, 30.5, "{{ config['backoff_time'] }}"], 101 title="Backoff Time", 102 ) 103 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters")
class
CursorPagination(pydantic.v1.main.BaseModel):
106class CursorPagination(BaseModel): 107 type: Literal["CursorPagination"] 108 cursor_value: str = Field( 109 ..., 110 description="Value of the cursor defining the next page to fetch.", 111 examples=[ 112 "{{ headers.link.next.cursor }}", 113 "{{ last_record['key'] }}", 114 "{{ response['nextPage'] }}", 115 ], 116 title="Cursor Value", 117 ) 118 page_size: Optional[int] = Field( 119 None, 120 description="The number of records to include in each pages.", 121 examples=[100], 122 title="Page Size", 123 ) 124 stop_condition: Optional[str] = Field( 125 None, 126 description="Template string evaluating when to stop paginating.", 127 examples=[ 128 "{{ response.data.has_more is false }}", 129 "{{ 'next' not in headers['link'] }}", 130 ], 131 title="Stop Condition", 132 ) 133 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters")
class
CustomAuthenticator(pydantic.v1.main.BaseModel):
136class CustomAuthenticator(BaseModel): 137 class Config: 138 extra = Extra.allow 139 140 type: Literal["CustomAuthenticator"] 141 class_name: str = Field( 142 ..., 143 description="Fully-qualified name of the class that will be implementing the custom authentication strategy. Has to be a sub class of DeclarativeAuthenticator. The format is `source_<name>.<package>.<class_name>`.", 144 examples=["source_railz.components.ShortLivedTokenAuthenticator"], 145 title="Class Name", 146 ) 147 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters")
class
CustomAuthenticator.Config:
class
CustomBackoffStrategy(pydantic.v1.main.BaseModel):
150class CustomBackoffStrategy(BaseModel): 151 class Config: 152 extra = Extra.allow 153 154 type: Literal["CustomBackoffStrategy"] 155 class_name: str = Field( 156 ..., 157 description="Fully-qualified name of the class that will be implementing the custom backoff strategy. The format is `source_<name>.<package>.<class_name>`.", 158 examples=["source_railz.components.MyCustomBackoffStrategy"], 159 title="Class Name", 160 ) 161 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters")
class
CustomBackoffStrategy.Config:
class
CustomErrorHandler(pydantic.v1.main.BaseModel):
164class CustomErrorHandler(BaseModel): 165 class Config: 166 extra = Extra.allow 167 168 type: Literal["CustomErrorHandler"] 169 class_name: str = Field( 170 ..., 171 description="Fully-qualified name of the class that will be implementing the custom error handler. The format is `source_<name>.<package>.<class_name>`.", 172 examples=["source_railz.components.MyCustomErrorHandler"], 173 title="Class Name", 174 ) 175 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters")
class
CustomErrorHandler.Config:
class
CustomIncrementalSync(pydantic.v1.main.BaseModel):
178class CustomIncrementalSync(BaseModel): 179 class Config: 180 extra = Extra.allow 181 182 type: Literal["CustomIncrementalSync"] 183 class_name: str = Field( 184 ..., 185 description="Fully-qualified name of the class that will be implementing the custom incremental sync. The format is `source_<name>.<package>.<class_name>`.", 186 examples=["source_railz.components.MyCustomIncrementalSync"], 187 title="Class Name", 188 ) 189 cursor_field: str = Field( 190 ..., 191 description="The location of the value on a record that will be used as a bookmark during sync.", 192 ) 193 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters")
class
CustomIncrementalSync.Config:
class
CustomPaginationStrategy(pydantic.v1.main.BaseModel):
196class CustomPaginationStrategy(BaseModel): 197 class Config: 198 extra = Extra.allow 199 200 type: Literal["CustomPaginationStrategy"] 201 class_name: str = Field( 202 ..., 203 description="Fully-qualified name of the class that will be implementing the custom pagination strategy. The format is `source_<name>.<package>.<class_name>`.", 204 examples=["source_railz.components.MyCustomPaginationStrategy"], 205 title="Class Name", 206 ) 207 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters")
class
CustomPaginationStrategy.Config:
class
CustomRecordExtractor(pydantic.v1.main.BaseModel):
210class CustomRecordExtractor(BaseModel): 211 class Config: 212 extra = Extra.allow 213 214 type: Literal["CustomRecordExtractor"] 215 class_name: str = Field( 216 ..., 217 description="Fully-qualified name of the class that will be implementing the custom record extraction strategy. The format is `source_<name>.<package>.<class_name>`.", 218 examples=["source_railz.components.MyCustomRecordExtractor"], 219 title="Class Name", 220 ) 221 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters")
class
CustomRecordExtractor.Config:
class
CustomRecordFilter(pydantic.v1.main.BaseModel):
224class CustomRecordFilter(BaseModel): 225 class Config: 226 extra = Extra.allow 227 228 type: Literal["CustomRecordFilter"] 229 class_name: str = Field( 230 ..., 231 description="Fully-qualified name of the class that will be implementing the custom record filter strategy. The format is `source_<name>.<package>.<class_name>`.", 232 examples=["source_railz.components.MyCustomCustomRecordFilter"], 233 title="Class Name", 234 ) 235 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters")
class
CustomRecordFilter.Config:
class
CustomRequester(pydantic.v1.main.BaseModel):
238class CustomRequester(BaseModel): 239 class Config: 240 extra = Extra.allow 241 242 type: Literal["CustomRequester"] 243 class_name: str = Field( 244 ..., 245 description="Fully-qualified name of the class that will be implementing the custom requester strategy. The format is `source_<name>.<package>.<class_name>`.", 246 examples=["source_railz.components.MyCustomRecordExtractor"], 247 title="Class Name", 248 ) 249 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters")
class
CustomRequester.Config:
class
CustomRetriever(pydantic.v1.main.BaseModel):
252class CustomRetriever(BaseModel): 253 class Config: 254 extra = Extra.allow 255 256 type: Literal["CustomRetriever"] 257 class_name: str = Field( 258 ..., 259 description="Fully-qualified name of the class that will be implementing the custom retriever strategy. The format is `source_<name>.<package>.<class_name>`.", 260 examples=["source_railz.components.MyCustomRetriever"], 261 title="Class Name", 262 ) 263 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters")
class
CustomRetriever.Config:
class
CustomPartitionRouter(pydantic.v1.main.BaseModel):
266class CustomPartitionRouter(BaseModel): 267 class Config: 268 extra = Extra.allow 269 270 type: Literal["CustomPartitionRouter"] 271 class_name: str = Field( 272 ..., 273 description="Fully-qualified name of the class that will be implementing the custom partition router. The format is `source_<name>.<package>.<class_name>`.", 274 examples=["source_railz.components.MyCustomPartitionRouter"], 275 title="Class Name", 276 ) 277 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters")
class
CustomPartitionRouter.Config:
class
CustomSchemaLoader(pydantic.v1.main.BaseModel):
280class CustomSchemaLoader(BaseModel): 281 class Config: 282 extra = Extra.allow 283 284 type: Literal["CustomSchemaLoader"] 285 class_name: str = Field( 286 ..., 287 description="Fully-qualified name of the class that will be implementing the custom schema loader. The format is `source_<name>.<package>.<class_name>`.", 288 examples=["source_railz.components.MyCustomSchemaLoader"], 289 title="Class Name", 290 ) 291 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters")
class
CustomSchemaLoader.Config:
class
CustomSchemaNormalization(pydantic.v1.main.BaseModel):
294class CustomSchemaNormalization(BaseModel): 295 class Config: 296 extra = Extra.allow 297 298 type: Literal["CustomSchemaNormalization"] 299 class_name: str = Field( 300 ..., 301 description="Fully-qualified name of the class that will be implementing the custom normalization. The format is `source_<name>.<package>.<class_name>`.", 302 examples=[ 303 "source_amazon_seller_partner.components.LedgerDetailedViewReportsTypeTransformer" 304 ], 305 title="Class Name", 306 ) 307 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters")
class
CustomSchemaNormalization.Config:
class
CustomStateMigration(pydantic.v1.main.BaseModel):
310class CustomStateMigration(BaseModel): 311 class Config: 312 extra = Extra.allow 313 314 type: Literal["CustomStateMigration"] 315 class_name: str = Field( 316 ..., 317 description="Fully-qualified name of the class that will be implementing the custom state migration. The format is `source_<name>.<package>.<class_name>`.", 318 examples=["source_railz.components.MyCustomStateMigration"], 319 title="Class Name", 320 ) 321 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters")
class
CustomStateMigration.Config:
class
CustomTransformation(pydantic.v1.main.BaseModel):
324class CustomTransformation(BaseModel): 325 class Config: 326 extra = Extra.allow 327 328 type: Literal["CustomTransformation"] 329 class_name: str = Field( 330 ..., 331 description="Fully-qualified name of the class that will be implementing the custom transformation. The format is `source_<name>.<package>.<class_name>`.", 332 examples=["source_railz.components.MyCustomTransformation"], 333 title="Class Name", 334 ) 335 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters")
class
CustomTransformation.Config:
class
LegacyToPerPartitionStateMigration(pydantic.v1.main.BaseModel):
class
LegacyToPerPartitionStateMigration.Config:
class
Clamping(pydantic.v1.main.BaseModel):
class
Algorithm(enum.Enum):
355class Algorithm(Enum): 356 HS256 = "HS256" 357 HS384 = "HS384" 358 HS512 = "HS512" 359 ES256 = "ES256" 360 ES256K = "ES256K" 361 ES384 = "ES384" 362 ES512 = "ES512" 363 RS256 = "RS256" 364 RS384 = "RS384" 365 RS512 = "RS512" 366 PS256 = "PS256" 367 PS384 = "PS384" 368 PS512 = "PS512" 369 EdDSA = "EdDSA"
An enumeration.
HS256 =
<Algorithm.HS256: 'HS256'>
HS384 =
<Algorithm.HS384: 'HS384'>
HS512 =
<Algorithm.HS512: 'HS512'>
ES256 =
<Algorithm.ES256: 'ES256'>
ES256K =
<Algorithm.ES256K: 'ES256K'>
ES384 =
<Algorithm.ES384: 'ES384'>
ES512 =
<Algorithm.ES512: 'ES512'>
RS256 =
<Algorithm.RS256: 'RS256'>
RS384 =
<Algorithm.RS384: 'RS384'>
RS512 =
<Algorithm.RS512: 'RS512'>
PS256 =
<Algorithm.PS256: 'PS256'>
PS384 =
<Algorithm.PS384: 'PS384'>
PS512 =
<Algorithm.PS512: 'PS512'>
EdDSA =
<Algorithm.EdDSA: 'EdDSA'>
class
JwtHeaders(pydantic.v1.main.BaseModel):
372class JwtHeaders(BaseModel): 373 class Config: 374 extra = Extra.forbid 375 376 kid: Optional[str] = Field( 377 None, 378 description="Private key ID for user account.", 379 examples=["{{ config['kid'] }}"], 380 title="Key Identifier", 381 ) 382 typ: Optional[str] = Field( 383 "JWT", 384 description="The media type of the complete JWT.", 385 examples=["JWT"], 386 title="Type", 387 ) 388 cty: Optional[str] = Field( 389 None, 390 description="Content type of JWT header.", 391 examples=["JWT"], 392 title="Content Type", 393 )
class
JwtHeaders.Config:
class
JwtPayload(pydantic.v1.main.BaseModel):
396class JwtPayload(BaseModel): 397 class Config: 398 extra = Extra.forbid 399 400 iss: Optional[str] = Field( 401 None, 402 description="The user/principal that issued the JWT. Commonly a value unique to the user.", 403 examples=["{{ config['iss'] }}"], 404 title="Issuer", 405 ) 406 sub: Optional[str] = Field( 407 None, 408 description="The subject of the JWT. Commonly defined by the API.", 409 title="Subject", 410 ) 411 aud: Optional[str] = Field( 412 None, 413 description="The recipient that the JWT is intended for. Commonly defined by the API.", 414 examples=["appstoreconnect-v1"], 415 title="Audience", 416 )
class
JwtPayload.Config:
class
JwtAuthenticator(pydantic.v1.main.BaseModel):
419class JwtAuthenticator(BaseModel): 420 type: Literal["JwtAuthenticator"] 421 secret_key: str = Field( 422 ..., 423 description="Secret used to sign the JSON web token.", 424 examples=["{{ config['secret_key'] }}"], 425 title="Secret Key", 426 ) 427 base64_encode_secret_key: Optional[bool] = Field( 428 False, 429 description='When set to true, the secret key will be base64 encoded prior to being encoded as part of the JWT. Only set to "true" when required by the API.', 430 title="Base64-encode Secret Key", 431 ) 432 algorithm: Algorithm = Field( 433 ..., 434 description="Algorithm used to sign the JSON web token.", 435 examples=["ES256", "HS256", "RS256", "{{ config['algorithm'] }}"], 436 title="Algorithm", 437 ) 438 token_duration: Optional[int] = Field( 439 1200, 440 description="The amount of time in seconds a JWT token can be valid after being issued.", 441 examples=[1200, 3600], 442 title="Token Duration", 443 ) 444 header_prefix: Optional[str] = Field( 445 None, 446 description="The prefix to be used within the Authentication header.", 447 examples=["Bearer", "Basic"], 448 title="Header Prefix", 449 ) 450 jwt_headers: Optional[JwtHeaders] = Field( 451 None, 452 description="JWT headers used when signing JSON web token.", 453 title="JWT Headers", 454 ) 455 additional_jwt_headers: Optional[Dict[str, Any]] = Field( 456 None, 457 description="Additional headers to be included with the JWT headers object.", 458 title="Additional JWT Headers", 459 ) 460 jwt_payload: Optional[JwtPayload] = Field( 461 None, 462 description="JWT Payload used when signing JSON web token.", 463 title="JWT Payload", 464 ) 465 additional_jwt_payload: Optional[Dict[str, Any]] = Field( 466 None, 467 description="Additional properties to be added to the JWT payload.", 468 title="Additional JWT Payload Properties", 469 ) 470 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters")
algorithm: Algorithm
jwt_headers: Optional[JwtHeaders]
jwt_payload: Optional[JwtPayload]
class
RefreshTokenUpdater(pydantic.v1.main.BaseModel):
473class RefreshTokenUpdater(BaseModel): 474 refresh_token_name: Optional[str] = Field( 475 "refresh_token", 476 description="The name of the property which contains the updated refresh token in the response from the token refresh endpoint.", 477 examples=["refresh_token"], 478 title="Refresh Token Property Name", 479 ) 480 access_token_config_path: Optional[List[str]] = Field( 481 ["credentials", "access_token"], 482 description="Config path to the access token. Make sure the field actually exists in the config.", 483 examples=[["credentials", "access_token"], ["access_token"]], 484 title="Config Path To Access Token", 485 ) 486 refresh_token_config_path: Optional[List[str]] = Field( 487 ["credentials", "refresh_token"], 488 description="Config path to the access token. Make sure the field actually exists in the config.", 489 examples=[["credentials", "refresh_token"], ["refresh_token"]], 490 title="Config Path To Refresh Token", 491 ) 492 token_expiry_date_config_path: Optional[List[str]] = Field( 493 ["credentials", "token_expiry_date"], 494 description="Config path to the expiry date. Make sure actually exists in the config.", 495 examples=[["credentials", "token_expiry_date"]], 496 title="Config Path To Expiry Date", 497 ) 498 refresh_token_error_status_codes: Optional[List[int]] = Field( 499 [], 500 description="Status Codes to Identify refresh token error in response (Refresh Token Error Key and Refresh Token Error Values should be also specified). Responses with one of the error status code and containing an error value will be flagged as a config error", 501 examples=[[400, 500]], 502 title="Refresh Token Error Status Codes", 503 ) 504 refresh_token_error_key: Optional[str] = Field( 505 "", 506 description="Key to Identify refresh token error in response (Refresh Token Error Status Codes and Refresh Token Error Values should be also specified).", 507 examples=["error"], 508 title="Refresh Token Error Key", 509 ) 510 refresh_token_error_values: Optional[List[str]] = Field( 511 [], 512 description='List of values to check for exception during token refresh process. Used to check if the error found in the response matches the key from the Refresh Token Error Key field (e.g. response={"error": "invalid_grant"}). Only responses with one of the error status code and containing an error value will be flagged as a config error', 513 examples=[["invalid_grant", "invalid_permissions"]], 514 title="Refresh Token Error Values", 515 )
class
OAuthAuthenticator(pydantic.v1.main.BaseModel):
518class OAuthAuthenticator(BaseModel): 519 type: Literal["OAuthAuthenticator"] 520 client_id_name: Optional[str] = Field( 521 "client_id", 522 description="The name of the property to use to refresh the `access_token`.", 523 examples=["custom_app_id"], 524 title="Client ID Property Name", 525 ) 526 client_id: Optional[str] = Field( 527 None, 528 description="The OAuth client ID. Fill it in the user inputs.", 529 examples=[ 530 "{{ config['client_id'] }}", 531 "{{ config['credentials']['client_id }}", 532 ], 533 title="Client ID", 534 ) 535 client_secret_name: Optional[str] = Field( 536 "client_secret", 537 description="The name of the property to use to refresh the `access_token`.", 538 examples=["custom_app_secret"], 539 title="Client Secret Property Name", 540 ) 541 client_secret: Optional[str] = Field( 542 None, 543 description="The OAuth client secret. Fill it in the user inputs.", 544 examples=[ 545 "{{ config['client_secret'] }}", 546 "{{ config['credentials']['client_secret }}", 547 ], 548 title="Client Secret", 549 ) 550 refresh_token_name: Optional[str] = Field( 551 "refresh_token", 552 description="The name of the property to use to refresh the `access_token`.", 553 examples=["custom_app_refresh_value"], 554 title="Refresh Token Property Name", 555 ) 556 refresh_token: Optional[str] = Field( 557 None, 558 description="Credential artifact used to get a new access token.", 559 examples=[ 560 "{{ config['refresh_token'] }}", 561 "{{ config['credentials]['refresh_token'] }}", 562 ], 563 title="Refresh Token", 564 ) 565 token_refresh_endpoint: Optional[str] = Field( 566 None, 567 description="The full URL to call to obtain a new access token.", 568 examples=["https://connect.squareup.com/oauth2/token"], 569 title="Token Refresh Endpoint", 570 ) 571 access_token_name: Optional[str] = Field( 572 "access_token", 573 description="The name of the property which contains the access token in the response from the token refresh endpoint.", 574 examples=["access_token"], 575 title="Access Token Property Name", 576 ) 577 access_token_value: Optional[str] = Field( 578 None, 579 description="The value of the access_token to bypass the token refreshing using `refresh_token`.", 580 examples=["secret_access_token_value"], 581 title="Access Token Value", 582 ) 583 expires_in_name: Optional[str] = Field( 584 "expires_in", 585 description="The name of the property which contains the expiry date in the response from the token refresh endpoint.", 586 examples=["expires_in"], 587 title="Token Expiry Property Name", 588 ) 589 grant_type_name: Optional[str] = Field( 590 "grant_type", 591 description="The name of the property to use to refresh the `access_token`.", 592 examples=["custom_grant_type"], 593 title="Grant Type Property Name", 594 ) 595 grant_type: Optional[str] = Field( 596 "refresh_token", 597 description="Specifies the OAuth2 grant type. If set to refresh_token, the refresh_token needs to be provided as well. For client_credentials, only client id and secret are required. Other grant types are not officially supported.", 598 examples=["refresh_token", "client_credentials"], 599 title="Grant Type", 600 ) 601 refresh_request_body: Optional[Dict[str, Any]] = Field( 602 None, 603 description="Body of the request sent to get a new access token.", 604 examples=[ 605 { 606 "applicationId": "{{ config['application_id'] }}", 607 "applicationSecret": "{{ config['application_secret'] }}", 608 "token": "{{ config['token'] }}", 609 } 610 ], 611 title="Refresh Request Body", 612 ) 613 refresh_request_headers: Optional[Dict[str, Any]] = Field( 614 None, 615 description="Headers of the request sent to get a new access token.", 616 examples=[ 617 { 618 "Authorization": "<AUTH_TOKEN>", 619 "Content-Type": "application/x-www-form-urlencoded", 620 } 621 ], 622 title="Refresh Request Headers", 623 ) 624 scopes: Optional[List[str]] = Field( 625 None, 626 description="List of scopes that should be granted to the access token.", 627 examples=[["crm.list.read", "crm.objects.contacts.read", "crm.schema.contacts.read"]], 628 title="Scopes", 629 ) 630 token_expiry_date: Optional[str] = Field( 631 None, 632 description="The access token expiry date.", 633 examples=["2023-04-06T07:12:10.421833+00:00", 1680842386], 634 title="Token Expiry Date", 635 ) 636 token_expiry_date_format: Optional[str] = Field( 637 None, 638 description="The format of the time to expiration datetime. Provide it if the time is returned as a date-time string instead of seconds.", 639 examples=["%Y-%m-%d %H:%M:%S.%f+00:00"], 640 title="Token Expiry Date Format", 641 ) 642 refresh_token_updater: Optional[RefreshTokenUpdater] = Field( 643 None, 644 description="When the refresh token updater is defined, new refresh tokens, access tokens and the access token expiry date are written back from the authentication response to the config object. This is important if the refresh token can only used once.", 645 title="Refresh Token Updater", 646 ) 647 profile_assertion: Optional[JwtAuthenticator] = Field( 648 None, 649 description="The authenticator being used to authenticate the client authenticator.", 650 title="Profile Assertion", 651 ) 652 use_profile_assertion: Optional[bool] = Field( 653 False, 654 description="Enable using profile assertion as a flow for OAuth authorization.", 655 title="Use Profile Assertion", 656 ) 657 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters")
refresh_token_updater: Optional[RefreshTokenUpdater]
profile_assertion: Optional[JwtAuthenticator]
class
Rate(pydantic.v1.main.BaseModel):
660class Rate(BaseModel): 661 class Config: 662 extra = Extra.allow 663 664 limit: Union[int, str] = Field( 665 ..., 666 description="The maximum number of calls allowed within the interval.", 667 title="Limit", 668 ) 669 interval: str = Field( 670 ..., 671 description="The time interval for the rate limit.", 672 examples=["PT1H", "P1D"], 673 title="Interval", 674 )
class
Rate.Config:
class
HttpRequestRegexMatcher(pydantic.v1.main.BaseModel):
677class HttpRequestRegexMatcher(BaseModel): 678 class Config: 679 extra = Extra.allow 680 681 method: Optional[str] = Field( 682 None, description="The HTTP method to match (e.g., GET, POST).", title="Method" 683 ) 684 url_base: Optional[str] = Field( 685 None, 686 description='The base URL (scheme and host, e.g. "https://api.example.com") to match.', 687 title="URL Base", 688 ) 689 url_path_pattern: Optional[str] = Field( 690 None, 691 description="A regular expression pattern to match the URL path.", 692 title="URL Path Pattern", 693 ) 694 params: Optional[Dict[str, Any]] = Field( 695 None, description="The query parameters to match.", title="Parameters" 696 ) 697 headers: Optional[Dict[str, Any]] = Field( 698 None, description="The headers to match.", title="Headers" 699 )
class
HttpRequestRegexMatcher.Config:
class
DpathExtractor(pydantic.v1.main.BaseModel):
702class DpathExtractor(BaseModel): 703 type: Literal["DpathExtractor"] 704 field_path: List[str] = Field( 705 ..., 706 description='List of potentially nested fields describing the full path of the field to extract. Use "*" to extract all values from an array. See more info in the [docs](https://docs.airbyte.com/connector-development/config-based/understanding-the-yaml-file/record-selector).', 707 examples=[ 708 ["data"], 709 ["data", "records"], 710 ["data", "{{ parameters.name }}"], 711 ["data", "*", "record"], 712 ], 713 title="Field Path", 714 ) 715 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters")
class
ResponseToFileExtractor(pydantic.v1.main.BaseModel):
class
ExponentialBackoffStrategy(pydantic.v1.main.BaseModel):
723class ExponentialBackoffStrategy(BaseModel): 724 type: Literal["ExponentialBackoffStrategy"] 725 factor: Optional[Union[float, str]] = Field( 726 5, 727 description="Multiplicative constant applied on each retry.", 728 examples=[5, 5.5, "10"], 729 title="Factor", 730 ) 731 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters")
class
GroupByKeyMergeStrategy(pydantic.v1.main.BaseModel):
734class GroupByKeyMergeStrategy(BaseModel): 735 type: Literal["GroupByKeyMergeStrategy"] 736 key: Union[str, List[str]] = Field( 737 ..., 738 description="The name of the field on the record whose value will be used to group properties that were retrieved through multiple API requests.", 739 examples=["id", ["parent_id", "end_date"]], 740 title="Key", 741 ) 742 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters")
class
SessionTokenRequestBearerAuthenticator(pydantic.v1.main.BaseModel):
class
HttpMethod(enum.Enum):
An enumeration.
GET =
<HttpMethod.GET: 'GET'>
POST =
<HttpMethod.POST: 'POST'>
class
Action(enum.Enum):
754class Action(Enum): 755 SUCCESS = "SUCCESS" 756 FAIL = "FAIL" 757 RETRY = "RETRY" 758 IGNORE = "IGNORE" 759 RATE_LIMITED = "RATE_LIMITED"
An enumeration.
SUCCESS =
<Action.SUCCESS: 'SUCCESS'>
FAIL =
<Action.FAIL: 'FAIL'>
RETRY =
<Action.RETRY: 'RETRY'>
IGNORE =
<Action.IGNORE: 'IGNORE'>
RATE_LIMITED =
<Action.RATE_LIMITED: 'RATE_LIMITED'>
class
FailureType(enum.Enum):
762class FailureType(Enum): 763 system_error = "system_error" 764 config_error = "config_error" 765 transient_error = "transient_error"
An enumeration.
system_error =
<FailureType.system_error: 'system_error'>
config_error =
<FailureType.config_error: 'config_error'>
transient_error =
<FailureType.transient_error: 'transient_error'>
class
HttpResponseFilter(pydantic.v1.main.BaseModel):
768class HttpResponseFilter(BaseModel): 769 type: Literal["HttpResponseFilter"] 770 action: Optional[Action] = Field( 771 None, 772 description="Action to execute if a response matches the filter.", 773 examples=["SUCCESS", "FAIL", "RETRY", "IGNORE", "RATE_LIMITED"], 774 title="Action", 775 ) 776 failure_type: Optional[FailureType] = Field( 777 None, 778 description="Failure type of traced exception if a response matches the filter.", 779 examples=["system_error", "config_error", "transient_error"], 780 title="Failure Type", 781 ) 782 error_message: Optional[str] = Field( 783 None, 784 description="Error Message to display if the response matches the filter.", 785 title="Error Message", 786 ) 787 error_message_contains: Optional[str] = Field( 788 None, 789 description="Match the response if its error message contains the substring.", 790 example=["This API operation is not enabled for this site"], 791 title="Error Message Substring", 792 ) 793 http_codes: Optional[List[int]] = Field( 794 None, 795 description="Match the response if its HTTP code is included in this list.", 796 examples=[[420, 429], [500]], 797 title="HTTP Codes", 798 unique_items=True, 799 ) 800 predicate: Optional[str] = Field( 801 None, 802 description="Match the response if the predicate evaluates to true.", 803 examples=[ 804 "{{ 'Too much requests' in response }}", 805 "{{ 'error_code' in response and response['error_code'] == 'ComplexityException' }}", 806 ], 807 title="Predicate", 808 ) 809 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters")
action: Optional[Action]
failure_type: Optional[FailureType]
class
ComplexFieldType(pydantic.v1.main.BaseModel):
812class ComplexFieldType(BaseModel): 813 field_type: str 814 items: Optional[Union[str, ComplexFieldType]] = None
items: Union[str, ComplexFieldType, NoneType]
class
TypesMap(pydantic.v1.main.BaseModel):
817class TypesMap(BaseModel): 818 target_type: Union[str, List[str], ComplexFieldType] 819 current_type: Union[str, List[str]] 820 condition: Optional[str] = None
target_type: Union[str, List[str], ComplexFieldType]
class
SchemaTypeIdentifier(pydantic.v1.main.BaseModel):
823class SchemaTypeIdentifier(BaseModel): 824 type: Optional[Literal["SchemaTypeIdentifier"]] = None 825 schema_pointer: Optional[List[str]] = Field( 826 [], 827 description="List of nested fields defining the schema field path to extract. Defaults to [].", 828 title="Schema Path", 829 ) 830 key_pointer: List[str] = Field( 831 ..., 832 description="List of potentially nested fields describing the full path of the field key to extract.", 833 title="Key Path", 834 ) 835 type_pointer: Optional[List[str]] = Field( 836 None, 837 description="List of potentially nested fields describing the full path of the field type to extract.", 838 title="Type Path", 839 ) 840 types_mapping: Optional[List[TypesMap]] = None 841 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters")
types_mapping: Optional[List[TypesMap]]
class
InlineSchemaLoader(pydantic.v1.main.BaseModel):
844class InlineSchemaLoader(BaseModel): 845 type: Literal["InlineSchemaLoader"] 846 schema_: Optional[Dict[str, Any]] = Field( 847 None, 848 alias="schema", 849 description='Describes a streams\' schema. Refer to the <a href="https://docs.airbyte.com/understanding-airbyte/supported-data-types/">Data Types documentation</a> for more details on which types are valid.', 850 title="Schema", 851 )
class
JsonFileSchemaLoader(pydantic.v1.main.BaseModel):
854class JsonFileSchemaLoader(BaseModel): 855 type: Literal["JsonFileSchemaLoader"] 856 file_path: Optional[str] = Field( 857 None, 858 description="Path to the JSON file defining the schema. The path is relative to the connector module's root.", 859 example=["./schemas/users.json"], 860 title="File Path", 861 ) 862 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters")
class
JsonDecoder(pydantic.v1.main.BaseModel):
class
JsonlDecoder(pydantic.v1.main.BaseModel):
class
KeysToLower(pydantic.v1.main.BaseModel):
class
KeysToSnakeCase(pydantic.v1.main.BaseModel):
class
FlattenFields(pydantic.v1.main.BaseModel):
883class FlattenFields(BaseModel): 884 type: Literal["FlattenFields"] 885 flatten_lists: Optional[bool] = Field( 886 True, 887 description="Whether to flatten lists or leave it as is. Default is True.", 888 title="Flatten Lists", 889 ) 890 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters")
class
KeyTransformation(pydantic.v1.main.BaseModel):
893class KeyTransformation(BaseModel): 894 type: Literal["KeyTransformation"] 895 prefix: Optional[str] = Field( 896 None, 897 description="Prefix to add for object keys. If not provided original keys remain unchanged.", 898 examples=["flattened_"], 899 title="Key Prefix", 900 ) 901 suffix: Optional[str] = Field( 902 None, 903 description="Suffix to add for object keys. If not provided original keys remain unchanged.", 904 examples=["_flattened"], 905 title="Key Suffix", 906 )
class
DpathFlattenFields(pydantic.v1.main.BaseModel):
909class DpathFlattenFields(BaseModel): 910 type: Literal["DpathFlattenFields"] 911 field_path: List[str] = Field( 912 ..., 913 description="A path to field that needs to be flattened.", 914 examples=[["data"], ["data", "*", "field"]], 915 title="Field Path", 916 ) 917 delete_origin_value: Optional[bool] = Field( 918 None, 919 description="Whether to delete the origin value or keep it. Default is False.", 920 title="Delete Origin Value", 921 ) 922 replace_record: Optional[bool] = Field( 923 None, 924 description="Whether to replace the origin record or not. Default is False.", 925 title="Replace Origin Record", 926 ) 927 key_transformation: Optional[KeyTransformation] = Field( 928 None, 929 description="Transformation for object keys. If not provided, original key will be used.", 930 title="Key transformation", 931 ) 932 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters")
key_transformation: Optional[KeyTransformation]
class
KeysReplace(pydantic.v1.main.BaseModel):
935class KeysReplace(BaseModel): 936 type: Literal["KeysReplace"] 937 old: str = Field( 938 ..., 939 description="Old value to replace.", 940 examples=[ 941 " ", 942 "{{ record.id }}", 943 "{{ config['id'] }}", 944 "{{ stream_slice['id'] }}", 945 ], 946 title="Old value", 947 ) 948 new: str = Field( 949 ..., 950 description="New value to set.", 951 examples=[ 952 "_", 953 "{{ record.id }}", 954 "{{ config['id'] }}", 955 "{{ stream_slice['id'] }}", 956 ], 957 title="New value", 958 ) 959 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters")
class
IterableDecoder(pydantic.v1.main.BaseModel):
class
XmlDecoder(pydantic.v1.main.BaseModel):
class
CustomDecoder(pydantic.v1.main.BaseModel):
970class CustomDecoder(BaseModel): 971 class Config: 972 extra = Extra.allow 973 974 type: Literal["CustomDecoder"] 975 class_name: str = Field( 976 ..., 977 description="Fully-qualified name of the class that will be implementing the custom decoding. Has to be a sub class of Decoder. The format is `source_<name>.<package>.<class_name>`.", 978 examples=["source_amazon_ads.components.GzipJsonlDecoder"], 979 title="Class Name", 980 ) 981 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters")
class
CustomDecoder.Config:
class
MinMaxDatetime(pydantic.v1.main.BaseModel):
984class MinMaxDatetime(BaseModel): 985 type: Literal["MinMaxDatetime"] 986 datetime: str = Field( 987 ..., 988 description="Datetime value.", 989 examples=[ 990 "2021-01-01", 991 "2021-01-01T00:00:00Z", 992 "{{ config['start_time'] }}", 993 "{{ now_utc().strftime('%Y-%m-%dT%H:%M:%SZ') }}", 994 ], 995 title="Datetime", 996 ) 997 datetime_format: Optional[str] = Field( 998 "", 999 description='Format of the datetime value. Defaults to "%Y-%m-%dT%H:%M:%S.%f%z" if left empty. Use placeholders starting with "%" to describe the format the API is using. The following placeholders are available:\n * **%s**: Epoch unix timestamp - `1686218963`\n * **%s_as_float**: Epoch unix timestamp in seconds as float with microsecond precision - `1686218963.123456`\n * **%ms**: Epoch unix timestamp - `1686218963123`\n * **%a**: Weekday (abbreviated) - `Sun`\n * **%A**: Weekday (full) - `Sunday`\n * **%w**: Weekday (decimal) - `0` (Sunday), `6` (Saturday)\n * **%d**: Day of the month (zero-padded) - `01`, `02`, ..., `31`\n * **%b**: Month (abbreviated) - `Jan`\n * **%B**: Month (full) - `January`\n * **%m**: Month (zero-padded) - `01`, `02`, ..., `12`\n * **%y**: Year (without century, zero-padded) - `00`, `01`, ..., `99`\n * **%Y**: Year (with century) - `0001`, `0002`, ..., `9999`\n * **%H**: Hour (24-hour, zero-padded) - `00`, `01`, ..., `23`\n * **%I**: Hour (12-hour, zero-padded) - `01`, `02`, ..., `12`\n * **%p**: AM/PM indicator\n * **%M**: Minute (zero-padded) - `00`, `01`, ..., `59`\n * **%S**: Second (zero-padded) - `00`, `01`, ..., `59`\n * **%f**: Microsecond (zero-padded to 6 digits) - `000000`, `000001`, ..., `999999`\n * **%_ms**: Millisecond (zero-padded to 3 digits) - `000`, `001`, ..., `999`\n * **%z**: UTC offset - `(empty)`, `+0000`, `-04:00`\n * **%Z**: Time zone name - `(empty)`, `UTC`, `GMT`\n * **%j**: Day of the year (zero-padded) - `001`, `002`, ..., `366`\n * **%U**: Week number of the year (Sunday as first day) - `00`, `01`, ..., `53`\n * **%W**: Week number of the year (Monday as first day) - `00`, `01`, ..., `53`\n * **%c**: Date and time representation - `Tue Aug 16 21:30:00 1988`\n * **%x**: Date representation - `08/16/1988`\n * **%X**: Time representation - `21:30:00`\n * **%%**: Literal \'%\' character\n\n Some placeholders depend on the locale of the underlying system - in most cases this locale is configured as en/US. For more information see the [Python documentation](https://docs.python.org/3/library/datetime.html#strftime-and-strptime-format-codes).\n', 1000 examples=["%Y-%m-%dT%H:%M:%S.%f%z", "%Y-%m-%d", "%s"], 1001 title="Datetime Format", 1002 ) 1003 max_datetime: Optional[str] = Field( 1004 None, 1005 description="Ceiling applied on the datetime value. Must be formatted with the datetime_format field.", 1006 examples=["2021-01-01T00:00:00Z", "2021-01-01"], 1007 title="Max Datetime", 1008 ) 1009 min_datetime: Optional[str] = Field( 1010 None, 1011 description="Floor applied on the datetime value. Must be formatted with the datetime_format field.", 1012 examples=["2010-01-01T00:00:00Z", "2010-01-01"], 1013 title="Min Datetime", 1014 ) 1015 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters")
class
NoAuth(pydantic.v1.main.BaseModel):
class
NoPagination(pydantic.v1.main.BaseModel):
class
State(pydantic.v1.main.BaseModel):
class
State.Config:
class
OauthConnectorInputSpecification(pydantic.v1.main.BaseModel):
1035class OauthConnectorInputSpecification(BaseModel): 1036 class Config: 1037 extra = Extra.allow 1038 1039 consent_url: str = Field( 1040 ..., 1041 description="The DeclarativeOAuth Specific string URL string template to initiate the authentication.\nThe placeholders are replaced during the processing to provide neccessary values.", 1042 examples=[ 1043 "https://domain.host.com/marketing_api/auth?{{client_id_key}}={{client_id_value}}&{{redirect_uri_key}}={{{{redirect_uri_value}} | urlEncoder}}&{{state_key}}={{state_value}}", 1044 "https://endpoint.host.com/oauth2/authorize?{{client_id_key}}={{client_id_value}}&{{redirect_uri_key}}={{{{redirect_uri_value}} | urlEncoder}}&{{scope_key}}={{{{scope_value}} | urlEncoder}}&{{state_key}}={{state_value}}&subdomain={{subdomain}}", 1045 ], 1046 title="Consent URL", 1047 ) 1048 scope: Optional[str] = Field( 1049 None, 1050 description="The DeclarativeOAuth Specific string of the scopes needed to be grant for authenticated user.", 1051 examples=["user:read user:read_orders workspaces:read"], 1052 title="Scopes", 1053 ) 1054 access_token_url: str = Field( 1055 ..., 1056 description="The DeclarativeOAuth Specific URL templated string to obtain the `access_token`, `refresh_token` etc.\nThe placeholders are replaced during the processing to provide neccessary values.", 1057 examples=[ 1058 "https://auth.host.com/oauth2/token?{{client_id_key}}={{client_id_value}}&{{client_secret_key}}={{client_secret_value}}&{{auth_code_key}}={{auth_code_value}}&{{redirect_uri_key}}={{{{redirect_uri_value}} | urlEncoder}}" 1059 ], 1060 title="Access Token URL", 1061 ) 1062 access_token_headers: Optional[Dict[str, Any]] = Field( 1063 None, 1064 description="The DeclarativeOAuth Specific optional headers to inject while exchanging the `auth_code` to `access_token` during `completeOAuthFlow` step.", 1065 examples=[ 1066 { 1067 "Authorization": "Basic {{ {{ client_id_value }}:{{ client_secret_value }} | base64Encoder }}" 1068 } 1069 ], 1070 title="Access Token Headers", 1071 ) 1072 access_token_params: Optional[Dict[str, Any]] = Field( 1073 None, 1074 description="The DeclarativeOAuth Specific optional query parameters to inject while exchanging the `auth_code` to `access_token` during `completeOAuthFlow` step.\nWhen this property is provided, the query params will be encoded as `Json` and included in the outgoing API request.", 1075 examples=[ 1076 { 1077 "{{ auth_code_key }}": "{{ auth_code_value }}", 1078 "{{ client_id_key }}": "{{ client_id_value }}", 1079 "{{ client_secret_key }}": "{{ client_secret_value }}", 1080 } 1081 ], 1082 title="Access Token Query Params (Json Encoded)", 1083 ) 1084 extract_output: Optional[List[str]] = Field( 1085 None, 1086 description="The DeclarativeOAuth Specific list of strings to indicate which keys should be extracted and returned back to the input config.", 1087 examples=[["access_token", "refresh_token", "other_field"]], 1088 title="Extract Output", 1089 ) 1090 state: Optional[State] = Field( 1091 None, 1092 description="The DeclarativeOAuth Specific object to provide the criteria of how the `state` query param should be constructed,\nincluding length and complexity.", 1093 examples=[{"min": 7, "max": 128}], 1094 title="Configurable State Query Param", 1095 ) 1096 client_id_key: Optional[str] = Field( 1097 None, 1098 description="The DeclarativeOAuth Specific optional override to provide the custom `client_id` key name, if required by data-provider.", 1099 examples=["my_custom_client_id_key_name"], 1100 title="Client ID Key Override", 1101 ) 1102 client_secret_key: Optional[str] = Field( 1103 None, 1104 description="The DeclarativeOAuth Specific optional override to provide the custom `client_secret` key name, if required by data-provider.", 1105 examples=["my_custom_client_secret_key_name"], 1106 title="Client Secret Key Override", 1107 ) 1108 scope_key: Optional[str] = Field( 1109 None, 1110 description="The DeclarativeOAuth Specific optional override to provide the custom `scope` key name, if required by data-provider.", 1111 examples=["my_custom_scope_key_key_name"], 1112 title="Scopes Key Override", 1113 ) 1114 state_key: Optional[str] = Field( 1115 None, 1116 description="The DeclarativeOAuth Specific optional override to provide the custom `state` key name, if required by data-provider.", 1117 examples=["my_custom_state_key_key_name"], 1118 title="State Key Override", 1119 ) 1120 auth_code_key: Optional[str] = Field( 1121 None, 1122 description="The DeclarativeOAuth Specific optional override to provide the custom `code` key name to something like `auth_code` or `custom_auth_code`, if required by data-provider.", 1123 examples=["my_custom_auth_code_key_name"], 1124 title="Auth Code Key Override", 1125 ) 1126 redirect_uri_key: Optional[str] = Field( 1127 None, 1128 description="The DeclarativeOAuth Specific optional override to provide the custom `redirect_uri` key name to something like `callback_uri`, if required by data-provider.", 1129 examples=["my_custom_redirect_uri_key_name"], 1130 title="Redirect URI Key Override", 1131 )
state: Optional[State]
class
OauthConnectorInputSpecification.Config:
class
OAuthConfigSpecification(pydantic.v1.main.BaseModel):
1134class OAuthConfigSpecification(BaseModel): 1135 class Config: 1136 extra = Extra.allow 1137 1138 oauth_user_input_from_connector_config_specification: Optional[Dict[str, Any]] = Field( 1139 None, 1140 description="OAuth specific blob. This is a Json Schema used to validate Json configurations used as input to OAuth.\nMust be a valid non-nested JSON that refers to properties from ConnectorSpecification.connectionSpecification\nusing special annotation 'path_in_connector_config'.\nThese are input values the user is entering through the UI to authenticate to the connector, that might also shared\nas inputs for syncing data via the connector.\nExamples:\nif no connector values is shared during oauth flow, oauth_user_input_from_connector_config_specification=[]\nif connector values such as 'app_id' inside the top level are used to generate the API url for the oauth flow,\n oauth_user_input_from_connector_config_specification={\n app_id: {\n type: string\n path_in_connector_config: ['app_id']\n }\n }\nif connector values such as 'info.app_id' nested inside another object are used to generate the API url for the oauth flow,\n oauth_user_input_from_connector_config_specification={\n app_id: {\n type: string\n path_in_connector_config: ['info', 'app_id']\n }\n }", 1141 examples=[ 1142 {"app_id": {"type": "string", "path_in_connector_config": ["app_id"]}}, 1143 { 1144 "app_id": { 1145 "type": "string", 1146 "path_in_connector_config": ["info", "app_id"], 1147 } 1148 }, 1149 ], 1150 title="OAuth user input", 1151 ) 1152 oauth_connector_input_specification: Optional[OauthConnectorInputSpecification] = Field( 1153 None, 1154 description='The DeclarativeOAuth specific blob.\nPertains to the fields defined by the connector relating to the OAuth flow.\n\nInterpolation capabilities:\n- The variables placeholders are declared as `{{my_var}}`.\n- The nested resolution variables like `{{ {{my_nested_var}} }}` is allowed as well.\n\n- The allowed interpolation context is:\n + base64Encoder - encode to `base64`, {{ {{my_var_a}}:{{my_var_b}} | base64Encoder }}\n + base64Decorer - decode from `base64` encoded string, {{ {{my_string_variable_or_string_value}} | base64Decoder }}\n + urlEncoder - encode the input string to URL-like format, {{ https://test.host.com/endpoint | urlEncoder}}\n + urlDecorer - decode the input url-encoded string into text format, {{ urlDecoder:https%3A%2F%2Fairbyte.io | urlDecoder}}\n + codeChallengeS256 - get the `codeChallenge` encoded value to provide additional data-provider specific authorisation values, {{ {{state_value}} | codeChallengeS256 }}\n\nExamples:\n - The TikTok Marketing DeclarativeOAuth spec:\n {\n "oauth_connector_input_specification": {\n "type": "object",\n "additionalProperties": false,\n "properties": {\n "consent_url": "https://ads.tiktok.com/marketing_api/auth?{{client_id_key}}={{client_id_value}}&{{redirect_uri_key}}={{ {{redirect_uri_value}} | urlEncoder}}&{{state_key}}={{state_value}}",\n "access_token_url": "https://business-api.tiktok.com/open_api/v1.3/oauth2/access_token/",\n "access_token_params": {\n "{{ auth_code_key }}": "{{ auth_code_value }}",\n "{{ client_id_key }}": "{{ client_id_value }}",\n "{{ client_secret_key }}": "{{ client_secret_value }}"\n },\n "access_token_headers": {\n "Content-Type": "application/json",\n "Accept": "application/json"\n },\n "extract_output": ["data.access_token"],\n "client_id_key": "app_id",\n "client_secret_key": "secret",\n "auth_code_key": "auth_code"\n }\n }\n }', 1155 title="DeclarativeOAuth Connector Specification", 1156 ) 1157 complete_oauth_output_specification: Optional[Dict[str, Any]] = Field( 1158 None, 1159 description="OAuth specific blob. This is a Json Schema used to validate Json configurations produced by the OAuth flows as they are\nreturned by the distant OAuth APIs.\nMust be a valid JSON describing the fields to merge back to `ConnectorSpecification.connectionSpecification`.\nFor each field, a special annotation `path_in_connector_config` can be specified to determine where to merge it,\nExamples:\n complete_oauth_output_specification={\n refresh_token: {\n type: string,\n path_in_connector_config: ['credentials', 'refresh_token']\n }\n }", 1160 examples=[ 1161 { 1162 "refresh_token": { 1163 "type": "string,", 1164 "path_in_connector_config": ["credentials", "refresh_token"], 1165 } 1166 } 1167 ], 1168 title="OAuth output specification", 1169 ) 1170 complete_oauth_server_input_specification: Optional[Dict[str, Any]] = Field( 1171 None, 1172 description="OAuth specific blob. This is a Json Schema used to validate Json configurations persisted as Airbyte Server configurations.\nMust be a valid non-nested JSON describing additional fields configured by the Airbyte Instance or Workspace Admins to be used by the\nserver when completing an OAuth flow (typically exchanging an auth code for refresh token).\nExamples:\n complete_oauth_server_input_specification={\n client_id: {\n type: string\n },\n client_secret: {\n type: string\n }\n }", 1173 examples=[{"client_id": {"type": "string"}, "client_secret": {"type": "string"}}], 1174 title="OAuth input specification", 1175 ) 1176 complete_oauth_server_output_specification: Optional[Dict[str, Any]] = Field( 1177 None, 1178 description="OAuth specific blob. This is a Json Schema used to validate Json configurations persisted as Airbyte Server configurations that\nalso need to be merged back into the connector configuration at runtime.\nThis is a subset configuration of `complete_oauth_server_input_specification` that filters fields out to retain only the ones that\nare necessary for the connector to function with OAuth. (some fields could be used during oauth flows but not needed afterwards, therefore\nthey would be listed in the `complete_oauth_server_input_specification` but not `complete_oauth_server_output_specification`)\nMust be a valid non-nested JSON describing additional fields configured by the Airbyte Instance or Workspace Admins to be used by the\nconnector when using OAuth flow APIs.\nThese fields are to be merged back to `ConnectorSpecification.connectionSpecification`.\nFor each field, a special annotation `path_in_connector_config` can be specified to determine where to merge it,\nExamples:\n complete_oauth_server_output_specification={\n client_id: {\n type: string,\n path_in_connector_config: ['credentials', 'client_id']\n },\n client_secret: {\n type: string,\n path_in_connector_config: ['credentials', 'client_secret']\n }\n }", 1179 examples=[ 1180 { 1181 "client_id": { 1182 "type": "string,", 1183 "path_in_connector_config": ["credentials", "client_id"], 1184 }, 1185 "client_secret": { 1186 "type": "string,", 1187 "path_in_connector_config": ["credentials", "client_secret"], 1188 }, 1189 } 1190 ], 1191 title="OAuth server output specification", 1192 )
oauth_connector_input_specification: Optional[OauthConnectorInputSpecification]
class
OAuthConfigSpecification.Config:
class
OffsetIncrement(pydantic.v1.main.BaseModel):
1195class OffsetIncrement(BaseModel): 1196 type: Literal["OffsetIncrement"] 1197 page_size: Optional[Union[int, str]] = Field( 1198 None, 1199 description="The number of records to include in each pages.", 1200 examples=[100, "{{ config['page_size'] }}"], 1201 title="Limit", 1202 ) 1203 inject_on_first_request: Optional[bool] = Field( 1204 False, 1205 description="Using the `offset` with value `0` during the first request", 1206 title="Inject Offset on First Request", 1207 ) 1208 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters")
class
PageIncrement(pydantic.v1.main.BaseModel):
1211class PageIncrement(BaseModel): 1212 type: Literal["PageIncrement"] 1213 page_size: Optional[Union[int, str]] = Field( 1214 None, 1215 description="The number of records to include in each pages.", 1216 examples=[100, "100", "{{ config['page_size'] }}"], 1217 title="Page Size", 1218 ) 1219 start_from_page: Optional[int] = Field( 1220 0, 1221 description="Index of the first page to request.", 1222 examples=[0, 1], 1223 title="Start From Page", 1224 ) 1225 inject_on_first_request: Optional[bool] = Field( 1226 False, 1227 description="Using the `page number` with value defined by `start_from_page` during the first request", 1228 title="Inject Page Number on First Request", 1229 ) 1230 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters")
class
PrimaryKey(pydantic.v1.main.BaseModel):
1233class PrimaryKey(BaseModel): 1234 __root__: Union[str, List[str], List[List[str]]] = Field( 1235 ..., 1236 description="The stream field to be used to distinguish unique records. Can either be a single field, an array of fields representing a composite key, or an array of arrays representing a composite key where the fields are nested fields.", 1237 examples=["id", ["code", "type"]], 1238 title="Primary Key", 1239 )
class
PropertyLimitType(enum.Enum):
1242class PropertyLimitType(Enum): 1243 characters = "characters" 1244 property_count = "property_count"
An enumeration.
characters =
<PropertyLimitType.characters: 'characters'>
property_count =
<PropertyLimitType.property_count: 'property_count'>
class
PropertyChunking(pydantic.v1.main.BaseModel):
1247class PropertyChunking(BaseModel): 1248 type: Literal["PropertyChunking"] 1249 property_limit_type: PropertyLimitType = Field( 1250 ..., 1251 description="The type used to determine the maximum number of properties per chunk", 1252 title="Property Limit Type", 1253 ) 1254 property_limit: Optional[int] = Field( 1255 None, 1256 description="The maximum amount of properties that can be retrieved per request according to the limit type.", 1257 title="Property Limit", 1258 ) 1259 record_merge_strategy: Optional[GroupByKeyMergeStrategy] = Field( 1260 None, 1261 description="Dictates how to records that require multiple requests to get all properties should be emitted to the destination", 1262 title="Record Merge Strategy", 1263 ) 1264 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters")
property_limit_type: PropertyLimitType
record_merge_strategy: Optional[GroupByKeyMergeStrategy]
class
RecordFilter(pydantic.v1.main.BaseModel):
1267class RecordFilter(BaseModel): 1268 type: Literal["RecordFilter"] 1269 condition: Optional[str] = Field( 1270 "", 1271 description="The predicate to filter a record. Records will be removed if evaluated to False.", 1272 examples=[ 1273 "{{ record['created_at'] >= stream_interval['start_time'] }}", 1274 "{{ record.status in ['active', 'expired'] }}", 1275 ], 1276 title="Condition", 1277 ) 1278 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters")
class
SchemaNormalization(enum.Enum):
An enumeration.
Default =
<SchemaNormalization.Default: 'Default'>
None_ =
<SchemaNormalization.None_: 'None'>
class
RemoveFields(pydantic.v1.main.BaseModel):
1286class RemoveFields(BaseModel): 1287 type: Literal["RemoveFields"] 1288 condition: Optional[str] = Field( 1289 "", 1290 description="The predicate to filter a property by a property value. Property will be removed if it is empty OR expression is evaluated to True.,", 1291 examples=[ 1292 "{{ property|string == '' }}", 1293 "{{ property is integer }}", 1294 "{{ property|length > 5 }}", 1295 "{{ property == 'some_string_to_match' }}", 1296 ], 1297 ) 1298 field_pointers: List[List[str]] = Field( 1299 ..., 1300 description="Array of paths defining the field to remove. Each item is an array whose field describe the path of a field to remove.", 1301 examples=[["tags"], [["content", "html"], ["content", "plain_text"]]], 1302 title="Field Paths", 1303 )
class
RequestPath(pydantic.v1.main.BaseModel):
class
InjectInto(enum.Enum):
1310class InjectInto(Enum): 1311 request_parameter = "request_parameter" 1312 header = "header" 1313 body_data = "body_data" 1314 body_json = "body_json"
An enumeration.
request_parameter =
<InjectInto.request_parameter: 'request_parameter'>
header =
<InjectInto.header: 'header'>
body_data =
<InjectInto.body_data: 'body_data'>
body_json =
<InjectInto.body_json: 'body_json'>
class
RequestOption(pydantic.v1.main.BaseModel):
1317class RequestOption(BaseModel): 1318 type: Literal["RequestOption"] 1319 inject_into: InjectInto = Field( 1320 ..., 1321 description="Configures where the descriptor should be set on the HTTP requests. Note that request parameters that are already encoded in the URL path will not be duplicated.", 1322 examples=["request_parameter", "header", "body_data", "body_json"], 1323 title="Inject Into", 1324 ) 1325 field_name: Optional[str] = Field( 1326 None, 1327 description="Configures which key should be used in the location that the descriptor is being injected into. We hope to eventually deprecate this field in favor of `field_path` for all request_options, but must currently maintain it for backwards compatibility in the Builder.", 1328 examples=["segment_id"], 1329 title="Field Name", 1330 ) 1331 field_path: Optional[List[str]] = Field( 1332 None, 1333 description="Configures a path to be used for nested structures in JSON body requests (e.g. GraphQL queries)", 1334 examples=[["data", "viewer", "id"]], 1335 title="Field Path", 1336 )
inject_into: InjectInto
class
Schemas(pydantic.v1.main.BaseModel):
class
Schemas.Config:
class
LegacySessionTokenAuthenticator(pydantic.v1.main.BaseModel):
1346class LegacySessionTokenAuthenticator(BaseModel): 1347 type: Literal["LegacySessionTokenAuthenticator"] 1348 header: str = Field( 1349 ..., 1350 description="The name of the session token header that will be injected in the request", 1351 examples=["X-Session"], 1352 title="Session Request Header", 1353 ) 1354 login_url: str = Field( 1355 ..., 1356 description="Path of the login URL (do not include the base URL)", 1357 examples=["session"], 1358 title="Login Path", 1359 ) 1360 session_token: Optional[str] = Field( 1361 None, 1362 description="Session token to use if using a pre-defined token. Not needed if authenticating with username + password pair", 1363 example=["{{ config['session_token'] }}"], 1364 title="Session Token", 1365 ) 1366 session_token_response_key: str = Field( 1367 ..., 1368 description="Name of the key of the session token to be extracted from the response", 1369 examples=["id"], 1370 title="Response Token Response Key", 1371 ) 1372 username: Optional[str] = Field( 1373 None, 1374 description="Username used to authenticate and obtain a session token", 1375 examples=[" {{ config['username'] }}"], 1376 title="Username", 1377 ) 1378 password: Optional[str] = Field( 1379 "", 1380 description="Password used to authenticate and obtain a session token", 1381 examples=["{{ config['password'] }}", ""], 1382 title="Password", 1383 ) 1384 validate_session_url: str = Field( 1385 ..., 1386 description="Path of the URL to use to validate that the session token is valid (do not include the base URL)", 1387 examples=["user/current"], 1388 title="Validate Session Path", 1389 ) 1390 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters")
class
CsvDecoder(pydantic.v1.main.BaseModel):
class
AsyncJobStatusMap(pydantic.v1.main.BaseModel):
class
ValueType(enum.Enum):
1408class ValueType(Enum): 1409 string = "string" 1410 number = "number" 1411 integer = "integer" 1412 boolean = "boolean"
An enumeration.
string =
<ValueType.string: 'string'>
number =
<ValueType.number: 'number'>
integer =
<ValueType.integer: 'integer'>
boolean =
<ValueType.boolean: 'boolean'>
class
WaitTimeFromHeader(pydantic.v1.main.BaseModel):
1415class WaitTimeFromHeader(BaseModel): 1416 type: Literal["WaitTimeFromHeader"] 1417 header: str = Field( 1418 ..., 1419 description="The name of the response header defining how long to wait before retrying.", 1420 examples=["Retry-After"], 1421 title="Response Header Name", 1422 ) 1423 regex: Optional[str] = Field( 1424 None, 1425 description="Optional regex to apply on the header to extract its value. The regex should define a capture group defining the wait time.", 1426 examples=["([-+]?\\d+)"], 1427 title="Extraction Regex", 1428 ) 1429 max_waiting_time_in_seconds: Optional[float] = Field( 1430 None, 1431 description="Given the value extracted from the header is greater than this value, stop the stream.", 1432 examples=[3600], 1433 title="Max Waiting Time in Seconds", 1434 ) 1435 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters")
class
WaitUntilTimeFromHeader(pydantic.v1.main.BaseModel):
1438class WaitUntilTimeFromHeader(BaseModel): 1439 type: Literal["WaitUntilTimeFromHeader"] 1440 header: str = Field( 1441 ..., 1442 description="The name of the response header defining how long to wait before retrying.", 1443 examples=["wait_time"], 1444 title="Response Header", 1445 ) 1446 min_wait: Optional[Union[float, str]] = Field( 1447 None, 1448 description="Minimum time to wait before retrying.", 1449 examples=[10, "60"], 1450 title="Minimum Wait Time", 1451 ) 1452 regex: Optional[str] = Field( 1453 None, 1454 description="Optional regex to apply on the header to extract its value. The regex should define a capture group defining the wait time.", 1455 examples=["([-+]?\\d+)"], 1456 title="Extraction Regex", 1457 ) 1458 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters")
class
ComponentMappingDefinition(pydantic.v1.main.BaseModel):
1461class ComponentMappingDefinition(BaseModel): 1462 type: Literal["ComponentMappingDefinition"] 1463 field_path: List[str] = Field( 1464 ..., 1465 description="A list of potentially nested fields indicating the full path where value will be added or updated.", 1466 examples=[ 1467 ["data"], 1468 ["data", "records"], 1469 ["data", 1, "name"], 1470 ["data", "{{ components_values.name }}"], 1471 ["data", "*", "record"], 1472 ["*", "**", "name"], 1473 ], 1474 title="Field Path", 1475 ) 1476 value: str = Field( 1477 ..., 1478 description="The dynamic or static value to assign to the key. Interpolated values can be used to dynamically determine the value during runtime.", 1479 examples=[ 1480 "{{ components_values['updates'] }}", 1481 "{{ components_values['MetaData']['LastUpdatedTime'] }}", 1482 "{{ config['segment_id'] }}", 1483 "{{ stream_slice['parent_id'] }}", 1484 "{{ stream_slice['extra_fields']['name'] }}", 1485 ], 1486 title="Value", 1487 ) 1488 value_type: Optional[ValueType] = Field( 1489 None, 1490 description="The expected data type of the value. If omitted, the type will be inferred from the value provided.", 1491 title="Value Type", 1492 ) 1493 create_or_update: Optional[bool] = Field( 1494 False, 1495 description="Determines whether to create a new path if it doesn't exist (true) or only update existing paths (false). When set to true, the resolver will create new paths in the stream template if they don't exist. When false (default), it will only update existing paths.", 1496 title="Create or Update", 1497 ) 1498 condition: Optional[str] = Field( 1499 None, 1500 description="A condition that must be met for the mapping to be applied. This property is only supported for `ConfigComponentsResolver`.", 1501 examples=[ 1502 "{{ components_values.get('cursor_field', None) }}", 1503 "{{ '_incremental' in components_values.get('stream_name', '') }}", 1504 ], 1505 title="Condition", 1506 ) 1507 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters")
value_type: Optional[ValueType]
class
StreamConfig(pydantic.v1.main.BaseModel):
1510class StreamConfig(BaseModel): 1511 type: Literal["StreamConfig"] 1512 configs_pointer: List[str] = Field( 1513 ..., 1514 description="A list of potentially nested fields indicating the full path in source config file where streams configs located.", 1515 examples=[["data"], ["data", "streams"], ["data", "{{ parameters.name }}"]], 1516 title="Configs Pointer", 1517 ) 1518 default_values: Optional[List[Dict[str, Any]]] = Field( 1519 None, 1520 description="A list of default values, each matching the structure expected from the parsed component value.", 1521 title="Default Values", 1522 ) 1523 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters")
class
ConfigComponentsResolver(pydantic.v1.main.BaseModel):
1526class ConfigComponentsResolver(BaseModel): 1527 type: Literal["ConfigComponentsResolver"] 1528 stream_config: Union[List[StreamConfig], StreamConfig] 1529 components_mapping: List[ComponentMappingDefinition] 1530 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters")
stream_config: Union[List[StreamConfig], StreamConfig]
components_mapping: List[ComponentMappingDefinition]
class
StreamParametersDefinition(pydantic.v1.main.BaseModel):
1533class StreamParametersDefinition(BaseModel): 1534 type: Literal["StreamParametersDefinition"] 1535 list_of_parameters_for_stream: List[Dict[str, Any]] = Field( 1536 ..., 1537 description="A list of object of parameters for stream, each object in the list represents params for one stream.", 1538 examples=[ 1539 [ 1540 { 1541 "name": "test stream", 1542 "$parameters": {"entity": "test entity"}, 1543 "primary_key": "test key", 1544 } 1545 ] 1546 ], 1547 title="Stream Parameters", 1548 )
class
ParametrizedComponentsResolver(pydantic.v1.main.BaseModel):
1551class ParametrizedComponentsResolver(BaseModel): 1552 type: Literal["ParametrizedComponentsResolver"] 1553 stream_parameters: StreamParametersDefinition 1554 components_mapping: List[ComponentMappingDefinition] 1555 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters")
stream_parameters: StreamParametersDefinition
components_mapping: List[ComponentMappingDefinition]
class
RequestBodyPlainText(pydantic.v1.main.BaseModel):
class
RequestBodyUrlEncodedForm(pydantic.v1.main.BaseModel):
class
RequestBodyJsonObject(pydantic.v1.main.BaseModel):
class
RequestBodyGraphQlQuery(pydantic.v1.main.BaseModel):
class
RequestBodyGraphQlQuery.Config:
class
ValidateAdheresToSchema(pydantic.v1.main.BaseModel):
1580class ValidateAdheresToSchema(BaseModel): 1581 type: Literal["ValidateAdheresToSchema"] 1582 base_schema: Union[str, Dict[str, Any]] = Field( 1583 ..., 1584 description="The base JSON schema against which the user-provided schema will be validated.", 1585 examples=[ 1586 "{{ config['report_validation_schema'] }}", 1587 '\'{\n "$schema": "http://json-schema.org/draft-07/schema#",\n "title": "Person",\n "type": "object",\n "properties": {\n "name": {\n "type": "string",\n "description": "The person\'s name"\n },\n "age": {\n "type": "integer",\n "minimum": 0,\n "description": "The person\'s age"\n }\n },\n "required": ["name", "age"]\n}\'\n', 1588 { 1589 "$schema": "http://json-schema.org/draft-07/schema#", 1590 "title": "Person", 1591 "type": "object", 1592 "properties": { 1593 "name": {"type": "string", "description": "The person's name"}, 1594 "age": { 1595 "type": "integer", 1596 "minimum": 0, 1597 "description": "The person's age", 1598 }, 1599 }, 1600 "required": ["name", "age"], 1601 }, 1602 ], 1603 title="Base JSON Schema", 1604 )
class
CustomValidationStrategy(pydantic.v1.main.BaseModel):
1607class CustomValidationStrategy(BaseModel): 1608 class Config: 1609 extra = Extra.allow 1610 1611 type: Literal["CustomValidationStrategy"] 1612 class_name: str = Field( 1613 ..., 1614 description="Fully-qualified name of the class that will be implementing the custom validation strategy. Has to be a sub class of ValidationStrategy. The format is `source_<name>.<package>.<class_name>`.", 1615 examples=["source_declarative_manifest.components.MyCustomValidationStrategy"], 1616 title="Class Name", 1617 )
class
CustomValidationStrategy.Config:
class
ConfigRemapField(pydantic.v1.main.BaseModel):
1620class ConfigRemapField(BaseModel): 1621 type: Literal["ConfigRemapField"] 1622 map: Union[Dict[str, Any], str] = Field( 1623 ..., 1624 description="A mapping of original values to new values. When a field value matches a key in this map, it will be replaced with the corresponding value.", 1625 examples=[ 1626 {"pending": "in_progress", "done": "completed", "cancelled": "terminated"}, 1627 "{{ config['status_mapping'] }}", 1628 ], 1629 title="Value Mapping", 1630 ) 1631 field_path: List[str] = Field( 1632 ..., 1633 description="The path to the field whose value should be remapped. Specified as a list of path components to navigate through nested objects.", 1634 examples=[ 1635 ["status"], 1636 ["data", "status"], 1637 ["data", "{{ config.name }}", "status"], 1638 ["data", "*", "status"], 1639 ], 1640 title="Field Path", 1641 )
class
ConfigRemoveFields(pydantic.v1.main.BaseModel):
1644class ConfigRemoveFields(BaseModel): 1645 type: Literal["ConfigRemoveFields"] 1646 field_pointers: List[List[str]] = Field( 1647 ..., 1648 description="A list of field pointers to be removed from the config.", 1649 examples=[["tags"], [["content", "html"], ["content", "plain_text"]]], 1650 title="Field Pointers", 1651 ) 1652 condition: Optional[str] = Field( 1653 "", 1654 description="Fields will be removed if expression is evaluated to True.", 1655 examples=[ 1656 "{{ config['environemnt'] == 'sandbox' }}", 1657 "{{ property is integer }}", 1658 "{{ property|length > 5 }}", 1659 "{{ property == 'some_string_to_match' }}", 1660 ], 1661 )
class
CustomConfigTransformation(pydantic.v1.main.BaseModel):
1664class CustomConfigTransformation(BaseModel): 1665 type: Literal["CustomConfigTransformation"] 1666 class_name: str = Field( 1667 ..., 1668 description="Fully-qualified name of the class that will be implementing the custom config transformation. The format is `source_<name>.<package>.<class_name>`.", 1669 examples=["source_declarative_manifest.components.MyCustomConfigTransformation"], 1670 ) 1671 parameters: Optional[Dict[str, Any]] = Field( 1672 None, 1673 alias="$parameters", 1674 description="Additional parameters to be passed to the custom config transformation.", 1675 )
class
AddedFieldDefinition(pydantic.v1.main.BaseModel):
1678class AddedFieldDefinition(BaseModel): 1679 type: Literal["AddedFieldDefinition"] 1680 path: List[str] = Field( 1681 ..., 1682 description="List of strings defining the path where to add the value on the record.", 1683 examples=[["segment_id"], ["metadata", "segment_id"]], 1684 title="Path", 1685 ) 1686 value: str = Field( 1687 ..., 1688 description="Value of the new field. Use {{ record['existing_field'] }} syntax to refer to other fields in the record.", 1689 examples=[ 1690 "{{ record['updates'] }}", 1691 "{{ record['MetaData']['LastUpdatedTime'] }}", 1692 "{{ stream_partition['segment_id'] }}", 1693 ], 1694 title="Value", 1695 ) 1696 value_type: Optional[ValueType] = Field( 1697 None, 1698 description="Type of the value. If not specified, the type will be inferred from the value.", 1699 title="Value Type", 1700 ) 1701 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters")
value_type: Optional[ValueType]
class
AddFields(pydantic.v1.main.BaseModel):
1704class AddFields(BaseModel): 1705 type: Literal["AddFields"] 1706 fields: List[AddedFieldDefinition] = Field( 1707 ..., 1708 description="List of transformations (path and corresponding value) that will be added to the record.", 1709 title="Fields", 1710 ) 1711 condition: Optional[str] = Field( 1712 "", 1713 description="Fields will be added if expression is evaluated to True.", 1714 examples=[ 1715 "{{ property|string == '' }}", 1716 "{{ property is integer }}", 1717 "{{ property|length > 5 }}", 1718 "{{ property == 'some_string_to_match' }}", 1719 ], 1720 ) 1721 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters")
fields: List[AddedFieldDefinition]
class
ApiKeyAuthenticator(pydantic.v1.main.BaseModel):
1724class ApiKeyAuthenticator(BaseModel): 1725 type: Literal["ApiKeyAuthenticator"] 1726 api_token: Optional[str] = Field( 1727 None, 1728 description="The API key to inject in the request. Fill it in the user inputs.", 1729 examples=["{{ config['api_key'] }}", "Token token={{ config['api_key'] }}"], 1730 title="API Key", 1731 ) 1732 header: Optional[str] = Field( 1733 None, 1734 description="The name of the HTTP header that will be set to the API key. This setting is deprecated, use inject_into instead. Header and inject_into can not be defined at the same time.", 1735 examples=["Authorization", "Api-Token", "X-Auth-Token"], 1736 title="Header Name", 1737 ) 1738 inject_into: Optional[RequestOption] = Field( 1739 None, 1740 description="Configure how the API Key will be sent in requests to the source API. Either inject_into or header has to be defined.", 1741 examples=[ 1742 {"inject_into": "header", "field_name": "Authorization"}, 1743 {"inject_into": "request_parameter", "field_name": "authKey"}, 1744 ], 1745 title="Inject API Key Into Outgoing HTTP Request", 1746 ) 1747 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters")
inject_into: Optional[RequestOption]
class
AuthFlow(pydantic.v1.main.BaseModel):
1750class AuthFlow(BaseModel): 1751 auth_flow_type: Optional[AuthFlowType] = Field( 1752 None, description="The type of auth to use", title="Auth flow type" 1753 ) 1754 predicate_key: Optional[List[str]] = Field( 1755 None, 1756 description="JSON path to a field in the connectorSpecification that should exist for the advanced auth to be applicable.", 1757 examples=[["credentials", "auth_type"]], 1758 title="Predicate key", 1759 ) 1760 predicate_value: Optional[str] = Field( 1761 None, 1762 description="Value of the predicate_key fields for the advanced auth to be applicable.", 1763 examples=["Oauth"], 1764 title="Predicate value", 1765 ) 1766 oauth_config_specification: Optional[OAuthConfigSpecification] = None
auth_flow_type: Optional[AuthFlowType]
oauth_config_specification: Optional[OAuthConfigSpecification]
class
CheckStream(pydantic.v1.main.BaseModel):
1769class CheckStream(BaseModel): 1770 type: Literal["CheckStream"] 1771 stream_names: Optional[List[str]] = Field( 1772 None, 1773 description="Names of the streams to try reading from when running a check operation.", 1774 examples=[["users"], ["users", "contacts"]], 1775 title="Stream Names", 1776 ) 1777 dynamic_streams_check_configs: Optional[List[DynamicStreamCheckConfig]] = None
dynamic_streams_check_configs: Optional[List[DynamicStreamCheckConfig]]
class
IncrementingCountCursor(pydantic.v1.main.BaseModel):
1780class IncrementingCountCursor(BaseModel): 1781 type: Literal["IncrementingCountCursor"] 1782 cursor_field: str = Field( 1783 ..., 1784 description="The location of the value on a record that will be used as a bookmark during sync. To ensure no data loss, the API must return records in ascending order based on the cursor field. Nested fields are not supported, so the field must be at the top level of the record. You can use a combination of Add Field and Remove Field transformations to move the nested field to the top.", 1785 examples=["created_at", "{{ config['record_cursor'] }}"], 1786 title="Cursor Field", 1787 ) 1788 start_value: Optional[Union[str, int]] = Field( 1789 None, 1790 description="The value that determines the earliest record that should be synced.", 1791 examples=[0, "{{ config['start_value'] }}"], 1792 title="Start Value", 1793 ) 1794 start_value_option: Optional[RequestOption] = Field( 1795 None, 1796 description="Optionally configures how the start value will be sent in requests to the source API.", 1797 title="Inject Start Value Into Outgoing HTTP Request", 1798 ) 1799 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters")
start_value_option: Optional[RequestOption]
class
DatetimeBasedCursor(pydantic.v1.main.BaseModel):
1802class DatetimeBasedCursor(BaseModel): 1803 type: Literal["DatetimeBasedCursor"] 1804 clamping: Optional[Clamping] = Field( 1805 None, 1806 description="This option is used to adjust the upper and lower boundaries of each datetime window to beginning and end of the provided target period (day, week, month)", 1807 title="Date Range Clamping", 1808 ) 1809 cursor_field: str = Field( 1810 ..., 1811 description="The location of the value on a record that will be used as a bookmark during sync. To ensure no data loss, the API must return records in ascending order based on the cursor field. Nested fields are not supported, so the field must be at the top level of the record. You can use a combination of Add Field and Remove Field transformations to move the nested field to the top.", 1812 examples=["created_at", "{{ config['record_cursor'] }}"], 1813 title="Cursor Field", 1814 ) 1815 cursor_datetime_formats: Optional[List[str]] = Field( 1816 None, 1817 description="The possible formats for the cursor field, in order of preference. The first format that matches the cursor field value will be used to parse it. If not provided, the Outgoing Datetime Format will be used.\nUse placeholders starting with \"%\" to describe the format the API is using. The following placeholders are available:\n * **%s**: Epoch unix timestamp - `1686218963`\n * **%s_as_float**: Epoch unix timestamp in seconds as float with microsecond precision - `1686218963.123456`\n * **%ms**: Epoch unix timestamp - `1686218963123`\n * **%a**: Weekday (abbreviated) - `Sun`\n * **%A**: Weekday (full) - `Sunday`\n * **%w**: Weekday (decimal) - `0` (Sunday), `6` (Saturday)\n * **%d**: Day of the month (zero-padded) - `01`, `02`, ..., `31`\n * **%b**: Month (abbreviated) - `Jan`\n * **%B**: Month (full) - `January`\n * **%m**: Month (zero-padded) - `01`, `02`, ..., `12`\n * **%y**: Year (without century, zero-padded) - `00`, `01`, ..., `99`\n * **%Y**: Year (with century) - `0001`, `0002`, ..., `9999`\n * **%H**: Hour (24-hour, zero-padded) - `00`, `01`, ..., `23`\n * **%I**: Hour (12-hour, zero-padded) - `01`, `02`, ..., `12`\n * **%p**: AM/PM indicator\n * **%M**: Minute (zero-padded) - `00`, `01`, ..., `59`\n * **%S**: Second (zero-padded) - `00`, `01`, ..., `59`\n * **%f**: Microsecond (zero-padded to 6 digits) - `000000`, `000001`, ..., `999999`\n * **%_ms**: Millisecond (zero-padded to 3 digits) - `000`, `001`, ..., `999`\n * **%z**: UTC offset - `(empty)`, `+0000`, `-04:00`\n * **%Z**: Time zone name - `(empty)`, `UTC`, `GMT`\n * **%j**: Day of the year (zero-padded) - `001`, `002`, ..., `366`\n * **%U**: Week number of the year (Sunday as first day) - `00`, `01`, ..., `53`\n * **%W**: Week number of the year (Monday as first day) - `00`, `01`, ..., `53`\n * **%c**: Date and time representation - `Tue Aug 16 21:30:00 1988`\n * **%x**: Date representation - `08/16/1988`\n * **%X**: Time representation - `21:30:00`\n * **%%**: Literal '%' character\n\n Some placeholders depend on the locale of the underlying system - in most cases this locale is configured as en/US. For more information see the [Python documentation](https://docs.python.org/3/library/datetime.html#strftime-and-strptime-format-codes).\n", 1818 examples=[ 1819 "%Y-%m-%d", 1820 "%Y-%m-%d %H:%M:%S", 1821 "%Y-%m-%dT%H:%M:%S", 1822 "%Y-%m-%dT%H:%M:%SZ", 1823 "%Y-%m-%dT%H:%M:%S%z", 1824 "%Y-%m-%dT%H:%M:%S.%fZ", 1825 "%Y-%m-%dT%H:%M:%S.%f%z", 1826 "%Y-%m-%d %H:%M:%S.%f+00:00", 1827 "%s", 1828 "%ms", 1829 ], 1830 title="Cursor Datetime Formats", 1831 ) 1832 start_datetime: Union[MinMaxDatetime, str] = Field( 1833 ..., 1834 description="The datetime that determines the earliest record that should be synced.", 1835 examples=["2020-01-1T00:00:00Z", "{{ config['start_time'] }}"], 1836 title="Start Datetime", 1837 ) 1838 start_time_option: Optional[RequestOption] = Field( 1839 None, 1840 description="Optionally configures how the start datetime will be sent in requests to the source API.", 1841 title="Inject Start Time Into Outgoing HTTP Request", 1842 ) 1843 end_datetime: Optional[Union[MinMaxDatetime, str]] = Field( 1844 None, 1845 description="The datetime that determines the last record that should be synced. If not provided, `{{ now_utc() }}` will be used.", 1846 examples=["2021-01-1T00:00:00Z", "{{ now_utc() }}", "{{ day_delta(-1) }}"], 1847 title="End Datetime", 1848 ) 1849 end_time_option: Optional[RequestOption] = Field( 1850 None, 1851 description="Optionally configures how the end datetime will be sent in requests to the source API.", 1852 title="Inject End Time Into Outgoing HTTP Request", 1853 ) 1854 datetime_format: str = Field( 1855 ..., 1856 description="The datetime format used to format the datetime values that are sent in outgoing requests to the API. Use placeholders starting with \"%\" to describe the format the API is using. The following placeholders are available:\n * **%s**: Epoch unix timestamp - `1686218963`\n * **%s_as_float**: Epoch unix timestamp in seconds as float with microsecond precision - `1686218963.123456`\n * **%ms**: Epoch unix timestamp (milliseconds) - `1686218963123`\n * **%a**: Weekday (abbreviated) - `Sun`\n * **%A**: Weekday (full) - `Sunday`\n * **%w**: Weekday (decimal) - `0` (Sunday), `6` (Saturday)\n * **%d**: Day of the month (zero-padded) - `01`, `02`, ..., `31`\n * **%b**: Month (abbreviated) - `Jan`\n * **%B**: Month (full) - `January`\n * **%m**: Month (zero-padded) - `01`, `02`, ..., `12`\n * **%y**: Year (without century, zero-padded) - `00`, `01`, ..., `99`\n * **%Y**: Year (with century) - `0001`, `0002`, ..., `9999`\n * **%H**: Hour (24-hour, zero-padded) - `00`, `01`, ..., `23`\n * **%I**: Hour (12-hour, zero-padded) - `01`, `02`, ..., `12`\n * **%p**: AM/PM indicator\n * **%M**: Minute (zero-padded) - `00`, `01`, ..., `59`\n * **%S**: Second (zero-padded) - `00`, `01`, ..., `59`\n * **%f**: Microsecond (zero-padded to 6 digits) - `000000`\n * **%_ms**: Millisecond (zero-padded to 3 digits) - `000`\n * **%z**: UTC offset - `(empty)`, `+0000`, `-04:00`\n * **%Z**: Time zone name - `(empty)`, `UTC`, `GMT`\n * **%j**: Day of the year (zero-padded) - `001`, `002`, ..., `366`\n * **%U**: Week number of the year (starting Sunday) - `00`, ..., `53`\n * **%W**: Week number of the year (starting Monday) - `00`, ..., `53`\n * **%c**: Date and time - `Tue Aug 16 21:30:00 1988`\n * **%x**: Date standard format - `08/16/1988`\n * **%X**: Time standard format - `21:30:00`\n * **%%**: Literal '%' character\n\n Some placeholders depend on the locale of the underlying system - in most cases this locale is configured as en/US. For more information see the [Python documentation](https://docs.python.org/3/library/datetime.html#strftime-and-strptime-format-codes).\n", 1857 examples=["%Y-%m-%dT%H:%M:%S.%f%z", "%Y-%m-%d", "%s", "%ms", "%s_as_float"], 1858 title="Outgoing Datetime Format", 1859 ) 1860 cursor_granularity: Optional[str] = Field( 1861 None, 1862 description="Smallest increment the datetime_format has (ISO 8601 duration) that is used to ensure the start of a slice does not overlap with the end of the previous one, e.g. for %Y-%m-%d the granularity should\nbe P1D, for %Y-%m-%dT%H:%M:%SZ the granularity should be PT1S. Given this field is provided, `step` needs to be provided as well.\n * **PT0.000001S**: 1 microsecond\n * **PT0.001S**: 1 millisecond\n * **PT1S**: 1 second\n * **PT1M**: 1 minute\n * **PT1H**: 1 hour\n * **P1D**: 1 day\n", 1863 examples=["PT1S"], 1864 title="Cursor Granularity", 1865 ) 1866 is_data_feed: Optional[bool] = Field( 1867 None, 1868 description="A data feed API is an API that does not allow filtering and paginates the content from the most recent to the least recent. Given this, the CDK needs to know when to stop paginating and this field will generate a stop condition for pagination.", 1869 title="Data Feed API", 1870 ) 1871 is_client_side_incremental: Optional[bool] = Field( 1872 None, 1873 description="Set to True if the target API endpoint does not take cursor values to filter records and returns all records anyway. This will cause the connector to filter out records locally, and only emit new records from the last sync, hence incremental. This means that all records would be read from the API, but only new records will be emitted to the destination.", 1874 title="Client-side Incremental Filtering", 1875 ) 1876 is_compare_strictly: Optional[bool] = Field( 1877 False, 1878 description="Set to True if the target API does not accept queries where the start time equal the end time. This will cause those requests to be skipped.", 1879 title="Strict Start-End Time Comparison", 1880 ) 1881 global_substream_cursor: Optional[bool] = Field( 1882 False, 1883 description="Setting to True causes the connector to store the cursor as one value, instead of per-partition. This setting optimizes performance when the parent stream has thousands of partitions. Notably, the substream state is updated only at the end of the sync, which helps prevent data loss in case of a sync failure. See more info in the [docs](https://docs.airbyte.com/connector-development/config-based/understanding-the-yaml-file/incremental-syncs).", 1884 title="Global Substream Cursor", 1885 ) 1886 lookback_window: Optional[str] = Field( 1887 None, 1888 description="Time interval (ISO8601 duration) before the start_datetime to read data for, e.g. P1M for looking back one month.\n * **PT1H**: 1 hour\n * **P1D**: 1 day\n * **P1W**: 1 week\n * **P1M**: 1 month\n * **P1Y**: 1 year\n", 1889 examples=["P1D", "P{{ config['lookback_days'] }}D"], 1890 title="Lookback Window", 1891 ) 1892 partition_field_end: Optional[str] = Field( 1893 None, 1894 description="Name of the partition start time field.", 1895 examples=["ending_time"], 1896 title="Partition Field End", 1897 ) 1898 partition_field_start: Optional[str] = Field( 1899 None, 1900 description="Name of the partition end time field.", 1901 examples=["starting_time"], 1902 title="Partition Field Start", 1903 ) 1904 step: Optional[str] = Field( 1905 None, 1906 description="The size of the time window (ISO8601 duration). Given this field is provided, `cursor_granularity` needs to be provided as well.\n * **PT1H**: 1 hour\n * **P1D**: 1 day\n * **P1W**: 1 week\n * **P1M**: 1 month\n * **P1Y**: 1 year\n", 1907 examples=["P1W", "{{ config['step_increment'] }}"], 1908 title="Step", 1909 ) 1910 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters")
clamping: Optional[Clamping]
start_datetime: Union[MinMaxDatetime, str]
start_time_option: Optional[RequestOption]
end_datetime: Union[MinMaxDatetime, str, NoneType]
end_time_option: Optional[RequestOption]
class
FixedWindowCallRatePolicy(pydantic.v1.main.BaseModel):
1913class FixedWindowCallRatePolicy(BaseModel): 1914 class Config: 1915 extra = Extra.allow 1916 1917 type: Literal["FixedWindowCallRatePolicy"] 1918 period: str = Field( 1919 ..., description="The time interval for the rate limit window.", title="Period" 1920 ) 1921 call_limit: int = Field( 1922 ..., 1923 description="The maximum number of calls allowed within the period.", 1924 title="Call Limit", 1925 ) 1926 matchers: List[HttpRequestRegexMatcher] = Field( 1927 ..., 1928 description="List of matchers that define which requests this policy applies to.", 1929 title="Matchers", 1930 )
matchers: List[HttpRequestRegexMatcher]
class
FixedWindowCallRatePolicy.Config:
class
MovingWindowCallRatePolicy(pydantic.v1.main.BaseModel):
1933class MovingWindowCallRatePolicy(BaseModel): 1934 class Config: 1935 extra = Extra.allow 1936 1937 type: Literal["MovingWindowCallRatePolicy"] 1938 rates: List[Rate] = Field( 1939 ..., 1940 description="List of rates that define the call limits for different time intervals.", 1941 title="Rates", 1942 ) 1943 matchers: List[HttpRequestRegexMatcher] = Field( 1944 ..., 1945 description="List of matchers that define which requests this policy applies to.", 1946 title="Matchers", 1947 )
rates: List[Rate]
matchers: List[HttpRequestRegexMatcher]
class
MovingWindowCallRatePolicy.Config:
class
UnlimitedCallRatePolicy(pydantic.v1.main.BaseModel):
1950class UnlimitedCallRatePolicy(BaseModel): 1951 class Config: 1952 extra = Extra.allow 1953 1954 type: Literal["UnlimitedCallRatePolicy"] 1955 matchers: List[HttpRequestRegexMatcher] = Field( 1956 ..., 1957 description="List of matchers that define which requests this policy applies to.", 1958 title="Matchers", 1959 )
matchers: List[HttpRequestRegexMatcher]
class
UnlimitedCallRatePolicy.Config:
class
DefaultErrorHandler(pydantic.v1.main.BaseModel):
1962class DefaultErrorHandler(BaseModel): 1963 type: Literal["DefaultErrorHandler"] 1964 backoff_strategies: Optional[ 1965 List[ 1966 Union[ 1967 ConstantBackoffStrategy, 1968 ExponentialBackoffStrategy, 1969 WaitTimeFromHeader, 1970 WaitUntilTimeFromHeader, 1971 CustomBackoffStrategy, 1972 ] 1973 ] 1974 ] = Field( 1975 None, 1976 description="List of backoff strategies to use to determine how long to wait before retrying a retryable request.", 1977 title="Backoff Strategies", 1978 ) 1979 max_retries: Optional[int] = Field( 1980 5, 1981 description="The maximum number of time to retry a retryable request before giving up and failing.", 1982 examples=[5, 0, 10], 1983 title="Max Retry Count", 1984 ) 1985 response_filters: Optional[List[HttpResponseFilter]] = Field( 1986 None, 1987 description="List of response filters to iterate on when deciding how to handle an error. When using an array of multiple filters, the filters will be applied sequentially and the response will be selected if it matches any of the filter's predicate.", 1988 title="Response Filters", 1989 ) 1990 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters")
backoff_strategies: Optional[List[Union[ConstantBackoffStrategy, ExponentialBackoffStrategy, WaitTimeFromHeader, WaitUntilTimeFromHeader, CustomBackoffStrategy]]]
response_filters: Optional[List[HttpResponseFilter]]
class
DefaultPaginator(pydantic.v1.main.BaseModel):
1993class DefaultPaginator(BaseModel): 1994 type: Literal["DefaultPaginator"] 1995 pagination_strategy: Union[ 1996 PageIncrement, OffsetIncrement, CursorPagination, CustomPaginationStrategy 1997 ] = Field( 1998 ..., 1999 description="Strategy defining how records are paginated.", 2000 title="Pagination Strategy", 2001 ) 2002 page_size_option: Optional[RequestOption] = Field( 2003 None, title="Inject Page Size Into Outgoing HTTP Request" 2004 ) 2005 page_token_option: Optional[Union[RequestOption, RequestPath]] = Field( 2006 None, 2007 description="Inject the page token into the outgoing HTTP requests by inserting it into either the request URL path or a field on the request.", 2008 title="Inject Page Token Into Outgoing HTTP Request", 2009 ) 2010 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters")
pagination_strategy: Union[PageIncrement, OffsetIncrement, CursorPagination, CustomPaginationStrategy]
page_size_option: Optional[RequestOption]
page_token_option: Union[RequestOption, RequestPath, NoneType]
class
SessionTokenRequestApiKeyAuthenticator(pydantic.v1.main.BaseModel):
2013class SessionTokenRequestApiKeyAuthenticator(BaseModel): 2014 type: Literal["ApiKey"] 2015 inject_into: RequestOption = Field( 2016 ..., 2017 description="Configure how the API Key will be sent in requests to the source API.", 2018 examples=[ 2019 {"inject_into": "header", "field_name": "Authorization"}, 2020 {"inject_into": "request_parameter", "field_name": "authKey"}, 2021 ], 2022 title="Inject API Key Into Outgoing HTTP Request", 2023 )
inject_into: RequestOption
class
ListPartitionRouter(pydantic.v1.main.BaseModel):
2026class ListPartitionRouter(BaseModel): 2027 type: Literal["ListPartitionRouter"] 2028 cursor_field: str = Field( 2029 ..., 2030 description='While iterating over list values, the name of field used to reference a list value. The partition value can be accessed with string interpolation. e.g. "{{ stream_partition[\'my_key\'] }}" where "my_key" is the value of the cursor_field.', 2031 examples=["section", "{{ config['section_key'] }}"], 2032 title="Current Partition Value Identifier", 2033 ) 2034 values: Union[str, List[str]] = Field( 2035 ..., 2036 description="The list of attributes being iterated over and used as input for the requests made to the source API.", 2037 examples=[["section_a", "section_b", "section_c"], "{{ config['sections'] }}"], 2038 title="Partition Values", 2039 ) 2040 request_option: Optional[RequestOption] = Field( 2041 None, 2042 description="A request option describing where the list value should be injected into and under what field name if applicable.", 2043 title="Inject Partition Value Into Outgoing HTTP Request", 2044 ) 2045 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters")
request_option: Optional[RequestOption]
class
RecordSelector(pydantic.v1.main.BaseModel):
2048class RecordSelector(BaseModel): 2049 type: Literal["RecordSelector"] 2050 extractor: Union[DpathExtractor, CustomRecordExtractor] 2051 record_filter: Optional[Union[RecordFilter, CustomRecordFilter]] = Field( 2052 None, 2053 description="Responsible for filtering records to be emitted by the Source.", 2054 title="Record Filter", 2055 ) 2056 schema_normalization: Optional[Union[SchemaNormalization, CustomSchemaNormalization]] = Field( 2057 None, 2058 description="Responsible for normalization according to the schema.", 2059 title="Schema Normalization", 2060 ) 2061 transform_before_filtering: Optional[bool] = Field( 2062 None, 2063 description="If true, transformation will be applied before record filtering.", 2064 title="Transform Before Filtering", 2065 ) 2066 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters")
extractor: Union[DpathExtractor, CustomRecordExtractor]
record_filter: Union[RecordFilter, CustomRecordFilter, NoneType]
schema_normalization: Union[SchemaNormalization, CustomSchemaNormalization, NoneType]
class
GzipDecoder(pydantic.v1.main.BaseModel):
2069class GzipDecoder(BaseModel): 2070 type: Literal["GzipDecoder"] 2071 decoder: Union[CsvDecoder, GzipDecoder, JsonDecoder, JsonlDecoder]
decoder: Union[CsvDecoder, GzipDecoder, JsonDecoder, JsonlDecoder]
class
RequestBodyGraphQL(pydantic.v1.main.BaseModel):
2074class RequestBodyGraphQL(BaseModel): 2075 type: Literal["RequestBodyGraphQL"] 2076 value: RequestBodyGraphQlQuery
value: RequestBodyGraphQlQuery
class
DpathValidator(pydantic.v1.main.BaseModel):
2079class DpathValidator(BaseModel): 2080 type: Literal["DpathValidator"] 2081 field_path: List[str] = Field( 2082 ..., 2083 description='List of potentially nested fields describing the full path of the field to validate. Use "*" to validate all values from an array.', 2084 examples=[ 2085 ["data"], 2086 ["data", "records"], 2087 ["data", "{{ parameters.name }}"], 2088 ["data", "*", "record"], 2089 ], 2090 title="Field Path", 2091 ) 2092 validation_strategy: Union[ValidateAdheresToSchema, CustomValidationStrategy] = Field( 2093 ..., 2094 description="The condition that the specified config value will be evaluated against", 2095 title="Validation Strategy", 2096 )
validation_strategy: Union[ValidateAdheresToSchema, CustomValidationStrategy]
class
PredicateValidator(pydantic.v1.main.BaseModel):
2099class PredicateValidator(BaseModel): 2100 type: Literal["PredicateValidator"] 2101 value: Optional[Union[str, float, Dict[str, Any], List[Any], bool]] = Field( 2102 ..., 2103 description="The value to be validated. Can be a literal value or interpolated from configuration.", 2104 examples=[ 2105 "test-value", 2106 "{{ config['api_version'] }}", 2107 "{{ config['tenant_id'] }}", 2108 123, 2109 ], 2110 title="Value", 2111 ) 2112 validation_strategy: Union[ValidateAdheresToSchema, CustomValidationStrategy] = Field( 2113 ..., 2114 description="The validation strategy to apply to the value.", 2115 title="Validation Strategy", 2116 )
validation_strategy: Union[ValidateAdheresToSchema, CustomValidationStrategy]
class
ConfigAddFields(pydantic.v1.main.BaseModel):
2119class ConfigAddFields(BaseModel): 2120 type: Literal["ConfigAddFields"] 2121 fields: List[AddedFieldDefinition] = Field( 2122 ..., 2123 description="A list of transformations (path and corresponding value) that will be added to the config.", 2124 title="Fields", 2125 ) 2126 condition: Optional[str] = Field( 2127 "", 2128 description="Fields will be added if expression is evaluated to True.", 2129 examples=[ 2130 "{{ config['environemnt'] == 'sandbox' }}", 2131 "{{ property is integer }}", 2132 "{{ property|length > 5 }}", 2133 "{{ property == 'some_string_to_match' }}", 2134 ], 2135 )
fields: List[AddedFieldDefinition]
class
CompositeErrorHandler(pydantic.v1.main.BaseModel):
2138class CompositeErrorHandler(BaseModel): 2139 type: Literal["CompositeErrorHandler"] 2140 error_handlers: List[Union[CompositeErrorHandler, DefaultErrorHandler]] = Field( 2141 ..., 2142 description="List of error handlers to iterate on to determine how to handle a failed response.", 2143 title="Error Handlers", 2144 ) 2145 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters")
error_handlers: List[Union[CompositeErrorHandler, DefaultErrorHandler]]
class
HTTPAPIBudget(pydantic.v1.main.BaseModel):
2148class HTTPAPIBudget(BaseModel): 2149 class Config: 2150 extra = Extra.allow 2151 2152 type: Literal["HTTPAPIBudget"] 2153 policies: List[ 2154 Union[ 2155 FixedWindowCallRatePolicy, 2156 MovingWindowCallRatePolicy, 2157 UnlimitedCallRatePolicy, 2158 ] 2159 ] = Field( 2160 ..., 2161 description="List of call rate policies that define how many calls are allowed.", 2162 title="Policies", 2163 ) 2164 ratelimit_reset_header: Optional[str] = Field( 2165 "ratelimit-reset", 2166 description="The HTTP response header name that indicates when the rate limit resets.", 2167 title="Rate Limit Reset Header", 2168 ) 2169 ratelimit_remaining_header: Optional[str] = Field( 2170 "ratelimit-remaining", 2171 description="The HTTP response header name that indicates the number of remaining allowed calls.", 2172 title="Rate Limit Remaining Header", 2173 ) 2174 status_codes_for_ratelimit_hit: Optional[List[int]] = Field( 2175 [429], 2176 description="List of HTTP status codes that indicate a rate limit has been hit.", 2177 title="Status Codes for Rate Limit Hit", 2178 )
policies: List[Union[FixedWindowCallRatePolicy, MovingWindowCallRatePolicy, UnlimitedCallRatePolicy]]
class
HTTPAPIBudget.Config:
class
ZipfileDecoder(pydantic.v1.main.BaseModel):
2181class ZipfileDecoder(BaseModel): 2182 class Config: 2183 extra = Extra.allow 2184 2185 type: Literal["ZipfileDecoder"] 2186 decoder: Union[CsvDecoder, GzipDecoder, JsonDecoder, JsonlDecoder] = Field( 2187 ..., 2188 description="Parser to parse the decompressed data from the zipfile(s).", 2189 title="Parser", 2190 )
decoder: Union[CsvDecoder, GzipDecoder, JsonDecoder, JsonlDecoder]
class
ZipfileDecoder.Config:
class
ConfigMigration(pydantic.v1.main.BaseModel):
2193class ConfigMigration(BaseModel): 2194 type: Literal["ConfigMigration"] 2195 description: Optional[str] = Field( 2196 None, description="The description/purpose of the config migration." 2197 ) 2198 transformations: List[ 2199 Union[ 2200 ConfigRemapField, 2201 ConfigAddFields, 2202 ConfigRemoveFields, 2203 CustomConfigTransformation, 2204 ] 2205 ] = Field( 2206 ..., 2207 description="The list of transformations that will attempt to be applied on an incoming unmigrated config. The transformations will be applied in the order they are defined.", 2208 title="Transformations", 2209 )
transformations: List[Union[ConfigRemapField, ConfigAddFields, ConfigRemoveFields, CustomConfigTransformation]]
class
ConfigNormalizationRules(pydantic.v1.main.BaseModel):
2212class ConfigNormalizationRules(BaseModel): 2213 class Config: 2214 extra = Extra.forbid 2215 2216 type: Literal["ConfigNormalizationRules"] 2217 config_migrations: Optional[List[ConfigMigration]] = Field( 2218 [], 2219 description="The discrete migrations that will be applied on the incoming config. Each migration will be applied in the order they are defined.", 2220 title="Config Migrations", 2221 ) 2222 transformations: Optional[ 2223 List[ 2224 Union[ 2225 ConfigRemapField, 2226 ConfigAddFields, 2227 ConfigRemoveFields, 2228 CustomConfigTransformation, 2229 ] 2230 ] 2231 ] = Field( 2232 [], 2233 description="The list of transformations that will be applied on the incoming config at the start of each sync. The transformations will be applied in the order they are defined.", 2234 title="Transformations", 2235 ) 2236 validations: Optional[List[Union[DpathValidator, PredicateValidator]]] = Field( 2237 [], 2238 description="The list of validations that will be performed on the incoming config at the start of each sync.", 2239 title="Validations", 2240 )
config_migrations: Optional[List[ConfigMigration]]
transformations: Optional[List[Union[ConfigRemapField, ConfigAddFields, ConfigRemoveFields, CustomConfigTransformation]]]
validations: Optional[List[Union[DpathValidator, PredicateValidator]]]
class
ConfigNormalizationRules.Config:
class
Spec(pydantic.v1.main.BaseModel):
2243class Spec(BaseModel): 2244 type: Literal["Spec"] 2245 connection_specification: Dict[str, Any] = Field( 2246 ..., 2247 description="A connection specification describing how a the connector can be configured.", 2248 title="Connection Specification", 2249 ) 2250 documentation_url: Optional[str] = Field( 2251 None, 2252 description="URL of the connector's documentation page.", 2253 examples=["https://docs.airbyte.com/integrations/sources/dremio"], 2254 title="Documentation URL", 2255 ) 2256 advanced_auth: Optional[AuthFlow] = Field( 2257 None, 2258 description="Advanced specification for configuring the authentication flow.", 2259 title="Advanced Auth", 2260 ) 2261 config_normalization_rules: Optional[ConfigNormalizationRules] = Field( 2262 None, title="Config Normalization Rules" 2263 )
advanced_auth: Optional[AuthFlow]
config_normalization_rules: Optional[ConfigNormalizationRules]
class
DeclarativeSource1(pydantic.v1.main.BaseModel):
2266class DeclarativeSource1(BaseModel): 2267 class Config: 2268 extra = Extra.forbid 2269 2270 type: Literal["DeclarativeSource"] 2271 check: Union[CheckStream, CheckDynamicStream] 2272 streams: List[Union[ConditionalStreams, DeclarativeStream, StateDelegatingStream]] 2273 dynamic_streams: Optional[List[DynamicDeclarativeStream]] = None 2274 version: str = Field( 2275 ..., 2276 description="The version of the Airbyte CDK used to build and test the source.", 2277 ) 2278 schemas: Optional[Schemas] = None 2279 definitions: Optional[Dict[str, Any]] = None 2280 spec: Optional[Spec] = None 2281 concurrency_level: Optional[ConcurrencyLevel] = None 2282 api_budget: Optional[HTTPAPIBudget] = None 2283 max_concurrent_async_job_count: Optional[Union[int, str]] = Field( 2284 None, 2285 description="Maximum number of concurrent asynchronous jobs to run. This property is only relevant for sources/streams that support asynchronous job execution through the AsyncRetriever (e.g. a report-based stream that initiates a job, polls the job status, and then fetches the job results). This is often set by the API's maximum number of concurrent jobs on the account level. Refer to the API's documentation for this information.", 2286 examples=[3, "{{ config['max_concurrent_async_job_count'] }}"], 2287 title="Maximum Concurrent Asynchronous Jobs", 2288 ) 2289 metadata: Optional[Dict[str, Any]] = Field( 2290 None, 2291 description="For internal Airbyte use only - DO NOT modify manually. Used by consumers of declarative manifests for storing related metadata.", 2292 ) 2293 description: Optional[str] = Field( 2294 None, 2295 description="A description of the connector. It will be presented on the Source documentation page.", 2296 )
check: Union[CheckStream, CheckDynamicStream]
streams: List[Union[ConditionalStreams, DeclarativeStream, StateDelegatingStream]]
dynamic_streams: Optional[List[DynamicDeclarativeStream]]
schemas: Optional[Schemas]
spec: Optional[Spec]
concurrency_level: Optional[ConcurrencyLevel]
api_budget: Optional[HTTPAPIBudget]
class
DeclarativeSource1.Config:
class
DeclarativeSource2(pydantic.v1.main.BaseModel):
2299class DeclarativeSource2(BaseModel): 2300 class Config: 2301 extra = Extra.forbid 2302 2303 type: Literal["DeclarativeSource"] 2304 check: Union[CheckStream, CheckDynamicStream] 2305 streams: Optional[List[Union[ConditionalStreams, DeclarativeStream, StateDelegatingStream]]] = ( 2306 None 2307 ) 2308 dynamic_streams: List[DynamicDeclarativeStream] 2309 version: str = Field( 2310 ..., 2311 description="The version of the Airbyte CDK used to build and test the source.", 2312 ) 2313 schemas: Optional[Schemas] = None 2314 definitions: Optional[Dict[str, Any]] = None 2315 spec: Optional[Spec] = None 2316 concurrency_level: Optional[ConcurrencyLevel] = None 2317 api_budget: Optional[HTTPAPIBudget] = None 2318 max_concurrent_async_job_count: Optional[Union[int, str]] = Field( 2319 None, 2320 description="Maximum number of concurrent asynchronous jobs to run. This property is only relevant for sources/streams that support asynchronous job execution through the AsyncRetriever (e.g. a report-based stream that initiates a job, polls the job status, and then fetches the job results). This is often set by the API's maximum number of concurrent jobs on the account level. Refer to the API's documentation for this information.", 2321 examples=[3, "{{ config['max_concurrent_async_job_count'] }}"], 2322 title="Maximum Concurrent Asynchronous Jobs", 2323 ) 2324 metadata: Optional[Dict[str, Any]] = Field( 2325 None, 2326 description="For internal Airbyte use only - DO NOT modify manually. Used by consumers of declarative manifests for storing related metadata.", 2327 ) 2328 description: Optional[str] = Field( 2329 None, 2330 description="A description of the connector. It will be presented on the Source documentation page.", 2331 )
check: Union[CheckStream, CheckDynamicStream]
streams: Optional[List[Union[ConditionalStreams, DeclarativeStream, StateDelegatingStream]]]
dynamic_streams: List[DynamicDeclarativeStream]
schemas: Optional[Schemas]
spec: Optional[Spec]
concurrency_level: Optional[ConcurrencyLevel]
api_budget: Optional[HTTPAPIBudget]
class
DeclarativeSource2.Config:
class
DeclarativeSource(pydantic.v1.main.BaseModel):
2334class DeclarativeSource(BaseModel): 2335 class Config: 2336 extra = Extra.forbid 2337 2338 __root__: Union[DeclarativeSource1, DeclarativeSource2] = Field( 2339 ..., 2340 description="An API source that extracts data according to its declarative components.", 2341 title="DeclarativeSource", 2342 )
class
DeclarativeSource.Config:
class
SelectiveAuthenticator(pydantic.v1.main.BaseModel):
2345class SelectiveAuthenticator(BaseModel): 2346 class Config: 2347 extra = Extra.allow 2348 2349 type: Literal["SelectiveAuthenticator"] 2350 authenticator_selection_path: List[str] = Field( 2351 ..., 2352 description="Path of the field in config with selected authenticator name", 2353 examples=[["auth"], ["auth", "type"]], 2354 title="Authenticator Selection Path", 2355 ) 2356 authenticators: Dict[ 2357 str, 2358 Union[ 2359 ApiKeyAuthenticator, 2360 BasicHttpAuthenticator, 2361 BearerAuthenticator, 2362 OAuthAuthenticator, 2363 JwtAuthenticator, 2364 SessionTokenAuthenticator, 2365 LegacySessionTokenAuthenticator, 2366 CustomAuthenticator, 2367 NoAuth, 2368 ], 2369 ] = Field( 2370 ..., 2371 description="Authenticators to select from.", 2372 examples=[ 2373 { 2374 "authenticators": { 2375 "token": "#/definitions/ApiKeyAuthenticator", 2376 "oauth": "#/definitions/OAuthAuthenticator", 2377 "jwt": "#/definitions/JwtAuthenticator", 2378 } 2379 } 2380 ], 2381 title="Authenticators", 2382 ) 2383 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters")
authenticators: Dict[str, Union[ApiKeyAuthenticator, BasicHttpAuthenticator, BearerAuthenticator, OAuthAuthenticator, JwtAuthenticator, SessionTokenAuthenticator, LegacySessionTokenAuthenticator, CustomAuthenticator, NoAuth]]
class
SelectiveAuthenticator.Config:
class
ConditionalStreams(pydantic.v1.main.BaseModel):
2386class ConditionalStreams(BaseModel): 2387 type: Literal["ConditionalStreams"] 2388 condition: str = Field( 2389 ..., 2390 description="Condition that will be evaluated to determine if a set of streams should be available.", 2391 examples=["{{ config['is_sandbox'] }}"], 2392 title="Condition", 2393 ) 2394 streams: List[DeclarativeStream] = Field( 2395 ..., 2396 description="Streams that will be used during an operation based on the condition.", 2397 title="Streams", 2398 ) 2399 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters")
streams: List[DeclarativeStream]
class
FileUploader(pydantic.v1.main.BaseModel):
2402class FileUploader(BaseModel): 2403 type: Literal["FileUploader"] 2404 requester: Union[HttpRequester, CustomRequester] = Field( 2405 ..., 2406 description="Requester component that describes how to prepare HTTP requests to send to the source API.", 2407 ) 2408 download_target_extractor: Union[DpathExtractor, CustomRecordExtractor] = Field( 2409 ..., 2410 description="Responsible for fetching the url where the file is located. This is applied on each records and not on the HTTP response", 2411 ) 2412 file_extractor: Optional[Union[DpathExtractor, CustomRecordExtractor]] = Field( 2413 None, 2414 description="Responsible for fetching the content of the file. If not defined, the assumption is that the whole response body is the file content", 2415 ) 2416 filename_extractor: Optional[str] = Field( 2417 None, 2418 description="Defines the name to store the file. Stream name is automatically added to the file path. File unique ID can be used to avoid overwriting files. Random UUID will be used if the extractor is not provided.", 2419 examples=[ 2420 "{{ record.id }}/{{ record.file_name }}/", 2421 "{{ record.id }}_{{ record.file_name }}/", 2422 ], 2423 ) 2424 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters")
requester: Union[HttpRequester, CustomRequester]
download_target_extractor: Union[DpathExtractor, CustomRecordExtractor]
file_extractor: Union[DpathExtractor, CustomRecordExtractor, NoneType]
class
DeclarativeStream(pydantic.v1.main.BaseModel):
2427class DeclarativeStream(BaseModel): 2428 class Config: 2429 extra = Extra.allow 2430 2431 type: Literal["DeclarativeStream"] 2432 name: Optional[str] = Field("", description="The stream name.", example=["Users"], title="Name") 2433 retriever: Union[SimpleRetriever, AsyncRetriever, CustomRetriever] = Field( 2434 ..., 2435 description="Component used to coordinate how records are extracted across stream slices and request pages.", 2436 title="Retriever", 2437 ) 2438 incremental_sync: Optional[ 2439 Union[DatetimeBasedCursor, IncrementingCountCursor, CustomIncrementalSync] 2440 ] = Field( 2441 None, 2442 description="Component used to fetch data incrementally based on a time field in the data.", 2443 title="Incremental Sync", 2444 ) 2445 primary_key: Optional[PrimaryKey] = Field("", title="Primary Key") 2446 schema_loader: Optional[ 2447 Union[ 2448 InlineSchemaLoader, 2449 DynamicSchemaLoader, 2450 JsonFileSchemaLoader, 2451 List[ 2452 Union[ 2453 InlineSchemaLoader, 2454 DynamicSchemaLoader, 2455 JsonFileSchemaLoader, 2456 CustomSchemaLoader, 2457 ] 2458 ], 2459 CustomSchemaLoader, 2460 ] 2461 ] = Field( 2462 None, 2463 description="One or many schema loaders can be used to retrieve the schema for the current stream. When multiple schema loaders are defined, schema properties will be merged together. Schema loaders defined first taking precedence in the event of a conflict.", 2464 title="Schema Loader", 2465 ) 2466 transformations: Optional[ 2467 List[ 2468 Union[ 2469 AddFields, 2470 RemoveFields, 2471 KeysToLower, 2472 KeysToSnakeCase, 2473 FlattenFields, 2474 DpathFlattenFields, 2475 KeysReplace, 2476 CustomTransformation, 2477 ] 2478 ] 2479 ] = Field( 2480 None, 2481 description="A list of transformations to be applied to each output record.", 2482 title="Transformations", 2483 ) 2484 state_migrations: Optional[ 2485 List[Union[LegacyToPerPartitionStateMigration, CustomStateMigration]] 2486 ] = Field( 2487 [], 2488 description="Array of state migrations to be applied on the input state", 2489 title="State Migrations", 2490 ) 2491 file_uploader: Optional[FileUploader] = Field( 2492 None, 2493 description="(experimental) Describes how to fetch a file", 2494 title="File Uploader", 2495 ) 2496 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters")
retriever: Union[SimpleRetriever, AsyncRetriever, CustomRetriever]
incremental_sync: Union[DatetimeBasedCursor, IncrementingCountCursor, CustomIncrementalSync, NoneType]
primary_key: Optional[PrimaryKey]
schema_loader: Union[InlineSchemaLoader, DynamicSchemaLoader, JsonFileSchemaLoader, List[Union[InlineSchemaLoader, DynamicSchemaLoader, JsonFileSchemaLoader, CustomSchemaLoader]], CustomSchemaLoader, NoneType]
transformations: Optional[List[Union[AddFields, RemoveFields, KeysToLower, KeysToSnakeCase, FlattenFields, DpathFlattenFields, KeysReplace, CustomTransformation]]]
state_migrations: Optional[List[Union[LegacyToPerPartitionStateMigration, CustomStateMigration]]]
file_uploader: Optional[FileUploader]
class
DeclarativeStream.Config:
class
SessionTokenAuthenticator(pydantic.v1.main.BaseModel):
2499class SessionTokenAuthenticator(BaseModel): 2500 type: Literal["SessionTokenAuthenticator"] 2501 login_requester: HttpRequester = Field( 2502 ..., 2503 description="Description of the request to perform to obtain a session token to perform data requests. The response body is expected to be a JSON object with a session token property.", 2504 examples=[ 2505 { 2506 "type": "HttpRequester", 2507 "url_base": "https://my_api.com", 2508 "path": "/login", 2509 "authenticator": { 2510 "type": "BasicHttpAuthenticator", 2511 "username": "{{ config.username }}", 2512 "password": "{{ config.password }}", 2513 }, 2514 } 2515 ], 2516 title="Login Requester", 2517 ) 2518 session_token_path: List[str] = Field( 2519 ..., 2520 description="The path in the response body returned from the login requester to the session token.", 2521 examples=[["access_token"], ["result", "token"]], 2522 title="Session Token Path", 2523 ) 2524 expiration_duration: Optional[str] = Field( 2525 None, 2526 description="The duration in ISO 8601 duration notation after which the session token expires, starting from the time it was obtained. Omitting it will result in the session token being refreshed for every request.\n * **PT1H**: 1 hour\n * **P1D**: 1 day\n * **P1W**: 1 week\n * **P1M**: 1 month\n * **P1Y**: 1 year\n", 2527 examples=["PT1H", "P1D"], 2528 title="Expiration Duration", 2529 ) 2530 request_authentication: Union[ 2531 SessionTokenRequestApiKeyAuthenticator, SessionTokenRequestBearerAuthenticator 2532 ] = Field( 2533 ..., 2534 description="Authentication method to use for requests sent to the API, specifying how to inject the session token.", 2535 title="Data Request Authentication", 2536 ) 2537 decoder: Optional[Union[JsonDecoder, XmlDecoder]] = Field( 2538 None, description="Component used to decode the response.", title="Decoder" 2539 ) 2540 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters")
login_requester: HttpRequester
request_authentication: Union[SessionTokenRequestApiKeyAuthenticator, SessionTokenRequestBearerAuthenticator]
decoder: Union[JsonDecoder, XmlDecoder, NoneType]
class
HttpRequester(airbyte_cdk.sources.declarative.models.base_model_with_deprecations.BaseModelWithDeprecations):
2543class HttpRequester(BaseModelWithDeprecations): 2544 type: Literal["HttpRequester"] 2545 url_base: Optional[str] = Field( 2546 None, 2547 deprecated=True, 2548 deprecation_message="Use `url` field instead.", 2549 description="Deprecated, use the `url` instead. Base URL of the API source. Do not put sensitive information (e.g. API tokens) into this field - Use the Authenticator component for this.", 2550 examples=[ 2551 "https://connect.squareup.com/v2", 2552 "{{ config['base_url'] or 'https://app.posthog.com'}}/api", 2553 "https://connect.squareup.com/v2/quotes/{{ stream_partition['id'] }}/quote_line_groups", 2554 "https://example.com/api/v1/resource/{{ next_page_token['id'] }}", 2555 ], 2556 title="API Base URL", 2557 ) 2558 url: Optional[str] = Field( 2559 None, 2560 description="The URL of the source API endpoint. Do not put sensitive information (e.g. API tokens) into this field - Use the Authenticator component for this.", 2561 examples=[ 2562 "https://connect.squareup.com/v2", 2563 "{{ config['url'] or 'https://app.posthog.com'}}/api", 2564 "https://connect.squareup.com/v2/quotes/{{ stream_partition['id'] }}/quote_line_groups", 2565 "https://example.com/api/v1/resource/{{ next_page_token['id'] }}", 2566 ], 2567 title="API Endpoint URL", 2568 ) 2569 path: Optional[str] = Field( 2570 None, 2571 deprecated=True, 2572 deprecation_message="Use `url` field instead.", 2573 description="Deprecated, use the `url` instead. Path the specific API endpoint that this stream represents. Do not put sensitive information (e.g. API tokens) into this field - Use the Authenticator component for this.", 2574 examples=[ 2575 "/products", 2576 "/quotes/{{ stream_partition['id'] }}/quote_line_groups", 2577 "/trades/{{ config['symbol_id'] }}/history", 2578 ], 2579 title="URL Path", 2580 ) 2581 http_method: Optional[HttpMethod] = Field( 2582 HttpMethod.GET, 2583 description="The HTTP method used to fetch data from the source (can be GET or POST).", 2584 examples=["GET", "POST"], 2585 title="HTTP Method", 2586 ) 2587 authenticator: Optional[ 2588 Union[ 2589 ApiKeyAuthenticator, 2590 BasicHttpAuthenticator, 2591 BearerAuthenticator, 2592 OAuthAuthenticator, 2593 JwtAuthenticator, 2594 SessionTokenAuthenticator, 2595 SelectiveAuthenticator, 2596 CustomAuthenticator, 2597 NoAuth, 2598 LegacySessionTokenAuthenticator, 2599 ] 2600 ] = Field( 2601 None, 2602 description="Authentication method to use for requests sent to the API.", 2603 title="Authenticator", 2604 ) 2605 fetch_properties_from_endpoint: Optional[PropertiesFromEndpoint] = Field( 2606 None, 2607 deprecated=True, 2608 deprecation_message="Use `query_properties` field instead.", 2609 description="Allows for retrieving a dynamic set of properties from an API endpoint which can be injected into outbound request using the stream_partition.extra_fields.", 2610 title="Fetch Properties from Endpoint", 2611 ) 2612 query_properties: Optional[QueryProperties] = Field( 2613 None, 2614 description="For APIs that require explicit specification of the properties to query for, this component will take a static or dynamic set of properties (which can be optionally split into chunks) and allow them to be injected into an outbound request by accessing stream_partition.extra_fields.", 2615 title="Query Properties", 2616 ) 2617 request_parameters: Optional[Union[Dict[str, Union[str, QueryProperties]], str]] = Field( 2618 None, 2619 description="Specifies the query parameters that should be set on an outgoing HTTP request given the inputs.", 2620 examples=[ 2621 {"unit": "day"}, 2622 { 2623 "query": 'last_event_time BETWEEN TIMESTAMP "{{ stream_interval.start_time }}" AND TIMESTAMP "{{ stream_interval.end_time }}"' 2624 }, 2625 {"searchIn": "{{ ','.join(config.get('search_in', [])) }}"}, 2626 {"sort_by[asc]": "updated_at"}, 2627 ], 2628 title="Query Parameters", 2629 ) 2630 request_headers: Optional[Union[Dict[str, str], str]] = Field( 2631 None, 2632 description="Return any non-auth headers. Authentication headers will overwrite any overlapping headers returned from this method.", 2633 examples=[{"Output-Format": "JSON"}, {"Version": "{{ config['version'] }}"}], 2634 title="Request Headers", 2635 ) 2636 request_body_data: Optional[Union[Dict[str, str], str]] = Field( 2637 None, 2638 deprecated=True, 2639 deprecation_message="Use `request_body` field instead.", 2640 description="Specifies how to populate the body of the request with a non-JSON payload. Plain text will be sent as is, whereas objects will be converted to a urlencoded form.", 2641 examples=[ 2642 '[{"clause": {"type": "timestamp", "operator": 10, "parameters":\n [{"value": {{ stream_interval[\'start_time\'] | int * 1000 }} }]\n }, "orderBy": 1, "columnName": "Timestamp"}]/\n' 2643 ], 2644 title="Request Body Payload (Non-JSON)", 2645 ) 2646 request_body_json: Optional[Union[Dict[str, Any], str]] = Field( 2647 None, 2648 deprecated=True, 2649 deprecation_message="Use `request_body` field instead.", 2650 description="Specifies how to populate the body of the request with a JSON payload. Can contain nested objects.", 2651 examples=[ 2652 {"sort_order": "ASC", "sort_field": "CREATED_AT"}, 2653 {"key": "{{ config['value'] }}"}, 2654 {"sort": {"field": "updated_at", "order": "ascending"}}, 2655 ], 2656 title="Request Body JSON Payload", 2657 ) 2658 request_body: Optional[ 2659 Union[ 2660 RequestBodyPlainText, 2661 RequestBodyUrlEncodedForm, 2662 RequestBodyJsonObject, 2663 RequestBodyGraphQL, 2664 ] 2665 ] = Field( 2666 None, 2667 description="Specifies how to populate the body of the request with a payload. Can contain nested objects.", 2668 title="Request Body", 2669 ) 2670 error_handler: Optional[ 2671 Union[DefaultErrorHandler, CompositeErrorHandler, CustomErrorHandler] 2672 ] = Field( 2673 None, 2674 description="Error handler component that defines how to handle errors.", 2675 title="Error Handler", 2676 ) 2677 use_cache: Optional[bool] = Field( 2678 False, 2679 description="Enables stream requests caching. This field is automatically set by the CDK.", 2680 title="Use Cache", 2681 ) 2682 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters")
Pydantic BaseModel that warns when deprecated fields are accessed. The deprecation message is stored in the field's extra attributes. This class is used to create models that can have deprecated fields and show warnings when those fields are accessed or initialized.
The _deprecation_logs attribute is stored in the model itself.
The collected deprecation warnings are further propagated to the Airbyte log messages,
during the component creation process, in model_to_component._collect_model_deprecations().
The component implementation is not responsible for handling the deprecation warnings, since the deprecation warnings are already handled in the model itself.
http_method: Optional[HttpMethod]
authenticator: Union[ApiKeyAuthenticator, BasicHttpAuthenticator, BearerAuthenticator, OAuthAuthenticator, JwtAuthenticator, SessionTokenAuthenticator, SelectiveAuthenticator, CustomAuthenticator, NoAuth, LegacySessionTokenAuthenticator, NoneType]
fetch_properties_from_endpoint: Optional[PropertiesFromEndpoint]
query_properties: Optional[QueryProperties]
request_parameters: Union[Dict[str, Union[str, QueryProperties]], str, NoneType]
request_body: Union[RequestBodyPlainText, RequestBodyUrlEncodedForm, RequestBodyJsonObject, RequestBodyGraphQL, NoneType]
error_handler: Union[DefaultErrorHandler, CompositeErrorHandler, CustomErrorHandler, NoneType]
class
DynamicSchemaLoader(pydantic.v1.main.BaseModel):
2685class DynamicSchemaLoader(BaseModel): 2686 type: Literal["DynamicSchemaLoader"] 2687 retriever: Union[SimpleRetriever, AsyncRetriever, CustomRetriever] = Field( 2688 ..., 2689 description="Component used to coordinate how records are extracted across stream slices and request pages.", 2690 title="Retriever", 2691 ) 2692 schema_filter: Optional[Union[RecordFilter, CustomRecordFilter]] = Field( 2693 None, 2694 description="Responsible for filtering fields to be added to json schema.", 2695 title="Schema Filter", 2696 ) 2697 schema_transformations: Optional[ 2698 List[ 2699 Union[ 2700 AddFields, 2701 RemoveFields, 2702 KeysToLower, 2703 KeysToSnakeCase, 2704 FlattenFields, 2705 DpathFlattenFields, 2706 KeysReplace, 2707 CustomTransformation, 2708 ] 2709 ] 2710 ] = Field( 2711 None, 2712 description="A list of transformations to be applied to the schema.", 2713 title="Schema Transformations", 2714 ) 2715 schema_type_identifier: SchemaTypeIdentifier 2716 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters")
retriever: Union[SimpleRetriever, AsyncRetriever, CustomRetriever]
schema_filter: Union[RecordFilter, CustomRecordFilter, NoneType]
schema_transformations: Optional[List[Union[AddFields, RemoveFields, KeysToLower, KeysToSnakeCase, FlattenFields, DpathFlattenFields, KeysReplace, CustomTransformation]]]
schema_type_identifier: SchemaTypeIdentifier
class
ParentStreamConfig(pydantic.v1.main.BaseModel):
2719class ParentStreamConfig(BaseModel): 2720 type: Literal["ParentStreamConfig"] 2721 stream: Union[DeclarativeStream, StateDelegatingStream] = Field( 2722 ..., description="Reference to the parent stream.", title="Parent Stream" 2723 ) 2724 parent_key: str = Field( 2725 ..., 2726 description="The primary key of records from the parent stream that will be used during the retrieval of records for the current substream. This parent identifier field is typically a characteristic of the child records being extracted from the source API.", 2727 examples=["id", "{{ config['parent_record_id'] }}"], 2728 title="Parent Key", 2729 ) 2730 partition_field: str = Field( 2731 ..., 2732 description="While iterating over parent records during a sync, the parent_key value can be referenced by using this field.", 2733 examples=["parent_id", "{{ config['parent_partition_field'] }}"], 2734 title="Current Parent Key Value Identifier", 2735 ) 2736 request_option: Optional[RequestOption] = Field( 2737 None, 2738 description="A request option describing where the parent key value should be injected into and under what field name if applicable.", 2739 title="Request Option", 2740 ) 2741 incremental_dependency: Optional[bool] = Field( 2742 False, 2743 description="Indicates whether the parent stream should be read incrementally based on updates in the child stream.", 2744 title="Incremental Dependency", 2745 ) 2746 lazy_read_pointer: Optional[List[str]] = Field( 2747 [], 2748 description="If set, this will enable lazy reading, using the initial read of parent records to extract child records.", 2749 title="Lazy Read Pointer", 2750 ) 2751 extra_fields: Optional[List[List[str]]] = Field( 2752 None, 2753 description="Array of field paths to include as additional fields in the stream slice. Each path is an array of strings representing keys to access fields in the respective parent record. Accessible via `stream_slice.extra_fields`. Missing fields are set to `None`.", 2754 title="Extra Fields", 2755 ) 2756 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters")
stream: Union[DeclarativeStream, StateDelegatingStream]
request_option: Optional[RequestOption]
class
PropertiesFromEndpoint(pydantic.v1.main.BaseModel):
2759class PropertiesFromEndpoint(BaseModel): 2760 type: Literal["PropertiesFromEndpoint"] 2761 property_field_path: List[str] = Field( 2762 ..., 2763 description="Describes the path to the field that should be extracted", 2764 examples=[["name"]], 2765 ) 2766 retriever: Union[SimpleRetriever, CustomRetriever] = Field( 2767 ..., 2768 description="Requester component that describes how to fetch the properties to query from a remote API endpoint.", 2769 ) 2770 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters")
retriever: Union[SimpleRetriever, CustomRetriever]
class
QueryProperties(pydantic.v1.main.BaseModel):
2773class QueryProperties(BaseModel): 2774 type: Literal["QueryProperties"] 2775 property_list: Union[List[str], PropertiesFromEndpoint] = Field( 2776 ..., 2777 description="The set of properties that will be queried for in the outbound request. This can either be statically defined or dynamic based on an API endpoint", 2778 title="Property List", 2779 ) 2780 always_include_properties: Optional[List[str]] = Field( 2781 None, 2782 description="The list of properties that should be included in every set of properties when multiple chunks of properties are being requested.", 2783 title="Always Include Properties", 2784 ) 2785 property_chunking: Optional[PropertyChunking] = Field( 2786 None, 2787 description="Defines how query properties will be grouped into smaller sets for APIs with limitations on the number of properties fetched per API request.", 2788 title="Property Chunking", 2789 ) 2790 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters")
property_list: Union[List[str], PropertiesFromEndpoint]
property_chunking: Optional[PropertyChunking]
class
StateDelegatingStream(pydantic.v1.main.BaseModel):
2793class StateDelegatingStream(BaseModel): 2794 type: Literal["StateDelegatingStream"] 2795 name: str = Field(..., description="The stream name.", example=["Users"], title="Name") 2796 full_refresh_stream: DeclarativeStream = Field( 2797 ..., 2798 description="Component used to coordinate how records are extracted across stream slices and request pages when the state is empty or not provided.", 2799 title="Full Refresh Stream", 2800 ) 2801 incremental_stream: DeclarativeStream = Field( 2802 ..., 2803 description="Component used to coordinate how records are extracted across stream slices and request pages when the state provided.", 2804 title="Incremental Stream", 2805 ) 2806 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters")
full_refresh_stream: DeclarativeStream
incremental_stream: DeclarativeStream
class
SimpleRetriever(pydantic.v1.main.BaseModel):
2809class SimpleRetriever(BaseModel): 2810 type: Literal["SimpleRetriever"] 2811 requester: Union[HttpRequester, CustomRequester] = Field( 2812 ..., 2813 description="Requester component that describes how to prepare HTTP requests to send to the source API.", 2814 ) 2815 decoder: Optional[ 2816 Union[ 2817 JsonDecoder, 2818 XmlDecoder, 2819 CsvDecoder, 2820 JsonlDecoder, 2821 GzipDecoder, 2822 IterableDecoder, 2823 ZipfileDecoder, 2824 CustomDecoder, 2825 ] 2826 ] = Field( 2827 None, 2828 description="Component decoding the response so records can be extracted.", 2829 title="HTTP Response Format", 2830 ) 2831 record_selector: RecordSelector = Field( 2832 ..., 2833 description="Component that describes how to extract records from a HTTP response.", 2834 ) 2835 paginator: Optional[Union[DefaultPaginator, NoPagination]] = Field( 2836 None, 2837 description="Paginator component that describes how to navigate through the API's pages.", 2838 ) 2839 ignore_stream_slicer_parameters_on_paginated_requests: Optional[bool] = Field( 2840 False, 2841 description="If true, the partition router and incremental request options will be ignored when paginating requests. Request options set directly on the requester will not be ignored.", 2842 ) 2843 partition_router: Optional[ 2844 Union[ 2845 SubstreamPartitionRouter, 2846 ListPartitionRouter, 2847 GroupingPartitionRouter, 2848 CustomPartitionRouter, 2849 List[ 2850 Union[ 2851 SubstreamPartitionRouter, 2852 ListPartitionRouter, 2853 GroupingPartitionRouter, 2854 CustomPartitionRouter, 2855 ] 2856 ], 2857 ] 2858 ] = Field( 2859 None, 2860 description="Used to iteratively execute requests over a set of values, such as a parent stream's records or a list of constant values.", 2861 title="Partition Router", 2862 ) 2863 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters")
requester: Union[HttpRequester, CustomRequester]
decoder: Union[JsonDecoder, XmlDecoder, CsvDecoder, JsonlDecoder, GzipDecoder, IterableDecoder, ZipfileDecoder, CustomDecoder, NoneType]
record_selector: RecordSelector
paginator: Union[DefaultPaginator, NoPagination, NoneType]
partition_router: Union[SubstreamPartitionRouter, ListPartitionRouter, GroupingPartitionRouter, CustomPartitionRouter, List[Union[SubstreamPartitionRouter, ListPartitionRouter, GroupingPartitionRouter, CustomPartitionRouter]], NoneType]
class
AsyncRetriever(pydantic.v1.main.BaseModel):
2866class AsyncRetriever(BaseModel): 2867 type: Literal["AsyncRetriever"] 2868 record_selector: RecordSelector = Field( 2869 ..., 2870 description="Component that describes how to extract records from a HTTP response.", 2871 ) 2872 status_mapping: AsyncJobStatusMap = Field( 2873 ..., description="Async Job Status to Airbyte CDK Async Job Status mapping." 2874 ) 2875 status_extractor: Union[DpathExtractor, CustomRecordExtractor] = Field( 2876 ..., description="Responsible for fetching the actual status of the async job." 2877 ) 2878 download_target_extractor: Union[DpathExtractor, CustomRecordExtractor] = Field( 2879 ..., 2880 description="Responsible for fetching the final result `urls` provided by the completed / finished / ready async job.", 2881 ) 2882 download_extractor: Optional[ 2883 Union[DpathExtractor, CustomRecordExtractor, ResponseToFileExtractor] 2884 ] = Field(None, description="Responsible for fetching the records from provided urls.") 2885 creation_requester: Union[HttpRequester, CustomRequester] = Field( 2886 ..., 2887 description="Requester component that describes how to prepare HTTP requests to send to the source API to create the async server-side job.", 2888 ) 2889 polling_requester: Union[HttpRequester, CustomRequester] = Field( 2890 ..., 2891 description="Requester component that describes how to prepare HTTP requests to send to the source API to fetch the status of the running async job.", 2892 ) 2893 polling_job_timeout: Optional[Union[int, str]] = Field( 2894 None, 2895 description="The time in minutes after which the single Async Job should be considered as Timed Out.", 2896 ) 2897 download_target_requester: Optional[Union[HttpRequester, CustomRequester]] = Field( 2898 None, 2899 description="Requester component that describes how to prepare HTTP requests to send to the source API to extract the url from polling response by the completed async job.", 2900 ) 2901 download_requester: Union[HttpRequester, CustomRequester] = Field( 2902 ..., 2903 description="Requester component that describes how to prepare HTTP requests to send to the source API to download the data provided by the completed async job.", 2904 ) 2905 download_paginator: Optional[Union[DefaultPaginator, NoPagination]] = Field( 2906 None, 2907 description="Paginator component that describes how to navigate through the API's pages during download.", 2908 ) 2909 abort_requester: Optional[Union[HttpRequester, CustomRequester]] = Field( 2910 None, 2911 description="Requester component that describes how to prepare HTTP requests to send to the source API to abort a job once it is timed out from the source's perspective.", 2912 ) 2913 delete_requester: Optional[Union[HttpRequester, CustomRequester]] = Field( 2914 None, 2915 description="Requester component that describes how to prepare HTTP requests to send to the source API to delete a job once the records are extracted.", 2916 ) 2917 partition_router: Optional[ 2918 Union[ 2919 ListPartitionRouter, 2920 SubstreamPartitionRouter, 2921 GroupingPartitionRouter, 2922 CustomPartitionRouter, 2923 List[ 2924 Union[ 2925 ListPartitionRouter, 2926 SubstreamPartitionRouter, 2927 GroupingPartitionRouter, 2928 CustomPartitionRouter, 2929 ] 2930 ], 2931 ] 2932 ] = Field( 2933 [], 2934 description="PartitionRouter component that describes how to partition the stream, enabling incremental syncs and checkpointing.", 2935 title="Partition Router", 2936 ) 2937 decoder: Optional[ 2938 Union[ 2939 CsvDecoder, 2940 GzipDecoder, 2941 JsonDecoder, 2942 JsonlDecoder, 2943 IterableDecoder, 2944 XmlDecoder, 2945 ZipfileDecoder, 2946 CustomDecoder, 2947 ] 2948 ] = Field( 2949 None, 2950 description="Component decoding the response so records can be extracted.", 2951 title="HTTP Response Format", 2952 ) 2953 download_decoder: Optional[ 2954 Union[ 2955 CsvDecoder, 2956 GzipDecoder, 2957 JsonDecoder, 2958 JsonlDecoder, 2959 IterableDecoder, 2960 XmlDecoder, 2961 ZipfileDecoder, 2962 CustomDecoder, 2963 ] 2964 ] = Field( 2965 None, 2966 description="Component decoding the download response so records can be extracted.", 2967 title="Download HTTP Response Format", 2968 ) 2969 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters")
record_selector: RecordSelector
status_mapping: AsyncJobStatusMap
status_extractor: Union[DpathExtractor, CustomRecordExtractor]
download_target_extractor: Union[DpathExtractor, CustomRecordExtractor]
download_extractor: Union[DpathExtractor, CustomRecordExtractor, ResponseToFileExtractor, NoneType]
creation_requester: Union[HttpRequester, CustomRequester]
polling_requester: Union[HttpRequester, CustomRequester]
download_target_requester: Union[HttpRequester, CustomRequester, NoneType]
download_requester: Union[HttpRequester, CustomRequester]
download_paginator: Union[DefaultPaginator, NoPagination, NoneType]
abort_requester: Union[HttpRequester, CustomRequester, NoneType]
delete_requester: Union[HttpRequester, CustomRequester, NoneType]
partition_router: Union[SubstreamPartitionRouter, ListPartitionRouter, GroupingPartitionRouter, CustomPartitionRouter, List[Union[SubstreamPartitionRouter, ListPartitionRouter, GroupingPartitionRouter, CustomPartitionRouter]], NoneType]
decoder: Union[JsonDecoder, XmlDecoder, CsvDecoder, JsonlDecoder, GzipDecoder, IterableDecoder, ZipfileDecoder, CustomDecoder, NoneType]
download_decoder: Union[JsonDecoder, XmlDecoder, CsvDecoder, JsonlDecoder, GzipDecoder, IterableDecoder, ZipfileDecoder, CustomDecoder, NoneType]
class
SubstreamPartitionRouter(pydantic.v1.main.BaseModel):
2972class SubstreamPartitionRouter(BaseModel): 2973 type: Literal["SubstreamPartitionRouter"] 2974 parent_stream_configs: List[ParentStreamConfig] = Field( 2975 ..., 2976 description="Specifies which parent streams are being iterated over and how parent records should be used to partition the child stream data set.", 2977 title="Parent Stream Configs", 2978 ) 2979 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters")
parent_stream_configs: List[ParentStreamConfig]
class
GroupingPartitionRouter(pydantic.v1.main.BaseModel):
2982class GroupingPartitionRouter(BaseModel): 2983 type: Literal["GroupingPartitionRouter"] 2984 group_size: int = Field( 2985 ..., 2986 description="The number of partitions to include in each group. This determines how many partition values are batched together in a single slice.", 2987 examples=[10, 50], 2988 title="Group Size", 2989 ) 2990 underlying_partition_router: Union[ 2991 ListPartitionRouter, SubstreamPartitionRouter, CustomPartitionRouter 2992 ] = Field( 2993 ..., 2994 description="The partition router whose output will be grouped. This can be any valid partition router component.", 2995 title="Underlying Partition Router", 2996 ) 2997 deduplicate: Optional[bool] = Field( 2998 True, 2999 description="If true, ensures that partitions are unique within each group by removing duplicates based on the partition key.", 3000 title="Deduplicate Partitions", 3001 ) 3002 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters")
underlying_partition_router: Union[ListPartitionRouter, SubstreamPartitionRouter, CustomPartitionRouter]
class
HttpComponentsResolver(pydantic.v1.main.BaseModel):
3005class HttpComponentsResolver(BaseModel): 3006 type: Literal["HttpComponentsResolver"] 3007 retriever: Union[SimpleRetriever, AsyncRetriever, CustomRetriever] = Field( 3008 ..., 3009 description="Component used to coordinate how records are extracted across stream slices and request pages.", 3010 title="Retriever", 3011 ) 3012 components_mapping: List[ComponentMappingDefinition] 3013 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters")
retriever: Union[SimpleRetriever, AsyncRetriever, CustomRetriever]
components_mapping: List[ComponentMappingDefinition]
class
DynamicDeclarativeStream(pydantic.v1.main.BaseModel):
3016class DynamicDeclarativeStream(BaseModel): 3017 type: Literal["DynamicDeclarativeStream"] 3018 name: Optional[str] = Field( 3019 "", description="The dynamic stream name.", example=["Tables"], title="Name" 3020 ) 3021 stream_template: Union[DeclarativeStream, StateDelegatingStream] = Field( 3022 ..., description="Reference to the stream template.", title="Stream Template" 3023 ) 3024 components_resolver: Union[ 3025 HttpComponentsResolver, ConfigComponentsResolver, ParametrizedComponentsResolver 3026 ] = Field( 3027 ..., 3028 description="Component resolve and populates stream templates with components values.", 3029 title="Components Resolver", 3030 ) 3031 use_parent_parameters: Optional[bool] = Field( 3032 True, 3033 description="Whether or not to prioritize parent parameters over component parameters when constructing dynamic streams. Defaults to true for backward compatibility.", 3034 title="Use Parent Parameters", 3035 )
stream_template: Union[DeclarativeStream, StateDelegatingStream]
components_resolver: Union[HttpComponentsResolver, ConfigComponentsResolver, ParametrizedComponentsResolver]