airbyte_cdk.sources.declarative.models.declarative_component_schema
1# Copyright (c) 2025 Airbyte, Inc., all rights reserved. 2 3# generated by datamodel-codegen: 4# filename: declarative_component_schema.yaml 5 6from __future__ import annotations 7 8from enum import Enum 9from typing import Any, Dict, List, Literal, Optional, Union 10 11from pydantic.v1 import BaseModel, Extra, Field 12 13from airbyte_cdk.sources.declarative.models.base_model_with_deprecations import ( 14 BaseModelWithDeprecations, 15) 16 17 18class AuthFlowType(Enum): 19 oauth2_0 = "oauth2.0" 20 oauth1_0 = "oauth1.0" 21 22 23class BasicHttpAuthenticator(BaseModel): 24 type: Literal["BasicHttpAuthenticator"] 25 username: str = Field( 26 ..., 27 description="The username that will be combined with the password, base64 encoded and used to make requests. Fill it in the user inputs.", 28 examples=["{{ config['username'] }}", "{{ config['api_key'] }}"], 29 title="Username", 30 ) 31 password: Optional[str] = Field( 32 "", 33 description="The password that will be combined with the username, base64 encoded and used to make requests. Fill it in the user inputs.", 34 examples=["{{ config['password'] }}", ""], 35 title="Password", 36 ) 37 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters") 38 39 40class BearerAuthenticator(BaseModel): 41 type: Literal["BearerAuthenticator"] 42 api_token: str = Field( 43 ..., 44 description="Token to inject as request header for authenticating with the API.", 45 examples=["{{ config['api_key'] }}", "{{ config['token'] }}"], 46 title="Bearer Token", 47 ) 48 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters") 49 50 51class DynamicStreamCheckConfig(BaseModel): 52 type: Literal["DynamicStreamCheckConfig"] 53 dynamic_stream_name: str = Field( 54 ..., description="The dynamic stream name.", title="Dynamic Stream Name" 55 ) 56 stream_count: Optional[int] = Field( 57 0, 58 description="The number of streams to attempt reading from during a check operation. If `stream_count` exceeds the total number of available streams, the minimum of the two values will be used.", 59 title="Stream Count", 60 ) 61 62 63class CheckDynamicStream(BaseModel): 64 type: Literal["CheckDynamicStream"] 65 stream_count: int = Field( 66 ..., 67 description="Numbers of the streams to try reading from when running a check operation.", 68 title="Stream Count", 69 ) 70 use_check_availability: Optional[bool] = Field( 71 True, 72 description="Enables stream check availability. This field is automatically set by the CDK.", 73 title="Use Check Availability", 74 ) 75 76 77class ConcurrencyLevel(BaseModel): 78 type: Optional[Literal["ConcurrencyLevel"]] = None 79 default_concurrency: Union[int, str] = Field( 80 ..., 81 description="The amount of concurrency that will applied during a sync. This value can be hardcoded or user-defined in the config if different users have varying volume thresholds in the target API.", 82 examples=[10, "{{ config['num_workers'] or 10 }}"], 83 title="Default Concurrency", 84 ) 85 max_concurrency: Optional[int] = Field( 86 None, 87 description="The maximum level of concurrency that will be used during a sync. This becomes a required field when the default_concurrency derives from the config, because it serves as a safeguard against a user-defined threshold that is too high.", 88 examples=[20, 100], 89 title="Max Concurrency", 90 ) 91 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters") 92 93 94class ConstantBackoffStrategy(BaseModel): 95 type: Literal["ConstantBackoffStrategy"] 96 backoff_time_in_seconds: Union[float, str] = Field( 97 ..., 98 description="Backoff time in seconds.", 99 examples=[30, 30.5, "{{ config['backoff_time'] }}"], 100 title="Backoff Time", 101 ) 102 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters") 103 104 105class CursorPagination(BaseModel): 106 type: Literal["CursorPagination"] 107 cursor_value: str = Field( 108 ..., 109 description="Value of the cursor defining the next page to fetch.", 110 examples=[ 111 "{{ headers.link.next.cursor }}", 112 "{{ last_record['key'] }}", 113 "{{ response['nextPage'] }}", 114 ], 115 title="Cursor Value", 116 ) 117 page_size: Optional[int] = Field( 118 None, 119 description="The number of records to include in each pages.", 120 examples=[100], 121 title="Page Size", 122 ) 123 stop_condition: Optional[str] = Field( 124 None, 125 description="Template string evaluating when to stop paginating.", 126 examples=[ 127 "{{ response.data.has_more is false }}", 128 "{{ 'next' not in headers['link'] }}", 129 ], 130 title="Stop Condition", 131 ) 132 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters") 133 134 135class CustomAuthenticator(BaseModel): 136 class Config: 137 extra = Extra.allow 138 139 type: Literal["CustomAuthenticator"] 140 class_name: str = Field( 141 ..., 142 description="Fully-qualified name of the class that will be implementing the custom authentication strategy. Has to be a sub class of DeclarativeAuthenticator. The format is `source_<name>.<package>.<class_name>`.", 143 examples=["source_railz.components.ShortLivedTokenAuthenticator"], 144 title="Class Name", 145 ) 146 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters") 147 148 149class CustomBackoffStrategy(BaseModel): 150 class Config: 151 extra = Extra.allow 152 153 type: Literal["CustomBackoffStrategy"] 154 class_name: str = Field( 155 ..., 156 description="Fully-qualified name of the class that will be implementing the custom backoff strategy. The format is `source_<name>.<package>.<class_name>`.", 157 examples=["source_railz.components.MyCustomBackoffStrategy"], 158 title="Class Name", 159 ) 160 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters") 161 162 163class CustomErrorHandler(BaseModel): 164 class Config: 165 extra = Extra.allow 166 167 type: Literal["CustomErrorHandler"] 168 class_name: str = Field( 169 ..., 170 description="Fully-qualified name of the class that will be implementing the custom error handler. The format is `source_<name>.<package>.<class_name>`.", 171 examples=["source_railz.components.MyCustomErrorHandler"], 172 title="Class Name", 173 ) 174 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters") 175 176 177class CustomIncrementalSync(BaseModel): 178 class Config: 179 extra = Extra.allow 180 181 type: Literal["CustomIncrementalSync"] 182 class_name: str = Field( 183 ..., 184 description="Fully-qualified name of the class that will be implementing the custom incremental sync. The format is `source_<name>.<package>.<class_name>`.", 185 examples=["source_railz.components.MyCustomIncrementalSync"], 186 title="Class Name", 187 ) 188 cursor_field: str = Field( 189 ..., 190 description="The location of the value on a record that will be used as a bookmark during sync.", 191 ) 192 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters") 193 194 195class CustomPaginationStrategy(BaseModel): 196 class Config: 197 extra = Extra.allow 198 199 type: Literal["CustomPaginationStrategy"] 200 class_name: str = Field( 201 ..., 202 description="Fully-qualified name of the class that will be implementing the custom pagination strategy. The format is `source_<name>.<package>.<class_name>`.", 203 examples=["source_railz.components.MyCustomPaginationStrategy"], 204 title="Class Name", 205 ) 206 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters") 207 208 209class CustomRecordExtractor(BaseModel): 210 class Config: 211 extra = Extra.allow 212 213 type: Literal["CustomRecordExtractor"] 214 class_name: str = Field( 215 ..., 216 description="Fully-qualified name of the class that will be implementing the custom record extraction strategy. The format is `source_<name>.<package>.<class_name>`.", 217 examples=["source_railz.components.MyCustomRecordExtractor"], 218 title="Class Name", 219 ) 220 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters") 221 222 223class CustomRecordFilter(BaseModel): 224 class Config: 225 extra = Extra.allow 226 227 type: Literal["CustomRecordFilter"] 228 class_name: str = Field( 229 ..., 230 description="Fully-qualified name of the class that will be implementing the custom record filter strategy. The format is `source_<name>.<package>.<class_name>`.", 231 examples=["source_railz.components.MyCustomCustomRecordFilter"], 232 title="Class Name", 233 ) 234 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters") 235 236 237class CustomRequester(BaseModel): 238 class Config: 239 extra = Extra.allow 240 241 type: Literal["CustomRequester"] 242 class_name: str = Field( 243 ..., 244 description="Fully-qualified name of the class that will be implementing the custom requester strategy. The format is `source_<name>.<package>.<class_name>`.", 245 examples=["source_railz.components.MyCustomRecordExtractor"], 246 title="Class Name", 247 ) 248 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters") 249 250 251class CustomRetriever(BaseModel): 252 class Config: 253 extra = Extra.allow 254 255 type: Literal["CustomRetriever"] 256 class_name: str = Field( 257 ..., 258 description="Fully-qualified name of the class that will be implementing the custom retriever strategy. The format is `source_<name>.<package>.<class_name>`.", 259 examples=["source_railz.components.MyCustomRetriever"], 260 title="Class Name", 261 ) 262 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters") 263 264 265class CustomPartitionRouter(BaseModel): 266 class Config: 267 extra = Extra.allow 268 269 type: Literal["CustomPartitionRouter"] 270 class_name: str = Field( 271 ..., 272 description="Fully-qualified name of the class that will be implementing the custom partition router. The format is `source_<name>.<package>.<class_name>`.", 273 examples=["source_railz.components.MyCustomPartitionRouter"], 274 title="Class Name", 275 ) 276 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters") 277 278 279class CustomSchemaLoader(BaseModel): 280 class Config: 281 extra = Extra.allow 282 283 type: Literal["CustomSchemaLoader"] 284 class_name: str = Field( 285 ..., 286 description="Fully-qualified name of the class that will be implementing the custom schema loader. The format is `source_<name>.<package>.<class_name>`.", 287 examples=["source_railz.components.MyCustomSchemaLoader"], 288 title="Class Name", 289 ) 290 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters") 291 292 293class CustomSchemaNormalization(BaseModel): 294 class Config: 295 extra = Extra.allow 296 297 type: Literal["CustomSchemaNormalization"] 298 class_name: str = Field( 299 ..., 300 description="Fully-qualified name of the class that will be implementing the custom normalization. The format is `source_<name>.<package>.<class_name>`.", 301 examples=[ 302 "source_amazon_seller_partner.components.LedgerDetailedViewReportsTypeTransformer" 303 ], 304 title="Class Name", 305 ) 306 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters") 307 308 309class CustomStateMigration(BaseModel): 310 class Config: 311 extra = Extra.allow 312 313 type: Literal["CustomStateMigration"] 314 class_name: str = Field( 315 ..., 316 description="Fully-qualified name of the class that will be implementing the custom state migration. The format is `source_<name>.<package>.<class_name>`.", 317 examples=["source_railz.components.MyCustomStateMigration"], 318 title="Class Name", 319 ) 320 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters") 321 322 323class CustomTransformation(BaseModel): 324 class Config: 325 extra = Extra.allow 326 327 type: Literal["CustomTransformation"] 328 class_name: str = Field( 329 ..., 330 description="Fully-qualified name of the class that will be implementing the custom transformation. The format is `source_<name>.<package>.<class_name>`.", 331 examples=["source_railz.components.MyCustomTransformation"], 332 title="Class Name", 333 ) 334 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters") 335 336 337class LegacyToPerPartitionStateMigration(BaseModel): 338 class Config: 339 extra = Extra.allow 340 341 type: Optional[Literal["LegacyToPerPartitionStateMigration"]] = None 342 343 344class Clamping(BaseModel): 345 target: str = Field( 346 ..., 347 description="The period of time that datetime windows will be clamped by", 348 examples=["DAY", "WEEK", "MONTH", "{{ config['target'] }}"], 349 title="Target", 350 ) 351 target_details: Optional[Dict[str, Any]] = None 352 353 354class Algorithm(Enum): 355 HS256 = "HS256" 356 HS384 = "HS384" 357 HS512 = "HS512" 358 ES256 = "ES256" 359 ES256K = "ES256K" 360 ES384 = "ES384" 361 ES512 = "ES512" 362 RS256 = "RS256" 363 RS384 = "RS384" 364 RS512 = "RS512" 365 PS256 = "PS256" 366 PS384 = "PS384" 367 PS512 = "PS512" 368 EdDSA = "EdDSA" 369 370 371class JwtHeaders(BaseModel): 372 class Config: 373 extra = Extra.forbid 374 375 kid: Optional[str] = Field( 376 None, 377 description="Private key ID for user account.", 378 examples=["{{ config['kid'] }}"], 379 title="Key Identifier", 380 ) 381 typ: Optional[str] = Field( 382 "JWT", 383 description="The media type of the complete JWT.", 384 examples=["JWT"], 385 title="Type", 386 ) 387 cty: Optional[str] = Field( 388 None, 389 description="Content type of JWT header.", 390 examples=["JWT"], 391 title="Content Type", 392 ) 393 394 395class JwtPayload(BaseModel): 396 class Config: 397 extra = Extra.forbid 398 399 iss: Optional[str] = Field( 400 None, 401 description="The user/principal that issued the JWT. Commonly a value unique to the user.", 402 examples=["{{ config['iss'] }}"], 403 title="Issuer", 404 ) 405 sub: Optional[str] = Field( 406 None, 407 description="The subject of the JWT. Commonly defined by the API.", 408 title="Subject", 409 ) 410 aud: Optional[str] = Field( 411 None, 412 description="The recipient that the JWT is intended for. Commonly defined by the API.", 413 examples=["appstoreconnect-v1"], 414 title="Audience", 415 ) 416 417 418class JwtAuthenticator(BaseModel): 419 type: Literal["JwtAuthenticator"] 420 secret_key: str = Field( 421 ..., 422 description="Secret used to sign the JSON web token.", 423 examples=["{{ config['secret_key'] }}"], 424 title="Secret Key", 425 ) 426 base64_encode_secret_key: Optional[bool] = Field( 427 False, 428 description='When set to true, the secret key will be base64 encoded prior to being encoded as part of the JWT. Only set to "true" when required by the API.', 429 title="Base64-encode Secret Key", 430 ) 431 algorithm: Algorithm = Field( 432 ..., 433 description="Algorithm used to sign the JSON web token.", 434 examples=["ES256", "HS256", "RS256", "{{ config['algorithm'] }}"], 435 title="Algorithm", 436 ) 437 token_duration: Optional[int] = Field( 438 1200, 439 description="The amount of time in seconds a JWT token can be valid after being issued.", 440 examples=[1200, 3600], 441 title="Token Duration", 442 ) 443 header_prefix: Optional[str] = Field( 444 None, 445 description="The prefix to be used within the Authentication header.", 446 examples=["Bearer", "Basic"], 447 title="Header Prefix", 448 ) 449 jwt_headers: Optional[JwtHeaders] = Field( 450 None, 451 description="JWT headers used when signing JSON web token.", 452 title="JWT Headers", 453 ) 454 additional_jwt_headers: Optional[Dict[str, Any]] = Field( 455 None, 456 description="Additional headers to be included with the JWT headers object.", 457 title="Additional JWT Headers", 458 ) 459 jwt_payload: Optional[JwtPayload] = Field( 460 None, 461 description="JWT Payload used when signing JSON web token.", 462 title="JWT Payload", 463 ) 464 additional_jwt_payload: Optional[Dict[str, Any]] = Field( 465 None, 466 description="Additional properties to be added to the JWT payload.", 467 title="Additional JWT Payload Properties", 468 ) 469 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters") 470 471 472class RefreshTokenUpdater(BaseModel): 473 refresh_token_name: Optional[str] = Field( 474 "refresh_token", 475 description="The name of the property which contains the updated refresh token in the response from the token refresh endpoint.", 476 examples=["refresh_token"], 477 title="Refresh Token Property Name", 478 ) 479 access_token_config_path: Optional[List[str]] = Field( 480 ["credentials", "access_token"], 481 description="Config path to the access token. Make sure the field actually exists in the config.", 482 examples=[["credentials", "access_token"], ["access_token"]], 483 title="Config Path To Access Token", 484 ) 485 refresh_token_config_path: Optional[List[str]] = Field( 486 ["credentials", "refresh_token"], 487 description="Config path to the access token. Make sure the field actually exists in the config.", 488 examples=[["credentials", "refresh_token"], ["refresh_token"]], 489 title="Config Path To Refresh Token", 490 ) 491 token_expiry_date_config_path: Optional[List[str]] = Field( 492 ["credentials", "token_expiry_date"], 493 description="Config path to the expiry date. Make sure actually exists in the config.", 494 examples=[["credentials", "token_expiry_date"]], 495 title="Config Path To Expiry Date", 496 ) 497 refresh_token_error_status_codes: Optional[List[int]] = Field( 498 [], 499 description="Status Codes to Identify refresh token error in response (Refresh Token Error Key and Refresh Token Error Values should be also specified). Responses with one of the error status code and containing an error value will be flagged as a config error", 500 examples=[[400, 500]], 501 title="Refresh Token Error Status Codes", 502 ) 503 refresh_token_error_key: Optional[str] = Field( 504 "", 505 description="Key to Identify refresh token error in response (Refresh Token Error Status Codes and Refresh Token Error Values should be also specified).", 506 examples=["error"], 507 title="Refresh Token Error Key", 508 ) 509 refresh_token_error_values: Optional[List[str]] = Field( 510 [], 511 description='List of values to check for exception during token refresh process. Used to check if the error found in the response matches the key from the Refresh Token Error Key field (e.g. response={"error": "invalid_grant"}). Only responses with one of the error status code and containing an error value will be flagged as a config error', 512 examples=[["invalid_grant", "invalid_permissions"]], 513 title="Refresh Token Error Values", 514 ) 515 516 517class OAuthAuthenticator(BaseModel): 518 type: Literal["OAuthAuthenticator"] 519 client_id_name: Optional[str] = Field( 520 "client_id", 521 description="The name of the property to use to refresh the `access_token`.", 522 examples=["custom_app_id"], 523 title="Client ID Property Name", 524 ) 525 client_id: Optional[str] = Field( 526 None, 527 description="The OAuth client ID. Fill it in the user inputs.", 528 examples=["{{ config['client_id }}", "{{ config['credentials']['client_id }}"], 529 title="Client ID", 530 ) 531 client_secret_name: Optional[str] = Field( 532 "client_secret", 533 description="The name of the property to use to refresh the `access_token`.", 534 examples=["custom_app_secret"], 535 title="Client Secret Property Name", 536 ) 537 client_secret: Optional[str] = Field( 538 None, 539 description="The OAuth client secret. Fill it in the user inputs.", 540 examples=[ 541 "{{ config['client_secret }}", 542 "{{ config['credentials']['client_secret }}", 543 ], 544 title="Client Secret", 545 ) 546 refresh_token_name: Optional[str] = Field( 547 "refresh_token", 548 description="The name of the property to use to refresh the `access_token`.", 549 examples=["custom_app_refresh_value"], 550 title="Refresh Token Property Name", 551 ) 552 refresh_token: Optional[str] = Field( 553 None, 554 description="Credential artifact used to get a new access token.", 555 examples=[ 556 "{{ config['refresh_token'] }}", 557 "{{ config['credentials]['refresh_token'] }}", 558 ], 559 title="Refresh Token", 560 ) 561 token_refresh_endpoint: Optional[str] = Field( 562 None, 563 description="The full URL to call to obtain a new access token.", 564 examples=["https://connect.squareup.com/oauth2/token"], 565 title="Token Refresh Endpoint", 566 ) 567 access_token_name: Optional[str] = Field( 568 "access_token", 569 description="The name of the property which contains the access token in the response from the token refresh endpoint.", 570 examples=["access_token"], 571 title="Access Token Property Name", 572 ) 573 access_token_value: Optional[str] = Field( 574 None, 575 description="The value of the access_token to bypass the token refreshing using `refresh_token`.", 576 examples=["secret_access_token_value"], 577 title="Access Token Value", 578 ) 579 expires_in_name: Optional[str] = Field( 580 "expires_in", 581 description="The name of the property which contains the expiry date in the response from the token refresh endpoint.", 582 examples=["expires_in"], 583 title="Token Expiry Property Name", 584 ) 585 grant_type_name: Optional[str] = Field( 586 "grant_type", 587 description="The name of the property to use to refresh the `access_token`.", 588 examples=["custom_grant_type"], 589 title="Grant Type Property Name", 590 ) 591 grant_type: Optional[str] = Field( 592 "refresh_token", 593 description="Specifies the OAuth2 grant type. If set to refresh_token, the refresh_token needs to be provided as well. For client_credentials, only client id and secret are required. Other grant types are not officially supported.", 594 examples=["refresh_token", "client_credentials"], 595 title="Grant Type", 596 ) 597 refresh_request_body: Optional[Dict[str, Any]] = Field( 598 None, 599 description="Body of the request sent to get a new access token.", 600 examples=[ 601 { 602 "applicationId": "{{ config['application_id'] }}", 603 "applicationSecret": "{{ config['application_secret'] }}", 604 "token": "{{ config['token'] }}", 605 } 606 ], 607 title="Refresh Request Body", 608 ) 609 refresh_request_headers: Optional[Dict[str, Any]] = Field( 610 None, 611 description="Headers of the request sent to get a new access token.", 612 examples=[ 613 { 614 "Authorization": "<AUTH_TOKEN>", 615 "Content-Type": "application/x-www-form-urlencoded", 616 } 617 ], 618 title="Refresh Request Headers", 619 ) 620 scopes: Optional[List[str]] = Field( 621 None, 622 description="List of scopes that should be granted to the access token.", 623 examples=[["crm.list.read", "crm.objects.contacts.read", "crm.schema.contacts.read"]], 624 title="Scopes", 625 ) 626 token_expiry_date: Optional[str] = Field( 627 None, 628 description="The access token expiry date.", 629 examples=["2023-04-06T07:12:10.421833+00:00", 1680842386], 630 title="Token Expiry Date", 631 ) 632 token_expiry_date_format: Optional[str] = Field( 633 None, 634 description="The format of the time to expiration datetime. Provide it if the time is returned as a date-time string instead of seconds.", 635 examples=["%Y-%m-%d %H:%M:%S.%f+00:00"], 636 title="Token Expiry Date Format", 637 ) 638 refresh_token_updater: Optional[RefreshTokenUpdater] = Field( 639 None, 640 description="When the token updater is defined, new refresh tokens, access tokens and the access token expiry date are written back from the authentication response to the config object. This is important if the refresh token can only used once.", 641 title="Token Updater", 642 ) 643 profile_assertion: Optional[JwtAuthenticator] = Field( 644 None, 645 description="The authenticator being used to authenticate the client authenticator.", 646 title="Profile Assertion", 647 ) 648 use_profile_assertion: Optional[bool] = Field( 649 False, 650 description="Enable using profile assertion as a flow for OAuth authorization.", 651 title="Use Profile Assertion", 652 ) 653 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters") 654 655 656class Rate(BaseModel): 657 class Config: 658 extra = Extra.allow 659 660 limit: Union[int, str] = Field( 661 ..., 662 description="The maximum number of calls allowed within the interval.", 663 title="Limit", 664 ) 665 interval: str = Field( 666 ..., 667 description="The time interval for the rate limit.", 668 examples=["PT1H", "P1D"], 669 title="Interval", 670 ) 671 672 673class HttpRequestRegexMatcher(BaseModel): 674 class Config: 675 extra = Extra.allow 676 677 method: Optional[str] = Field( 678 None, description="The HTTP method to match (e.g., GET, POST).", title="Method" 679 ) 680 url_base: Optional[str] = Field( 681 None, 682 description='The base URL (scheme and host, e.g. "https://api.example.com") to match.', 683 title="URL Base", 684 ) 685 url_path_pattern: Optional[str] = Field( 686 None, 687 description="A regular expression pattern to match the URL path.", 688 title="URL Path Pattern", 689 ) 690 params: Optional[Dict[str, Any]] = Field( 691 None, description="The query parameters to match.", title="Parameters" 692 ) 693 headers: Optional[Dict[str, Any]] = Field( 694 None, description="The headers to match.", title="Headers" 695 ) 696 697 698class DpathExtractor(BaseModel): 699 type: Literal["DpathExtractor"] 700 field_path: List[str] = Field( 701 ..., 702 description='List of potentially nested fields describing the full path of the field to extract. Use "*" to extract all values from an array. See more info in the [docs](https://docs.airbyte.com/connector-development/config-based/understanding-the-yaml-file/record-selector).', 703 examples=[ 704 ["data"], 705 ["data", "records"], 706 ["data", "{{ parameters.name }}"], 707 ["data", "*", "record"], 708 ], 709 title="Field Path", 710 ) 711 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters") 712 713 714class ResponseToFileExtractor(BaseModel): 715 type: Literal["ResponseToFileExtractor"] 716 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters") 717 718 719class ExponentialBackoffStrategy(BaseModel): 720 type: Literal["ExponentialBackoffStrategy"] 721 factor: Optional[Union[float, str]] = Field( 722 5, 723 description="Multiplicative constant applied on each retry.", 724 examples=[5, 5.5, "10"], 725 title="Factor", 726 ) 727 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters") 728 729 730class GroupByKeyMergeStrategy(BaseModel): 731 type: Literal["GroupByKeyMergeStrategy"] 732 key: Union[str, List[str]] = Field( 733 ..., 734 description="The name of the field on the record whose value will be used to group properties that were retrieved through multiple API requests.", 735 examples=["id", ["parent_id", "end_date"]], 736 title="Key", 737 ) 738 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters") 739 740 741class SessionTokenRequestBearerAuthenticator(BaseModel): 742 type: Literal["Bearer"] 743 744 745class HttpMethod(Enum): 746 GET = "GET" 747 POST = "POST" 748 749 750class Action(Enum): 751 SUCCESS = "SUCCESS" 752 FAIL = "FAIL" 753 RETRY = "RETRY" 754 IGNORE = "IGNORE" 755 RATE_LIMITED = "RATE_LIMITED" 756 757 758class FailureType(Enum): 759 system_error = "system_error" 760 config_error = "config_error" 761 transient_error = "transient_error" 762 763 764class HttpResponseFilter(BaseModel): 765 type: Literal["HttpResponseFilter"] 766 action: Optional[Action] = Field( 767 None, 768 description="Action to execute if a response matches the filter.", 769 examples=["SUCCESS", "FAIL", "RETRY", "IGNORE", "RATE_LIMITED"], 770 title="Action", 771 ) 772 failure_type: Optional[FailureType] = Field( 773 None, 774 description="Failure type of traced exception if a response matches the filter.", 775 examples=["system_error", "config_error", "transient_error"], 776 title="Failure Type", 777 ) 778 error_message: Optional[str] = Field( 779 None, 780 description="Error Message to display if the response matches the filter.", 781 title="Error Message", 782 ) 783 error_message_contains: Optional[str] = Field( 784 None, 785 description="Match the response if its error message contains the substring.", 786 example=["This API operation is not enabled for this site"], 787 title="Error Message Substring", 788 ) 789 http_codes: Optional[List[int]] = Field( 790 None, 791 description="Match the response if its HTTP code is included in this list.", 792 examples=[[420, 429], [500]], 793 title="HTTP Codes", 794 unique_items=True, 795 ) 796 predicate: Optional[str] = Field( 797 None, 798 description="Match the response if the predicate evaluates to true.", 799 examples=[ 800 "{{ 'Too much requests' in response }}", 801 "{{ 'error_code' in response and response['error_code'] == 'ComplexityException' }}", 802 ], 803 title="Predicate", 804 ) 805 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters") 806 807 808class ComplexFieldType(BaseModel): 809 field_type: str 810 items: Optional[Union[str, ComplexFieldType]] = None 811 812 813class TypesMap(BaseModel): 814 target_type: Union[str, List[str], ComplexFieldType] 815 current_type: Union[str, List[str]] 816 condition: Optional[str] = None 817 818 819class SchemaTypeIdentifier(BaseModel): 820 type: Optional[Literal["SchemaTypeIdentifier"]] = None 821 schema_pointer: Optional[List[str]] = Field( 822 [], 823 description="List of nested fields defining the schema field path to extract. Defaults to [].", 824 title="Schema Path", 825 ) 826 key_pointer: List[str] = Field( 827 ..., 828 description="List of potentially nested fields describing the full path of the field key to extract.", 829 title="Key Path", 830 ) 831 type_pointer: Optional[List[str]] = Field( 832 None, 833 description="List of potentially nested fields describing the full path of the field type to extract.", 834 title="Type Path", 835 ) 836 types_mapping: Optional[List[TypesMap]] = None 837 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters") 838 839 840class InlineSchemaLoader(BaseModel): 841 type: Literal["InlineSchemaLoader"] 842 schema_: Optional[Dict[str, Any]] = Field( 843 None, 844 alias="schema", 845 description='Describes a streams\' schema. Refer to the <a href="https://docs.airbyte.com/understanding-airbyte/supported-data-types/">Data Types documentation</a> for more details on which types are valid.', 846 title="Schema", 847 ) 848 849 850class JsonFileSchemaLoader(BaseModel): 851 type: Literal["JsonFileSchemaLoader"] 852 file_path: Optional[str] = Field( 853 None, 854 description="Path to the JSON file defining the schema. The path is relative to the connector module's root.", 855 example=["./schemas/users.json"], 856 title="File Path", 857 ) 858 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters") 859 860 861class JsonDecoder(BaseModel): 862 type: Literal["JsonDecoder"] 863 864 865class JsonlDecoder(BaseModel): 866 type: Literal["JsonlDecoder"] 867 868 869class KeysToLower(BaseModel): 870 type: Literal["KeysToLower"] 871 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters") 872 873 874class KeysToSnakeCase(BaseModel): 875 type: Literal["KeysToSnakeCase"] 876 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters") 877 878 879class FlattenFields(BaseModel): 880 type: Literal["FlattenFields"] 881 flatten_lists: Optional[bool] = Field( 882 True, 883 description="Whether to flatten lists or leave it as is. Default is True.", 884 title="Flatten Lists", 885 ) 886 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters") 887 888 889class KeyTransformation(BaseModel): 890 type: Literal["KeyTransformation"] 891 prefix: Optional[str] = Field( 892 None, 893 description="Prefix to add for object keys. If not provided original keys remain unchanged.", 894 examples=["flattened_"], 895 title="Key Prefix", 896 ) 897 suffix: Optional[str] = Field( 898 None, 899 description="Suffix to add for object keys. If not provided original keys remain unchanged.", 900 examples=["_flattened"], 901 title="Key Suffix", 902 ) 903 904 905class DpathFlattenFields(BaseModel): 906 type: Literal["DpathFlattenFields"] 907 field_path: List[str] = Field( 908 ..., 909 description="A path to field that needs to be flattened.", 910 examples=[["data"], ["data", "*", "field"]], 911 title="Field Path", 912 ) 913 delete_origin_value: Optional[bool] = Field( 914 None, 915 description="Whether to delete the origin value or keep it. Default is False.", 916 title="Delete Origin Value", 917 ) 918 replace_record: Optional[bool] = Field( 919 None, 920 description="Whether to replace the origin record or not. Default is False.", 921 title="Replace Origin Record", 922 ) 923 key_transformation: Optional[KeyTransformation] = Field( 924 None, 925 description="Transformation for object keys. If not provided, original key will be used.", 926 title="Key transformation", 927 ) 928 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters") 929 930 931class KeysReplace(BaseModel): 932 type: Literal["KeysReplace"] 933 old: str = Field( 934 ..., 935 description="Old value to replace.", 936 examples=[ 937 " ", 938 "{{ record.id }}", 939 "{{ config['id'] }}", 940 "{{ stream_slice['id'] }}", 941 ], 942 title="Old value", 943 ) 944 new: str = Field( 945 ..., 946 description="New value to set.", 947 examples=[ 948 "_", 949 "{{ record.id }}", 950 "{{ config['id'] }}", 951 "{{ stream_slice['id'] }}", 952 ], 953 title="New value", 954 ) 955 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters") 956 957 958class IterableDecoder(BaseModel): 959 type: Literal["IterableDecoder"] 960 961 962class XmlDecoder(BaseModel): 963 type: Literal["XmlDecoder"] 964 965 966class CustomDecoder(BaseModel): 967 class Config: 968 extra = Extra.allow 969 970 type: Literal["CustomDecoder"] 971 class_name: str = Field( 972 ..., 973 description="Fully-qualified name of the class that will be implementing the custom decoding. Has to be a sub class of Decoder. The format is `source_<name>.<package>.<class_name>`.", 974 examples=["source_amazon_ads.components.GzipJsonlDecoder"], 975 title="Class Name", 976 ) 977 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters") 978 979 980class MinMaxDatetime(BaseModel): 981 type: Literal["MinMaxDatetime"] 982 datetime: str = Field( 983 ..., 984 description="Datetime value.", 985 examples=["2021-01-01", "2021-01-01T00:00:00Z", "{{ config['start_time'] }}"], 986 title="Datetime", 987 ) 988 datetime_format: Optional[str] = Field( 989 "", 990 description='Format of the datetime value. Defaults to "%Y-%m-%dT%H:%M:%S.%f%z" if left empty. Use placeholders starting with "%" to describe the format the API is using. The following placeholders are available:\n * **%s**: Epoch unix timestamp - `1686218963`\n * **%s_as_float**: Epoch unix timestamp in seconds as float with microsecond precision - `1686218963.123456`\n * **%ms**: Epoch unix timestamp - `1686218963123`\n * **%a**: Weekday (abbreviated) - `Sun`\n * **%A**: Weekday (full) - `Sunday`\n * **%w**: Weekday (decimal) - `0` (Sunday), `6` (Saturday)\n * **%d**: Day of the month (zero-padded) - `01`, `02`, ..., `31`\n * **%b**: Month (abbreviated) - `Jan`\n * **%B**: Month (full) - `January`\n * **%m**: Month (zero-padded) - `01`, `02`, ..., `12`\n * **%y**: Year (without century, zero-padded) - `00`, `01`, ..., `99`\n * **%Y**: Year (with century) - `0001`, `0002`, ..., `9999`\n * **%H**: Hour (24-hour, zero-padded) - `00`, `01`, ..., `23`\n * **%I**: Hour (12-hour, zero-padded) - `01`, `02`, ..., `12`\n * **%p**: AM/PM indicator\n * **%M**: Minute (zero-padded) - `00`, `01`, ..., `59`\n * **%S**: Second (zero-padded) - `00`, `01`, ..., `59`\n * **%f**: Microsecond (zero-padded to 6 digits) - `000000`, `000001`, ..., `999999`\n * **%_ms**: Millisecond (zero-padded to 3 digits) - `000`, `001`, ..., `999`\n * **%z**: UTC offset - `(empty)`, `+0000`, `-04:00`\n * **%Z**: Time zone name - `(empty)`, `UTC`, `GMT`\n * **%j**: Day of the year (zero-padded) - `001`, `002`, ..., `366`\n * **%U**: Week number of the year (Sunday as first day) - `00`, `01`, ..., `53`\n * **%W**: Week number of the year (Monday as first day) - `00`, `01`, ..., `53`\n * **%c**: Date and time representation - `Tue Aug 16 21:30:00 1988`\n * **%x**: Date representation - `08/16/1988`\n * **%X**: Time representation - `21:30:00`\n * **%%**: Literal \'%\' character\n\n Some placeholders depend on the locale of the underlying system - in most cases this locale is configured as en/US. For more information see the [Python documentation](https://docs.python.org/3/library/datetime.html#strftime-and-strptime-format-codes).\n', 991 examples=["%Y-%m-%dT%H:%M:%S.%f%z", "%Y-%m-%d", "%s"], 992 title="Datetime Format", 993 ) 994 max_datetime: Optional[str] = Field( 995 None, 996 description="Ceiling applied on the datetime value. Must be formatted with the datetime_format field.", 997 examples=["2021-01-01T00:00:00Z", "2021-01-01"], 998 title="Max Datetime", 999 ) 1000 min_datetime: Optional[str] = Field( 1001 None, 1002 description="Floor applied on the datetime value. Must be formatted with the datetime_format field.", 1003 examples=["2010-01-01T00:00:00Z", "2010-01-01"], 1004 title="Min Datetime", 1005 ) 1006 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters") 1007 1008 1009class NoAuth(BaseModel): 1010 type: Literal["NoAuth"] 1011 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters") 1012 1013 1014class NoPagination(BaseModel): 1015 type: Literal["NoPagination"] 1016 1017 1018class State(BaseModel): 1019 class Config: 1020 extra = Extra.allow 1021 1022 min: int 1023 max: int 1024 1025 1026class OauthConnectorInputSpecification(BaseModel): 1027 class Config: 1028 extra = Extra.allow 1029 1030 consent_url: str = Field( 1031 ..., 1032 description="The DeclarativeOAuth Specific string URL string template to initiate the authentication.\nThe placeholders are replaced during the processing to provide neccessary values.", 1033 examples=[ 1034 "https://domain.host.com/marketing_api/auth?{{client_id_key}}={{client_id_value}}&{{redirect_uri_key}}={{{{redirect_uri_value}} | urlEncoder}}&{{state_key}}={{state_value}}", 1035 "https://endpoint.host.com/oauth2/authorize?{{client_id_key}}={{client_id_value}}&{{redirect_uri_key}}={{{{redirect_uri_value}} | urlEncoder}}&{{scope_key}}={{{{scope_value}} | urlEncoder}}&{{state_key}}={{state_value}}&subdomain={{subdomain}}", 1036 ], 1037 title="Consent URL", 1038 ) 1039 scope: Optional[str] = Field( 1040 None, 1041 description="The DeclarativeOAuth Specific string of the scopes needed to be grant for authenticated user.", 1042 examples=["user:read user:read_orders workspaces:read"], 1043 title="Scopes", 1044 ) 1045 access_token_url: str = Field( 1046 ..., 1047 description="The DeclarativeOAuth Specific URL templated string to obtain the `access_token`, `refresh_token` etc.\nThe placeholders are replaced during the processing to provide neccessary values.", 1048 examples=[ 1049 "https://auth.host.com/oauth2/token?{{client_id_key}}={{client_id_value}}&{{client_secret_key}}={{client_secret_value}}&{{auth_code_key}}={{auth_code_value}}&{{redirect_uri_key}}={{{{redirect_uri_value}} | urlEncoder}}" 1050 ], 1051 title="Access Token URL", 1052 ) 1053 access_token_headers: Optional[Dict[str, Any]] = Field( 1054 None, 1055 description="The DeclarativeOAuth Specific optional headers to inject while exchanging the `auth_code` to `access_token` during `completeOAuthFlow` step.", 1056 examples=[ 1057 { 1058 "Authorization": "Basic {{ {{ client_id_value }}:{{ client_secret_value }} | base64Encoder }}" 1059 } 1060 ], 1061 title="Access Token Headers", 1062 ) 1063 access_token_params: Optional[Dict[str, Any]] = Field( 1064 None, 1065 description="The DeclarativeOAuth Specific optional query parameters to inject while exchanging the `auth_code` to `access_token` during `completeOAuthFlow` step.\nWhen this property is provided, the query params will be encoded as `Json` and included in the outgoing API request.", 1066 examples=[ 1067 { 1068 "{{ auth_code_key }}": "{{ auth_code_value }}", 1069 "{{ client_id_key }}": "{{ client_id_value }}", 1070 "{{ client_secret_key }}": "{{ client_secret_value }}", 1071 } 1072 ], 1073 title="Access Token Query Params (Json Encoded)", 1074 ) 1075 extract_output: Optional[List[str]] = Field( 1076 None, 1077 description="The DeclarativeOAuth Specific list of strings to indicate which keys should be extracted and returned back to the input config.", 1078 examples=[["access_token", "refresh_token", "other_field"]], 1079 title="Extract Output", 1080 ) 1081 state: Optional[State] = Field( 1082 None, 1083 description="The DeclarativeOAuth Specific object to provide the criteria of how the `state` query param should be constructed,\nincluding length and complexity.", 1084 examples=[{"min": 7, "max": 128}], 1085 title="Configurable State Query Param", 1086 ) 1087 client_id_key: Optional[str] = Field( 1088 None, 1089 description="The DeclarativeOAuth Specific optional override to provide the custom `client_id` key name, if required by data-provider.", 1090 examples=["my_custom_client_id_key_name"], 1091 title="Client ID Key Override", 1092 ) 1093 client_secret_key: Optional[str] = Field( 1094 None, 1095 description="The DeclarativeOAuth Specific optional override to provide the custom `client_secret` key name, if required by data-provider.", 1096 examples=["my_custom_client_secret_key_name"], 1097 title="Client Secret Key Override", 1098 ) 1099 scope_key: Optional[str] = Field( 1100 None, 1101 description="The DeclarativeOAuth Specific optional override to provide the custom `scope` key name, if required by data-provider.", 1102 examples=["my_custom_scope_key_key_name"], 1103 title="Scopes Key Override", 1104 ) 1105 state_key: Optional[str] = Field( 1106 None, 1107 description="The DeclarativeOAuth Specific optional override to provide the custom `state` key name, if required by data-provider.", 1108 examples=["my_custom_state_key_key_name"], 1109 title="State Key Override", 1110 ) 1111 auth_code_key: Optional[str] = Field( 1112 None, 1113 description="The DeclarativeOAuth Specific optional override to provide the custom `code` key name to something like `auth_code` or `custom_auth_code`, if required by data-provider.", 1114 examples=["my_custom_auth_code_key_name"], 1115 title="Auth Code Key Override", 1116 ) 1117 redirect_uri_key: Optional[str] = Field( 1118 None, 1119 description="The DeclarativeOAuth Specific optional override to provide the custom `redirect_uri` key name to something like `callback_uri`, if required by data-provider.", 1120 examples=["my_custom_redirect_uri_key_name"], 1121 title="Redirect URI Key Override", 1122 ) 1123 1124 1125class OAuthConfigSpecification(BaseModel): 1126 class Config: 1127 extra = Extra.allow 1128 1129 oauth_user_input_from_connector_config_specification: Optional[Dict[str, Any]] = Field( 1130 None, 1131 description="OAuth specific blob. This is a Json Schema used to validate Json configurations used as input to OAuth.\nMust be a valid non-nested JSON that refers to properties from ConnectorSpecification.connectionSpecification\nusing special annotation 'path_in_connector_config'.\nThese are input values the user is entering through the UI to authenticate to the connector, that might also shared\nas inputs for syncing data via the connector.\nExamples:\nif no connector values is shared during oauth flow, oauth_user_input_from_connector_config_specification=[]\nif connector values such as 'app_id' inside the top level are used to generate the API url for the oauth flow,\n oauth_user_input_from_connector_config_specification={\n app_id: {\n type: string\n path_in_connector_config: ['app_id']\n }\n }\nif connector values such as 'info.app_id' nested inside another object are used to generate the API url for the oauth flow,\n oauth_user_input_from_connector_config_specification={\n app_id: {\n type: string\n path_in_connector_config: ['info', 'app_id']\n }\n }", 1132 examples=[ 1133 {"app_id": {"type": "string", "path_in_connector_config": ["app_id"]}}, 1134 { 1135 "app_id": { 1136 "type": "string", 1137 "path_in_connector_config": ["info", "app_id"], 1138 } 1139 }, 1140 ], 1141 title="OAuth user input", 1142 ) 1143 oauth_connector_input_specification: Optional[OauthConnectorInputSpecification] = Field( 1144 None, 1145 description='The DeclarativeOAuth specific blob.\nPertains to the fields defined by the connector relating to the OAuth flow.\n\nInterpolation capabilities:\n- The variables placeholders are declared as `{{my_var}}`.\n- The nested resolution variables like `{{ {{my_nested_var}} }}` is allowed as well.\n\n- The allowed interpolation context is:\n + base64Encoder - encode to `base64`, {{ {{my_var_a}}:{{my_var_b}} | base64Encoder }}\n + base64Decorer - decode from `base64` encoded string, {{ {{my_string_variable_or_string_value}} | base64Decoder }}\n + urlEncoder - encode the input string to URL-like format, {{ https://test.host.com/endpoint | urlEncoder}}\n + urlDecorer - decode the input url-encoded string into text format, {{ urlDecoder:https%3A%2F%2Fairbyte.io | urlDecoder}}\n + codeChallengeS256 - get the `codeChallenge` encoded value to provide additional data-provider specific authorisation values, {{ {{state_value}} | codeChallengeS256 }}\n\nExamples:\n - The TikTok Marketing DeclarativeOAuth spec:\n {\n "oauth_connector_input_specification": {\n "type": "object",\n "additionalProperties": false,\n "properties": {\n "consent_url": "https://ads.tiktok.com/marketing_api/auth?{{client_id_key}}={{client_id_value}}&{{redirect_uri_key}}={{ {{redirect_uri_value}} | urlEncoder}}&{{state_key}}={{state_value}}",\n "access_token_url": "https://business-api.tiktok.com/open_api/v1.3/oauth2/access_token/",\n "access_token_params": {\n "{{ auth_code_key }}": "{{ auth_code_value }}",\n "{{ client_id_key }}": "{{ client_id_value }}",\n "{{ client_secret_key }}": "{{ client_secret_value }}"\n },\n "access_token_headers": {\n "Content-Type": "application/json",\n "Accept": "application/json"\n },\n "extract_output": ["data.access_token"],\n "client_id_key": "app_id",\n "client_secret_key": "secret",\n "auth_code_key": "auth_code"\n }\n }\n }', 1146 title="DeclarativeOAuth Connector Specification", 1147 ) 1148 complete_oauth_output_specification: Optional[Dict[str, Any]] = Field( 1149 None, 1150 description="OAuth specific blob. This is a Json Schema used to validate Json configurations produced by the OAuth flows as they are\nreturned by the distant OAuth APIs.\nMust be a valid JSON describing the fields to merge back to `ConnectorSpecification.connectionSpecification`.\nFor each field, a special annotation `path_in_connector_config` can be specified to determine where to merge it,\nExamples:\n complete_oauth_output_specification={\n refresh_token: {\n type: string,\n path_in_connector_config: ['credentials', 'refresh_token']\n }\n }", 1151 examples=[ 1152 { 1153 "refresh_token": { 1154 "type": "string,", 1155 "path_in_connector_config": ["credentials", "refresh_token"], 1156 } 1157 } 1158 ], 1159 title="OAuth output specification", 1160 ) 1161 complete_oauth_server_input_specification: Optional[Dict[str, Any]] = Field( 1162 None, 1163 description="OAuth specific blob. This is a Json Schema used to validate Json configurations persisted as Airbyte Server configurations.\nMust be a valid non-nested JSON describing additional fields configured by the Airbyte Instance or Workspace Admins to be used by the\nserver when completing an OAuth flow (typically exchanging an auth code for refresh token).\nExamples:\n complete_oauth_server_input_specification={\n client_id: {\n type: string\n },\n client_secret: {\n type: string\n }\n }", 1164 examples=[{"client_id": {"type": "string"}, "client_secret": {"type": "string"}}], 1165 title="OAuth input specification", 1166 ) 1167 complete_oauth_server_output_specification: Optional[Dict[str, Any]] = Field( 1168 None, 1169 description="OAuth specific blob. This is a Json Schema used to validate Json configurations persisted as Airbyte Server configurations that\nalso need to be merged back into the connector configuration at runtime.\nThis is a subset configuration of `complete_oauth_server_input_specification` that filters fields out to retain only the ones that\nare necessary for the connector to function with OAuth. (some fields could be used during oauth flows but not needed afterwards, therefore\nthey would be listed in the `complete_oauth_server_input_specification` but not `complete_oauth_server_output_specification`)\nMust be a valid non-nested JSON describing additional fields configured by the Airbyte Instance or Workspace Admins to be used by the\nconnector when using OAuth flow APIs.\nThese fields are to be merged back to `ConnectorSpecification.connectionSpecification`.\nFor each field, a special annotation `path_in_connector_config` can be specified to determine where to merge it,\nExamples:\n complete_oauth_server_output_specification={\n client_id: {\n type: string,\n path_in_connector_config: ['credentials', 'client_id']\n },\n client_secret: {\n type: string,\n path_in_connector_config: ['credentials', 'client_secret']\n }\n }", 1170 examples=[ 1171 { 1172 "client_id": { 1173 "type": "string,", 1174 "path_in_connector_config": ["credentials", "client_id"], 1175 }, 1176 "client_secret": { 1177 "type": "string,", 1178 "path_in_connector_config": ["credentials", "client_secret"], 1179 }, 1180 } 1181 ], 1182 title="OAuth server output specification", 1183 ) 1184 1185 1186class OffsetIncrement(BaseModel): 1187 type: Literal["OffsetIncrement"] 1188 page_size: Optional[Union[int, str]] = Field( 1189 None, 1190 description="The number of records to include in each pages.", 1191 examples=[100, "{{ config['page_size'] }}"], 1192 title="Limit", 1193 ) 1194 inject_on_first_request: Optional[bool] = Field( 1195 False, 1196 description="Using the `offset` with value `0` during the first request", 1197 title="Inject Offset", 1198 ) 1199 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters") 1200 1201 1202class PageIncrement(BaseModel): 1203 type: Literal["PageIncrement"] 1204 page_size: Optional[Union[int, str]] = Field( 1205 None, 1206 description="The number of records to include in each pages.", 1207 examples=[100, "100", "{{ config['page_size'] }}"], 1208 title="Page Size", 1209 ) 1210 start_from_page: Optional[int] = Field( 1211 0, 1212 description="Index of the first page to request.", 1213 examples=[0, 1], 1214 title="Start From Page", 1215 ) 1216 inject_on_first_request: Optional[bool] = Field( 1217 False, 1218 description="Using the `page number` with value defined by `start_from_page` during the first request", 1219 title="Inject Page Number", 1220 ) 1221 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters") 1222 1223 1224class PrimaryKey(BaseModel): 1225 __root__: Union[str, List[str], List[List[str]]] = Field( 1226 ..., 1227 description="The stream field to be used to distinguish unique records. Can either be a single field, an array of fields representing a composite key, or an array of arrays representing a composite key where the fields are nested fields.", 1228 examples=["id", ["code", "type"]], 1229 title="Primary Key", 1230 ) 1231 1232 1233class PropertyLimitType(Enum): 1234 characters = "characters" 1235 property_count = "property_count" 1236 1237 1238class PropertyChunking(BaseModel): 1239 type: Literal["PropertyChunking"] 1240 property_limit_type: PropertyLimitType = Field( 1241 ..., 1242 description="The type used to determine the maximum number of properties per chunk", 1243 title="Property Limit Type", 1244 ) 1245 property_limit: Optional[int] = Field( 1246 None, 1247 description="The maximum amount of properties that can be retrieved per request according to the limit type.", 1248 title="Property Limit", 1249 ) 1250 record_merge_strategy: Optional[GroupByKeyMergeStrategy] = Field( 1251 None, 1252 description="Dictates how to records that require multiple requests to get all properties should be emitted to the destination", 1253 title="Record Merge Strategy", 1254 ) 1255 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters") 1256 1257 1258class RecordFilter(BaseModel): 1259 type: Literal["RecordFilter"] 1260 condition: Optional[str] = Field( 1261 "", 1262 description="The predicate to filter a record. Records will be removed if evaluated to False.", 1263 examples=[ 1264 "{{ record['created_at'] >= stream_interval['start_time'] }}", 1265 "{{ record.status in ['active', 'expired'] }}", 1266 ], 1267 title="Condition", 1268 ) 1269 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters") 1270 1271 1272class SchemaNormalization(Enum): 1273 Default = "Default" 1274 None_ = "None" 1275 1276 1277class RemoveFields(BaseModel): 1278 type: Literal["RemoveFields"] 1279 condition: Optional[str] = Field( 1280 "", 1281 description="The predicate to filter a property by a property value. Property will be removed if it is empty OR expression is evaluated to True.,", 1282 examples=[ 1283 "{{ property|string == '' }}", 1284 "{{ property is integer }}", 1285 "{{ property|length > 5 }}", 1286 "{{ property == 'some_string_to_match' }}", 1287 ], 1288 ) 1289 field_pointers: List[List[str]] = Field( 1290 ..., 1291 description="Array of paths defining the field to remove. Each item is an array whose field describe the path of a field to remove.", 1292 examples=[["tags"], [["content", "html"], ["content", "plain_text"]]], 1293 title="Field Paths", 1294 ) 1295 1296 1297class RequestPath(BaseModel): 1298 type: Literal["RequestPath"] 1299 1300 1301class InjectInto(Enum): 1302 request_parameter = "request_parameter" 1303 header = "header" 1304 body_data = "body_data" 1305 body_json = "body_json" 1306 1307 1308class RequestOption(BaseModel): 1309 type: Literal["RequestOption"] 1310 field_name: Optional[str] = Field( 1311 None, 1312 description="Configures which key should be used in the location that the descriptor is being injected into. We hope to eventually deprecate this field in favor of `field_path` for all request_options, but must currently maintain it for backwards compatibility in the Builder.", 1313 examples=["segment_id"], 1314 title="Field Name", 1315 ) 1316 field_path: Optional[List[str]] = Field( 1317 None, 1318 description="Configures a path to be used for nested structures in JSON body requests (e.g. GraphQL queries)", 1319 examples=[["data", "viewer", "id"]], 1320 title="Field Path", 1321 ) 1322 inject_into: InjectInto = Field( 1323 ..., 1324 description="Configures where the descriptor should be set on the HTTP requests. Note that request parameters that are already encoded in the URL path will not be duplicated.", 1325 examples=["request_parameter", "header", "body_data", "body_json"], 1326 title="Inject Into", 1327 ) 1328 1329 1330class Schemas(BaseModel): 1331 pass 1332 1333 class Config: 1334 extra = Extra.allow 1335 1336 1337class LegacySessionTokenAuthenticator(BaseModel): 1338 type: Literal["LegacySessionTokenAuthenticator"] 1339 header: str = Field( 1340 ..., 1341 description="The name of the session token header that will be injected in the request", 1342 examples=["X-Session"], 1343 title="Session Request Header", 1344 ) 1345 login_url: str = Field( 1346 ..., 1347 description="Path of the login URL (do not include the base URL)", 1348 examples=["session"], 1349 title="Login Path", 1350 ) 1351 session_token: Optional[str] = Field( 1352 None, 1353 description="Session token to use if using a pre-defined token. Not needed if authenticating with username + password pair", 1354 example=["{{ config['session_token'] }}"], 1355 title="Session Token", 1356 ) 1357 session_token_response_key: str = Field( 1358 ..., 1359 description="Name of the key of the session token to be extracted from the response", 1360 examples=["id"], 1361 title="Response Token Response Key", 1362 ) 1363 username: Optional[str] = Field( 1364 None, 1365 description="Username used to authenticate and obtain a session token", 1366 examples=[" {{ config['username'] }}"], 1367 title="Username", 1368 ) 1369 password: Optional[str] = Field( 1370 "", 1371 description="Password used to authenticate and obtain a session token", 1372 examples=["{{ config['password'] }}", ""], 1373 title="Password", 1374 ) 1375 validate_session_url: str = Field( 1376 ..., 1377 description="Path of the URL to use to validate that the session token is valid (do not include the base URL)", 1378 examples=["user/current"], 1379 title="Validate Session Path", 1380 ) 1381 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters") 1382 1383 1384class CsvDecoder(BaseModel): 1385 type: Literal["CsvDecoder"] 1386 encoding: Optional[str] = "utf-8" 1387 delimiter: Optional[str] = "," 1388 set_values_to_none: Optional[List[str]] = None 1389 1390 1391class AsyncJobStatusMap(BaseModel): 1392 type: Optional[Literal["AsyncJobStatusMap"]] = None 1393 running: List[str] 1394 completed: List[str] 1395 failed: List[str] 1396 timeout: List[str] 1397 1398 1399class ValueType(Enum): 1400 string = "string" 1401 number = "number" 1402 integer = "integer" 1403 boolean = "boolean" 1404 1405 1406class WaitTimeFromHeader(BaseModel): 1407 type: Literal["WaitTimeFromHeader"] 1408 header: str = Field( 1409 ..., 1410 description="The name of the response header defining how long to wait before retrying.", 1411 examples=["Retry-After"], 1412 title="Response Header Name", 1413 ) 1414 regex: Optional[str] = Field( 1415 None, 1416 description="Optional regex to apply on the header to extract its value. The regex should define a capture group defining the wait time.", 1417 examples=["([-+]?\\d+)"], 1418 title="Extraction Regex", 1419 ) 1420 max_waiting_time_in_seconds: Optional[float] = Field( 1421 None, 1422 description="Given the value extracted from the header is greater than this value, stop the stream.", 1423 examples=[3600], 1424 title="Max Waiting Time in Seconds", 1425 ) 1426 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters") 1427 1428 1429class WaitUntilTimeFromHeader(BaseModel): 1430 type: Literal["WaitUntilTimeFromHeader"] 1431 header: str = Field( 1432 ..., 1433 description="The name of the response header defining how long to wait before retrying.", 1434 examples=["wait_time"], 1435 title="Response Header", 1436 ) 1437 min_wait: Optional[Union[float, str]] = Field( 1438 None, 1439 description="Minimum time to wait before retrying.", 1440 examples=[10, "60"], 1441 title="Minimum Wait Time", 1442 ) 1443 regex: Optional[str] = Field( 1444 None, 1445 description="Optional regex to apply on the header to extract its value. The regex should define a capture group defining the wait time.", 1446 examples=["([-+]?\\d+)"], 1447 title="Extraction Regex", 1448 ) 1449 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters") 1450 1451 1452class ComponentMappingDefinition(BaseModel): 1453 type: Literal["ComponentMappingDefinition"] 1454 field_path: List[str] = Field( 1455 ..., 1456 description="A list of potentially nested fields indicating the full path where value will be added or updated.", 1457 examples=[ 1458 ["data"], 1459 ["data", "records"], 1460 ["data", 1, "name"], 1461 ["data", "{{ components_values.name }}"], 1462 ["data", "*", "record"], 1463 ["*", "**", "name"], 1464 ], 1465 title="Field Path", 1466 ) 1467 value: str = Field( 1468 ..., 1469 description="The dynamic or static value to assign to the key. Interpolated values can be used to dynamically determine the value during runtime.", 1470 examples=[ 1471 "{{ components_values['updates'] }}", 1472 "{{ components_values['MetaData']['LastUpdatedTime'] }}", 1473 "{{ config['segment_id'] }}", 1474 "{{ stream_slice['parent_id'] }}", 1475 "{{ stream_slice['extra_fields']['name'] }}", 1476 ], 1477 title="Value", 1478 ) 1479 value_type: Optional[ValueType] = Field( 1480 None, 1481 description="The expected data type of the value. If omitted, the type will be inferred from the value provided.", 1482 title="Value Type", 1483 ) 1484 create_or_update: Optional[bool] = Field( 1485 False, 1486 description="Determines whether to create a new path if it doesn't exist (true) or only update existing paths (false). When set to true, the resolver will create new paths in the stream template if they don't exist. When false (default), it will only update existing paths.", 1487 title="Create or Update", 1488 ) 1489 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters") 1490 1491 1492class StreamConfig(BaseModel): 1493 type: Literal["StreamConfig"] 1494 configs_pointer: List[str] = Field( 1495 ..., 1496 description="A list of potentially nested fields indicating the full path in source config file where streams configs located.", 1497 examples=[["data"], ["data", "streams"], ["data", "{{ parameters.name }}"]], 1498 title="Configs Pointer", 1499 ) 1500 default_values: Optional[List[Dict[str, Any]]] = Field( 1501 None, 1502 description="A list of default values, each matching the structure expected from the parsed component value.", 1503 title="Default Values", 1504 ) 1505 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters") 1506 1507 1508class ConfigComponentsResolver(BaseModel): 1509 type: Literal["ConfigComponentsResolver"] 1510 stream_config: Union[List[StreamConfig], StreamConfig] 1511 components_mapping: List[ComponentMappingDefinition] 1512 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters") 1513 1514 1515class StreamParametersDefinition(BaseModel): 1516 type: Literal["StreamParametersDefinition"] 1517 list_of_parameters_for_stream: List[Dict[str, Any]] = Field( 1518 ..., 1519 description="A list of object of parameters for stream, each object in the list represents params for one stream.", 1520 examples=[ 1521 [ 1522 { 1523 "name": "test stream", 1524 "$parameters": {"entity": "test entity"}, 1525 "primary_key": "test key", 1526 } 1527 ] 1528 ], 1529 title="Stream Parameters", 1530 ) 1531 1532 1533class ParametrizedComponentsResolver(BaseModel): 1534 type: Literal["ParametrizedComponentsResolver"] 1535 stream_parameters: StreamParametersDefinition 1536 components_mapping: List[ComponentMappingDefinition] 1537 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters") 1538 1539 1540class RequestBodyPlainText(BaseModel): 1541 type: Literal["RequestBodyPlainText"] 1542 value: str 1543 1544 1545class RequestBodyUrlEncodedForm(BaseModel): 1546 type: Literal["RequestBodyUrlEncodedForm"] 1547 value: Dict[str, str] 1548 1549 1550class RequestBodyJsonObject(BaseModel): 1551 type: Literal["RequestBodyJsonObject"] 1552 value: Dict[str, Any] 1553 1554 1555class RequestBodyGraphQlQuery(BaseModel): 1556 class Config: 1557 extra = Extra.allow 1558 1559 query: str = Field(..., description="The GraphQL query to be executed") 1560 1561 1562class ValidateAdheresToSchema(BaseModel): 1563 type: Literal["ValidateAdheresToSchema"] 1564 base_schema: Union[str, Dict[str, Any]] = Field( 1565 ..., 1566 description="The base JSON schema against which the user-provided schema will be validated.", 1567 examples=[ 1568 "{{ config['report_validation_schema'] }}", 1569 '\'{\n "$schema": "http://json-schema.org/draft-07/schema#",\n "title": "Person",\n "type": "object",\n "properties": {\n "name": {\n "type": "string",\n "description": "The person\'s name"\n },\n "age": {\n "type": "integer",\n "minimum": 0,\n "description": "The person\'s age"\n }\n },\n "required": ["name", "age"]\n}\'\n', 1570 { 1571 "$schema": "http://json-schema.org/draft-07/schema#", 1572 "title": "Person", 1573 "type": "object", 1574 "properties": { 1575 "name": {"type": "string", "description": "The person's name"}, 1576 "age": { 1577 "type": "integer", 1578 "minimum": 0, 1579 "description": "The person's age", 1580 }, 1581 }, 1582 "required": ["name", "age"], 1583 }, 1584 ], 1585 title="Base JSON Schema", 1586 ) 1587 1588 1589class CustomValidationStrategy(BaseModel): 1590 class Config: 1591 extra = Extra.allow 1592 1593 type: Literal["CustomValidationStrategy"] 1594 class_name: str = Field( 1595 ..., 1596 description="Fully-qualified name of the class that will be implementing the custom validation strategy. Has to be a sub class of ValidationStrategy. The format is `source_<name>.<package>.<class_name>`.", 1597 examples=["source_declarative_manifest.components.MyCustomValidationStrategy"], 1598 title="Class Name", 1599 ) 1600 1601 1602class ConfigRemapField(BaseModel): 1603 type: Literal["ConfigRemapField"] 1604 map: Union[Dict[str, Any], str] = Field( 1605 ..., 1606 description="A mapping of original values to new values. When a field value matches a key in this map, it will be replaced with the corresponding value.", 1607 examples=[ 1608 {"pending": "in_progress", "done": "completed", "cancelled": "terminated"}, 1609 "{{ config['status_mapping'] }}", 1610 ], 1611 title="Value Mapping", 1612 ) 1613 field_path: List[str] = Field( 1614 ..., 1615 description="The path to the field whose value should be remapped. Specified as a list of path components to navigate through nested objects.", 1616 examples=[ 1617 ["status"], 1618 ["data", "status"], 1619 ["data", "{{ config.name }}", "status"], 1620 ["data", "*", "status"], 1621 ], 1622 title="Field Path", 1623 ) 1624 1625 1626class ConfigRemoveFields(BaseModel): 1627 type: Literal["ConfigRemoveFields"] 1628 field_pointers: List[List[str]] = Field( 1629 ..., 1630 description="A list of field pointers to be removed from the config.", 1631 examples=[["tags"], [["content", "html"], ["content", "plain_text"]]], 1632 title="Field Pointers", 1633 ) 1634 condition: Optional[str] = Field( 1635 "", 1636 description="Fields will be removed if expression is evaluated to True.", 1637 examples=[ 1638 "{{ config['environemnt'] == 'sandbox' }}", 1639 "{{ property is integer }}", 1640 "{{ property|length > 5 }}", 1641 "{{ property == 'some_string_to_match' }}", 1642 ], 1643 ) 1644 1645 1646class AddedFieldDefinition(BaseModel): 1647 type: Literal["AddedFieldDefinition"] 1648 path: List[str] = Field( 1649 ..., 1650 description="List of strings defining the path where to add the value on the record.", 1651 examples=[["segment_id"], ["metadata", "segment_id"]], 1652 title="Path", 1653 ) 1654 value: str = Field( 1655 ..., 1656 description="Value of the new field. Use {{ record['existing_field'] }} syntax to refer to other fields in the record.", 1657 examples=[ 1658 "{{ record['updates'] }}", 1659 "{{ record['MetaData']['LastUpdatedTime'] }}", 1660 "{{ stream_partition['segment_id'] }}", 1661 ], 1662 title="Value", 1663 ) 1664 value_type: Optional[ValueType] = Field( 1665 None, 1666 description="Type of the value. If not specified, the type will be inferred from the value.", 1667 title="Value Type", 1668 ) 1669 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters") 1670 1671 1672class AddFields(BaseModel): 1673 type: Literal["AddFields"] 1674 fields: List[AddedFieldDefinition] = Field( 1675 ..., 1676 description="List of transformations (path and corresponding value) that will be added to the record.", 1677 title="Fields", 1678 ) 1679 condition: Optional[str] = Field( 1680 "", 1681 description="Fields will be added if expression is evaluated to True.", 1682 examples=[ 1683 "{{ property|string == '' }}", 1684 "{{ property is integer }}", 1685 "{{ property|length > 5 }}", 1686 "{{ property == 'some_string_to_match' }}", 1687 ], 1688 ) 1689 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters") 1690 1691 1692class ApiKeyAuthenticator(BaseModel): 1693 type: Literal["ApiKeyAuthenticator"] 1694 api_token: Optional[str] = Field( 1695 None, 1696 description="The API key to inject in the request. Fill it in the user inputs.", 1697 examples=["{{ config['api_key'] }}", "Token token={{ config['api_key'] }}"], 1698 title="API Key", 1699 ) 1700 header: Optional[str] = Field( 1701 None, 1702 description="The name of the HTTP header that will be set to the API key. This setting is deprecated, use inject_into instead. Header and inject_into can not be defined at the same time.", 1703 examples=["Authorization", "Api-Token", "X-Auth-Token"], 1704 title="Header Name", 1705 ) 1706 inject_into: Optional[RequestOption] = Field( 1707 None, 1708 description="Configure how the API Key will be sent in requests to the source API. Either inject_into or header has to be defined.", 1709 examples=[ 1710 {"inject_into": "header", "field_name": "Authorization"}, 1711 {"inject_into": "request_parameter", "field_name": "authKey"}, 1712 ], 1713 title="Inject API Key Into Outgoing HTTP Request", 1714 ) 1715 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters") 1716 1717 1718class AuthFlow(BaseModel): 1719 auth_flow_type: Optional[AuthFlowType] = Field( 1720 None, description="The type of auth to use", title="Auth flow type" 1721 ) 1722 predicate_key: Optional[List[str]] = Field( 1723 None, 1724 description="JSON path to a field in the connectorSpecification that should exist for the advanced auth to be applicable.", 1725 examples=[["credentials", "auth_type"]], 1726 title="Predicate key", 1727 ) 1728 predicate_value: Optional[str] = Field( 1729 None, 1730 description="Value of the predicate_key fields for the advanced auth to be applicable.", 1731 examples=["Oauth"], 1732 title="Predicate value", 1733 ) 1734 oauth_config_specification: Optional[OAuthConfigSpecification] = None 1735 1736 1737class CheckStream(BaseModel): 1738 type: Literal["CheckStream"] 1739 stream_names: Optional[List[str]] = Field( 1740 None, 1741 description="Names of the streams to try reading from when running a check operation.", 1742 examples=[["users"], ["users", "contacts"]], 1743 title="Stream Names", 1744 ) 1745 dynamic_streams_check_configs: Optional[List[DynamicStreamCheckConfig]] = None 1746 1747 1748class IncrementingCountCursor(BaseModel): 1749 type: Literal["IncrementingCountCursor"] 1750 cursor_field: str = Field( 1751 ..., 1752 description="The location of the value on a record that will be used as a bookmark during sync. To ensure no data loss, the API must return records in ascending order based on the cursor field. Nested fields are not supported, so the field must be at the top level of the record. You can use a combination of Add Field and Remove Field transformations to move the nested field to the top.", 1753 examples=["created_at", "{{ config['record_cursor'] }}"], 1754 title="Cursor Field", 1755 ) 1756 start_value: Optional[Union[str, int]] = Field( 1757 None, 1758 description="The value that determines the earliest record that should be synced.", 1759 examples=[0, "{{ config['start_value'] }}"], 1760 title="Start Value", 1761 ) 1762 start_value_option: Optional[RequestOption] = Field( 1763 None, 1764 description="Optionally configures how the start value will be sent in requests to the source API.", 1765 title="Inject Start Value Into Outgoing HTTP Request", 1766 ) 1767 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters") 1768 1769 1770class DatetimeBasedCursor(BaseModel): 1771 type: Literal["DatetimeBasedCursor"] 1772 clamping: Optional[Clamping] = Field( 1773 None, 1774 description="This option is used to adjust the upper and lower boundaries of each datetime window to beginning and end of the provided target period (day, week, month)", 1775 title="Date Range Clamping", 1776 ) 1777 cursor_field: str = Field( 1778 ..., 1779 description="The location of the value on a record that will be used as a bookmark during sync. To ensure no data loss, the API must return records in ascending order based on the cursor field. Nested fields are not supported, so the field must be at the top level of the record. You can use a combination of Add Field and Remove Field transformations to move the nested field to the top.", 1780 examples=["created_at", "{{ config['record_cursor'] }}"], 1781 title="Cursor Field", 1782 ) 1783 cursor_datetime_formats: Optional[List[str]] = Field( 1784 None, 1785 description="The possible formats for the cursor field, in order of preference. The first format that matches the cursor field value will be used to parse it. If not provided, the `datetime_format` will be used.", 1786 title="Cursor Datetime Formats", 1787 ) 1788 start_datetime: Union[MinMaxDatetime, str] = Field( 1789 ..., 1790 description="The datetime that determines the earliest record that should be synced.", 1791 examples=["2020-01-1T00:00:00Z", "{{ config['start_time'] }}"], 1792 title="Start Datetime", 1793 ) 1794 start_time_option: Optional[RequestOption] = Field( 1795 None, 1796 description="Optionally configures how the start datetime will be sent in requests to the source API.", 1797 title="Inject Start Time Into Outgoing HTTP Request", 1798 ) 1799 end_datetime: Optional[Union[MinMaxDatetime, str]] = Field( 1800 None, 1801 description="The datetime that determines the last record that should be synced. If not provided, `{{ now_utc() }}` will be used.", 1802 examples=["2021-01-1T00:00:00Z", "{{ now_utc() }}", "{{ day_delta(-1) }}"], 1803 title="End Datetime", 1804 ) 1805 end_time_option: Optional[RequestOption] = Field( 1806 None, 1807 description="Optionally configures how the end datetime will be sent in requests to the source API.", 1808 title="Inject End Time Into Outgoing HTTP Request", 1809 ) 1810 datetime_format: str = Field( 1811 ..., 1812 description="The datetime format used to format the datetime values that are sent in outgoing requests to the API. Use placeholders starting with \"%\" to describe the format the API is using. The following placeholders are available:\n * **%s**: Epoch unix timestamp - `1686218963`\n * **%s_as_float**: Epoch unix timestamp in seconds as float with microsecond precision - `1686218963.123456`\n * **%ms**: Epoch unix timestamp (milliseconds) - `1686218963123`\n * **%a**: Weekday (abbreviated) - `Sun`\n * **%A**: Weekday (full) - `Sunday`\n * **%w**: Weekday (decimal) - `0` (Sunday), `6` (Saturday)\n * **%d**: Day of the month (zero-padded) - `01`, `02`, ..., `31`\n * **%b**: Month (abbreviated) - `Jan`\n * **%B**: Month (full) - `January`\n * **%m**: Month (zero-padded) - `01`, `02`, ..., `12`\n * **%y**: Year (without century, zero-padded) - `00`, `01`, ..., `99`\n * **%Y**: Year (with century) - `0001`, `0002`, ..., `9999`\n * **%H**: Hour (24-hour, zero-padded) - `00`, `01`, ..., `23`\n * **%I**: Hour (12-hour, zero-padded) - `01`, `02`, ..., `12`\n * **%p**: AM/PM indicator\n * **%M**: Minute (zero-padded) - `00`, `01`, ..., `59`\n * **%S**: Second (zero-padded) - `00`, `01`, ..., `59`\n * **%f**: Microsecond (zero-padded to 6 digits) - `000000`\n * **%_ms**: Millisecond (zero-padded to 3 digits) - `000`\n * **%z**: UTC offset - `(empty)`, `+0000`, `-04:00`\n * **%Z**: Time zone name - `(empty)`, `UTC`, `GMT`\n * **%j**: Day of the year (zero-padded) - `001`, `002`, ..., `366`\n * **%U**: Week number of the year (starting Sunday) - `00`, ..., `53`\n * **%W**: Week number of the year (starting Monday) - `00`, ..., `53`\n * **%c**: Date and time - `Tue Aug 16 21:30:00 1988`\n * **%x**: Date standard format - `08/16/1988`\n * **%X**: Time standard format - `21:30:00`\n * **%%**: Literal '%' character\n\n Some placeholders depend on the locale of the underlying system - in most cases this locale is configured as en/US. For more information see the [Python documentation](https://docs.python.org/3/library/datetime.html#strftime-and-strptime-format-codes).\n", 1813 examples=["%Y-%m-%dT%H:%M:%S.%f%z", "%Y-%m-%d", "%s", "%ms", "%s_as_float"], 1814 title="Outgoing Datetime Format", 1815 ) 1816 cursor_granularity: Optional[str] = Field( 1817 None, 1818 description="Smallest increment the datetime_format has (ISO 8601 duration) that is used to ensure the start of a slice does not overlap with the end of the previous one, e.g. for %Y-%m-%d the granularity should be P1D, for %Y-%m-%dT%H:%M:%SZ the granularity should be PT1S. Given this field is provided, `step` needs to be provided as well.", 1819 examples=["PT1S"], 1820 title="Cursor Granularity", 1821 ) 1822 is_data_feed: Optional[bool] = Field( 1823 None, 1824 description="A data feed API is an API that does not allow filtering and paginates the content from the most recent to the least recent. Given this, the CDK needs to know when to stop paginating and this field will generate a stop condition for pagination.", 1825 title="Whether the target API is formatted as a data feed", 1826 ) 1827 is_client_side_incremental: Optional[bool] = Field( 1828 None, 1829 description="If the target API endpoint does not take cursor values to filter records and returns all records anyway, the connector with this cursor will filter out records locally, and only emit new records from the last sync, hence incremental. This means that all records would be read from the API, but only new records will be emitted to the destination.", 1830 title="Whether the target API does not support filtering and returns all data (the cursor filters records in the client instead of the API side)", 1831 ) 1832 is_compare_strictly: Optional[bool] = Field( 1833 False, 1834 description="Set to True if the target API does not accept queries where the start time equal the end time.", 1835 title="Whether to skip requests if the start time equals the end time", 1836 ) 1837 global_substream_cursor: Optional[bool] = Field( 1838 False, 1839 description="This setting optimizes performance when the parent stream has thousands of partitions by storing the cursor as a single value rather than per partition. Notably, the substream state is updated only at the end of the sync, which helps prevent data loss in case of a sync failure. See more info in the [docs](https://docs.airbyte.com/connector-development/config-based/understanding-the-yaml-file/incremental-syncs).", 1840 title="Whether to store cursor as one value instead of per partition", 1841 ) 1842 lookback_window: Optional[str] = Field( 1843 None, 1844 description="Time interval before the start_datetime to read data for, e.g. P1M for looking back one month.", 1845 examples=["P1D", "P{{ config['lookback_days'] }}D"], 1846 title="Lookback Window", 1847 ) 1848 partition_field_end: Optional[str] = Field( 1849 None, 1850 description="Name of the partition start time field.", 1851 examples=["ending_time"], 1852 title="Partition Field End", 1853 ) 1854 partition_field_start: Optional[str] = Field( 1855 None, 1856 description="Name of the partition end time field.", 1857 examples=["starting_time"], 1858 title="Partition Field Start", 1859 ) 1860 step: Optional[str] = Field( 1861 None, 1862 description="The size of the time window (ISO8601 duration). Given this field is provided, `cursor_granularity` needs to be provided as well.", 1863 examples=["P1W", "{{ config['step_increment'] }}"], 1864 title="Step", 1865 ) 1866 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters") 1867 1868 1869class FixedWindowCallRatePolicy(BaseModel): 1870 class Config: 1871 extra = Extra.allow 1872 1873 type: Literal["FixedWindowCallRatePolicy"] 1874 period: str = Field( 1875 ..., description="The time interval for the rate limit window.", title="Period" 1876 ) 1877 call_limit: int = Field( 1878 ..., 1879 description="The maximum number of calls allowed within the period.", 1880 title="Call Limit", 1881 ) 1882 matchers: List[HttpRequestRegexMatcher] = Field( 1883 ..., 1884 description="List of matchers that define which requests this policy applies to.", 1885 title="Matchers", 1886 ) 1887 1888 1889class MovingWindowCallRatePolicy(BaseModel): 1890 class Config: 1891 extra = Extra.allow 1892 1893 type: Literal["MovingWindowCallRatePolicy"] 1894 rates: List[Rate] = Field( 1895 ..., 1896 description="List of rates that define the call limits for different time intervals.", 1897 title="Rates", 1898 ) 1899 matchers: List[HttpRequestRegexMatcher] = Field( 1900 ..., 1901 description="List of matchers that define which requests this policy applies to.", 1902 title="Matchers", 1903 ) 1904 1905 1906class UnlimitedCallRatePolicy(BaseModel): 1907 class Config: 1908 extra = Extra.allow 1909 1910 type: Literal["UnlimitedCallRatePolicy"] 1911 matchers: List[HttpRequestRegexMatcher] = Field( 1912 ..., 1913 description="List of matchers that define which requests this policy applies to.", 1914 title="Matchers", 1915 ) 1916 1917 1918class DefaultErrorHandler(BaseModel): 1919 type: Literal["DefaultErrorHandler"] 1920 backoff_strategies: Optional[ 1921 List[ 1922 Union[ 1923 ConstantBackoffStrategy, 1924 ExponentialBackoffStrategy, 1925 WaitTimeFromHeader, 1926 WaitUntilTimeFromHeader, 1927 CustomBackoffStrategy, 1928 ] 1929 ] 1930 ] = Field( 1931 None, 1932 description="List of backoff strategies to use to determine how long to wait before retrying a retryable request.", 1933 title="Backoff Strategies", 1934 ) 1935 max_retries: Optional[int] = Field( 1936 5, 1937 description="The maximum number of time to retry a retryable request before giving up and failing.", 1938 examples=[5, 0, 10], 1939 title="Max Retry Count", 1940 ) 1941 response_filters: Optional[List[HttpResponseFilter]] = Field( 1942 None, 1943 description="List of response filters to iterate on when deciding how to handle an error. When using an array of multiple filters, the filters will be applied sequentially and the response will be selected if it matches any of the filter's predicate.", 1944 title="Response Filters", 1945 ) 1946 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters") 1947 1948 1949class DefaultPaginator(BaseModel): 1950 type: Literal["DefaultPaginator"] 1951 pagination_strategy: Union[ 1952 PageIncrement, OffsetIncrement, CursorPagination, CustomPaginationStrategy 1953 ] = Field( 1954 ..., 1955 description="Strategy defining how records are paginated.", 1956 title="Pagination Strategy", 1957 ) 1958 page_size_option: Optional[RequestOption] = Field( 1959 None, title="Inject Page Size Into Outgoing HTTP Request" 1960 ) 1961 page_token_option: Optional[Union[RequestOption, RequestPath]] = Field( 1962 None, title="Inject Page Token Into Outgoing HTTP Request" 1963 ) 1964 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters") 1965 1966 1967class SessionTokenRequestApiKeyAuthenticator(BaseModel): 1968 type: Literal["ApiKey"] 1969 inject_into: RequestOption = Field( 1970 ..., 1971 description="Configure how the API Key will be sent in requests to the source API.", 1972 examples=[ 1973 {"inject_into": "header", "field_name": "Authorization"}, 1974 {"inject_into": "request_parameter", "field_name": "authKey"}, 1975 ], 1976 title="Inject API Key Into Outgoing HTTP Request", 1977 ) 1978 1979 1980class ListPartitionRouter(BaseModel): 1981 type: Literal["ListPartitionRouter"] 1982 cursor_field: str = Field( 1983 ..., 1984 description='While iterating over list values, the name of field used to reference a list value. The partition value can be accessed with string interpolation. e.g. "{{ stream_partition[\'my_key\'] }}" where "my_key" is the value of the cursor_field.', 1985 examples=["section", "{{ config['section_key'] }}"], 1986 title="Current Partition Value Identifier", 1987 ) 1988 values: Union[str, List[str]] = Field( 1989 ..., 1990 description="The list of attributes being iterated over and used as input for the requests made to the source API.", 1991 examples=[["section_a", "section_b", "section_c"], "{{ config['sections'] }}"], 1992 title="Partition Values", 1993 ) 1994 request_option: Optional[RequestOption] = Field( 1995 None, 1996 description="A request option describing where the list value should be injected into and under what field name if applicable.", 1997 title="Inject Partition Value Into Outgoing HTTP Request", 1998 ) 1999 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters") 2000 2001 2002class RecordSelector(BaseModel): 2003 type: Literal["RecordSelector"] 2004 extractor: Union[DpathExtractor, CustomRecordExtractor] 2005 record_filter: Optional[Union[RecordFilter, CustomRecordFilter]] = Field( 2006 None, 2007 description="Responsible for filtering records to be emitted by the Source.", 2008 title="Record Filter", 2009 ) 2010 schema_normalization: Optional[Union[SchemaNormalization, CustomSchemaNormalization]] = Field( 2011 None, 2012 description="Responsible for normalization according to the schema.", 2013 title="Schema Normalization", 2014 ) 2015 transform_before_filtering: Optional[bool] = Field( 2016 None, 2017 description="If true, transformation will be applied before record filtering.", 2018 title="Transform Before Filtering", 2019 ) 2020 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters") 2021 2022 2023class GzipDecoder(BaseModel): 2024 type: Literal["GzipDecoder"] 2025 decoder: Union[CsvDecoder, GzipDecoder, JsonDecoder, JsonlDecoder] 2026 2027 2028class RequestBodyGraphQL(BaseModel): 2029 type: Literal["RequestBodyGraphQL"] 2030 value: RequestBodyGraphQlQuery 2031 2032 2033class DpathValidator(BaseModel): 2034 type: Literal["DpathValidator"] 2035 field_path: List[str] = Field( 2036 ..., 2037 description='List of potentially nested fields describing the full path of the field to validate. Use "*" to validate all values from an array.', 2038 examples=[ 2039 ["data"], 2040 ["data", "records"], 2041 ["data", "{{ parameters.name }}"], 2042 ["data", "*", "record"], 2043 ], 2044 title="Field Path", 2045 ) 2046 validation_strategy: Union[ValidateAdheresToSchema, CustomValidationStrategy] = Field( 2047 ..., 2048 description="The condition that the specified config value will be evaluated against", 2049 title="Validation Strategy", 2050 ) 2051 2052 2053class PredicateValidator(BaseModel): 2054 type: Literal["PredicateValidator"] 2055 value: Optional[Union[str, float, Dict[str, Any], List[Any], bool]] = Field( 2056 ..., 2057 description="The value to be validated. Can be a literal value or interpolated from configuration.", 2058 examples=[ 2059 "test-value", 2060 "{{ config['api_version'] }}", 2061 "{{ config['tenant_id'] }}", 2062 123, 2063 ], 2064 title="Value", 2065 ) 2066 validation_strategy: Union[ValidateAdheresToSchema, CustomValidationStrategy] = Field( 2067 ..., 2068 description="The validation strategy to apply to the value.", 2069 title="Validation Strategy", 2070 ) 2071 2072 2073class ConfigAddFields(BaseModel): 2074 type: Literal["ConfigAddFields"] 2075 fields: List[AddedFieldDefinition] = Field( 2076 ..., 2077 description="A list of transformations (path and corresponding value) that will be added to the config.", 2078 title="Fields", 2079 ) 2080 condition: Optional[str] = Field( 2081 "", 2082 description="Fields will be added if expression is evaluated to True.", 2083 examples=[ 2084 "{{ config['environemnt'] == 'sandbox' }}", 2085 "{{ property is integer }}", 2086 "{{ property|length > 5 }}", 2087 "{{ property == 'some_string_to_match' }}", 2088 ], 2089 ) 2090 2091 2092class CompositeErrorHandler(BaseModel): 2093 type: Literal["CompositeErrorHandler"] 2094 error_handlers: List[Union[CompositeErrorHandler, DefaultErrorHandler]] = Field( 2095 ..., 2096 description="List of error handlers to iterate on to determine how to handle a failed response.", 2097 title="Error Handlers", 2098 ) 2099 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters") 2100 2101 2102class HTTPAPIBudget(BaseModel): 2103 class Config: 2104 extra = Extra.allow 2105 2106 type: Literal["HTTPAPIBudget"] 2107 policies: List[ 2108 Union[ 2109 FixedWindowCallRatePolicy, 2110 MovingWindowCallRatePolicy, 2111 UnlimitedCallRatePolicy, 2112 ] 2113 ] = Field( 2114 ..., 2115 description="List of call rate policies that define how many calls are allowed.", 2116 title="Policies", 2117 ) 2118 ratelimit_reset_header: Optional[str] = Field( 2119 "ratelimit-reset", 2120 description="The HTTP response header name that indicates when the rate limit resets.", 2121 title="Rate Limit Reset Header", 2122 ) 2123 ratelimit_remaining_header: Optional[str] = Field( 2124 "ratelimit-remaining", 2125 description="The HTTP response header name that indicates the number of remaining allowed calls.", 2126 title="Rate Limit Remaining Header", 2127 ) 2128 status_codes_for_ratelimit_hit: Optional[List[int]] = Field( 2129 [429], 2130 description="List of HTTP status codes that indicate a rate limit has been hit.", 2131 title="Status Codes for Rate Limit Hit", 2132 ) 2133 2134 2135class ZipfileDecoder(BaseModel): 2136 class Config: 2137 extra = Extra.allow 2138 2139 type: Literal["ZipfileDecoder"] 2140 decoder: Union[CsvDecoder, GzipDecoder, JsonDecoder, JsonlDecoder] = Field( 2141 ..., 2142 description="Parser to parse the decompressed data from the zipfile(s).", 2143 title="Parser", 2144 ) 2145 2146 2147class ConfigMigration(BaseModel): 2148 type: Literal["ConfigMigration"] 2149 description: Optional[str] = Field( 2150 None, description="The description/purpose of the config migration." 2151 ) 2152 transformations: List[Union[ConfigRemapField, ConfigAddFields, ConfigRemoveFields]] = Field( 2153 ..., 2154 description="The list of transformations that will attempt to be applied on an incoming unmigrated config. The transformations will be applied in the order they are defined.", 2155 title="Transformations", 2156 ) 2157 2158 2159class ConfigNormalizationRules(BaseModel): 2160 class Config: 2161 extra = Extra.forbid 2162 2163 config_migrations: Optional[List[ConfigMigration]] = Field( 2164 [], 2165 description="The discrete migrations that will be applied on the incoming config. Each migration will be applied in the order they are defined.", 2166 title="Config Migrations", 2167 ) 2168 transformations: Optional[ 2169 List[Union[ConfigRemapField, ConfigAddFields, ConfigRemoveFields]] 2170 ] = Field( 2171 [], 2172 description="The list of transformations that will be applied on the incoming config at the start of each sync. The transformations will be applied in the order they are defined.", 2173 title="Transformations", 2174 ) 2175 validations: Optional[List[Union[DpathValidator, PredicateValidator]]] = Field( 2176 [], 2177 description="The list of validations that will be performed on the incoming config at the start of each sync.", 2178 title="Validations", 2179 ) 2180 2181 2182class Spec(BaseModel): 2183 type: Literal["Spec"] 2184 connection_specification: Dict[str, Any] = Field( 2185 ..., 2186 description="A connection specification describing how a the connector can be configured.", 2187 title="Connection Specification", 2188 ) 2189 documentation_url: Optional[str] = Field( 2190 None, 2191 description="URL of the connector's documentation page.", 2192 examples=["https://docs.airbyte.com/integrations/sources/dremio"], 2193 title="Documentation URL", 2194 ) 2195 advanced_auth: Optional[AuthFlow] = Field( 2196 None, 2197 description="Advanced specification for configuring the authentication flow.", 2198 title="Advanced Auth", 2199 ) 2200 config_normalization_rules: Optional[ConfigNormalizationRules] = Field( 2201 None, title="Config Normalization Rules" 2202 ) 2203 2204 2205class DeclarativeSource1(BaseModel): 2206 class Config: 2207 extra = Extra.forbid 2208 2209 type: Literal["DeclarativeSource"] 2210 check: Union[CheckStream, CheckDynamicStream] 2211 streams: List[Union[ConditionalStreams, DeclarativeStream, StateDelegatingStream]] 2212 dynamic_streams: Optional[List[DynamicDeclarativeStream]] = None 2213 version: str = Field( 2214 ..., 2215 description="The version of the Airbyte CDK used to build and test the source.", 2216 ) 2217 schemas: Optional[Schemas] = None 2218 definitions: Optional[Dict[str, Any]] = None 2219 spec: Optional[Spec] = None 2220 concurrency_level: Optional[ConcurrencyLevel] = None 2221 api_budget: Optional[HTTPAPIBudget] = None 2222 max_concurrent_async_job_count: Optional[Union[int, str]] = Field( 2223 None, 2224 description="Maximum number of concurrent asynchronous jobs to run. This property is only relevant for sources/streams that support asynchronous job execution through the AsyncRetriever (e.g. a report-based stream that initiates a job, polls the job status, and then fetches the job results). This is often set by the API's maximum number of concurrent jobs on the account level. Refer to the API's documentation for this information.", 2225 examples=[3, "{{ config['max_concurrent_async_job_count'] }}"], 2226 title="Maximum Concurrent Asynchronous Jobs", 2227 ) 2228 metadata: Optional[Dict[str, Any]] = Field( 2229 None, 2230 description="For internal Airbyte use only - DO NOT modify manually. Used by consumers of declarative manifests for storing related metadata.", 2231 ) 2232 description: Optional[str] = Field( 2233 None, 2234 description="A description of the connector. It will be presented on the Source documentation page.", 2235 ) 2236 2237 2238class DeclarativeSource2(BaseModel): 2239 class Config: 2240 extra = Extra.forbid 2241 2242 type: Literal["DeclarativeSource"] 2243 check: Union[CheckStream, CheckDynamicStream] 2244 streams: Optional[List[Union[ConditionalStreams, DeclarativeStream, StateDelegatingStream]]] = ( 2245 None 2246 ) 2247 dynamic_streams: List[DynamicDeclarativeStream] 2248 version: str = Field( 2249 ..., 2250 description="The version of the Airbyte CDK used to build and test the source.", 2251 ) 2252 schemas: Optional[Schemas] = None 2253 definitions: Optional[Dict[str, Any]] = None 2254 spec: Optional[Spec] = None 2255 concurrency_level: Optional[ConcurrencyLevel] = None 2256 api_budget: Optional[HTTPAPIBudget] = None 2257 max_concurrent_async_job_count: Optional[Union[int, str]] = Field( 2258 None, 2259 description="Maximum number of concurrent asynchronous jobs to run. This property is only relevant for sources/streams that support asynchronous job execution through the AsyncRetriever (e.g. a report-based stream that initiates a job, polls the job status, and then fetches the job results). This is often set by the API's maximum number of concurrent jobs on the account level. Refer to the API's documentation for this information.", 2260 examples=[3, "{{ config['max_concurrent_async_job_count'] }}"], 2261 title="Maximum Concurrent Asynchronous Jobs", 2262 ) 2263 metadata: Optional[Dict[str, Any]] = Field( 2264 None, 2265 description="For internal Airbyte use only - DO NOT modify manually. Used by consumers of declarative manifests for storing related metadata.", 2266 ) 2267 description: Optional[str] = Field( 2268 None, 2269 description="A description of the connector. It will be presented on the Source documentation page.", 2270 ) 2271 2272 2273class DeclarativeSource(BaseModel): 2274 class Config: 2275 extra = Extra.forbid 2276 2277 __root__: Union[DeclarativeSource1, DeclarativeSource2] = Field( 2278 ..., 2279 description="An API source that extracts data according to its declarative components.", 2280 title="DeclarativeSource", 2281 ) 2282 2283 2284class SelectiveAuthenticator(BaseModel): 2285 class Config: 2286 extra = Extra.allow 2287 2288 type: Literal["SelectiveAuthenticator"] 2289 authenticator_selection_path: List[str] = Field( 2290 ..., 2291 description="Path of the field in config with selected authenticator name", 2292 examples=[["auth"], ["auth", "type"]], 2293 title="Authenticator Selection Path", 2294 ) 2295 authenticators: Dict[ 2296 str, 2297 Union[ 2298 ApiKeyAuthenticator, 2299 BasicHttpAuthenticator, 2300 BearerAuthenticator, 2301 OAuthAuthenticator, 2302 JwtAuthenticator, 2303 SessionTokenAuthenticator, 2304 LegacySessionTokenAuthenticator, 2305 CustomAuthenticator, 2306 NoAuth, 2307 ], 2308 ] = Field( 2309 ..., 2310 description="Authenticators to select from.", 2311 examples=[ 2312 { 2313 "authenticators": { 2314 "token": "#/definitions/ApiKeyAuthenticator", 2315 "oauth": "#/definitions/OAuthAuthenticator", 2316 "jwt": "#/definitions/JwtAuthenticator", 2317 } 2318 } 2319 ], 2320 title="Authenticators", 2321 ) 2322 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters") 2323 2324 2325class ConditionalStreams(BaseModel): 2326 type: Literal["ConditionalStreams"] 2327 condition: str = Field( 2328 ..., 2329 description="Condition that will be evaluated to determine if a set of streams should be available.", 2330 examples=["{{ config['is_sandbox'] }}"], 2331 title="Condition", 2332 ) 2333 streams: List[DeclarativeStream] = Field( 2334 ..., 2335 description="Streams that will be used during an operation based on the condition.", 2336 title="Streams", 2337 ) 2338 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters") 2339 2340 2341class FileUploader(BaseModel): 2342 type: Literal["FileUploader"] 2343 requester: Union[HttpRequester, CustomRequester] = Field( 2344 ..., 2345 description="Requester component that describes how to prepare HTTP requests to send to the source API.", 2346 ) 2347 download_target_extractor: Union[DpathExtractor, CustomRecordExtractor] = Field( 2348 ..., 2349 description="Responsible for fetching the url where the file is located. This is applied on each records and not on the HTTP response", 2350 ) 2351 file_extractor: Optional[Union[DpathExtractor, CustomRecordExtractor]] = Field( 2352 None, 2353 description="Responsible for fetching the content of the file. If not defined, the assumption is that the whole response body is the file content", 2354 ) 2355 filename_extractor: Optional[str] = Field( 2356 None, 2357 description="Defines the name to store the file. Stream name is automatically added to the file path. File unique ID can be used to avoid overwriting files. Random UUID will be used if the extractor is not provided.", 2358 examples=[ 2359 "{{ record.id }}/{{ record.file_name }}/", 2360 "{{ record.id }}_{{ record.file_name }}/", 2361 ], 2362 ) 2363 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters") 2364 2365 2366class DeclarativeStream(BaseModel): 2367 class Config: 2368 extra = Extra.allow 2369 2370 type: Literal["DeclarativeStream"] 2371 name: Optional[str] = Field("", description="The stream name.", example=["Users"], title="Name") 2372 retriever: Union[SimpleRetriever, AsyncRetriever, CustomRetriever] = Field( 2373 ..., 2374 description="Component used to coordinate how records are extracted across stream slices and request pages.", 2375 title="Retriever", 2376 ) 2377 incremental_sync: Optional[ 2378 Union[DatetimeBasedCursor, IncrementingCountCursor, CustomIncrementalSync] 2379 ] = Field( 2380 None, 2381 description="Component used to fetch data incrementally based on a time field in the data.", 2382 title="Incremental Sync", 2383 ) 2384 primary_key: Optional[PrimaryKey] = Field("", title="Primary Key") 2385 schema_loader: Optional[ 2386 Union[ 2387 InlineSchemaLoader, 2388 DynamicSchemaLoader, 2389 JsonFileSchemaLoader, 2390 List[ 2391 Union[ 2392 InlineSchemaLoader, 2393 DynamicSchemaLoader, 2394 JsonFileSchemaLoader, 2395 CustomSchemaLoader, 2396 ] 2397 ], 2398 CustomSchemaLoader, 2399 ] 2400 ] = Field( 2401 None, 2402 description="One or many schema loaders can be used to retrieve the schema for the current stream. When multiple schema loaders are defined, schema properties will be merged together. Schema loaders defined first taking precedence in the event of a conflict.", 2403 title="Schema Loader", 2404 ) 2405 transformations: Optional[ 2406 List[ 2407 Union[ 2408 AddFields, 2409 RemoveFields, 2410 KeysToLower, 2411 KeysToSnakeCase, 2412 FlattenFields, 2413 DpathFlattenFields, 2414 KeysReplace, 2415 CustomTransformation, 2416 ] 2417 ] 2418 ] = Field( 2419 None, 2420 description="A list of transformations to be applied to each output record.", 2421 title="Transformations", 2422 ) 2423 state_migrations: Optional[ 2424 List[Union[LegacyToPerPartitionStateMigration, CustomStateMigration]] 2425 ] = Field( 2426 [], 2427 description="Array of state migrations to be applied on the input state", 2428 title="State Migrations", 2429 ) 2430 file_uploader: Optional[FileUploader] = Field( 2431 None, 2432 description="(experimental) Describes how to fetch a file", 2433 title="File Uploader", 2434 ) 2435 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters") 2436 2437 2438class SessionTokenAuthenticator(BaseModel): 2439 type: Literal["SessionTokenAuthenticator"] 2440 login_requester: HttpRequester = Field( 2441 ..., 2442 description="Description of the request to perform to obtain a session token to perform data requests. The response body is expected to be a JSON object with a session token property.", 2443 examples=[ 2444 { 2445 "type": "HttpRequester", 2446 "url_base": "https://my_api.com", 2447 "path": "/login", 2448 "authenticator": { 2449 "type": "BasicHttpAuthenticator", 2450 "username": "{{ config.username }}", 2451 "password": "{{ config.password }}", 2452 }, 2453 } 2454 ], 2455 title="Login Requester", 2456 ) 2457 session_token_path: List[str] = Field( 2458 ..., 2459 description="The path in the response body returned from the login requester to the session token.", 2460 examples=[["access_token"], ["result", "token"]], 2461 title="Session Token Path", 2462 ) 2463 expiration_duration: Optional[str] = Field( 2464 None, 2465 description="The duration in ISO 8601 duration notation after which the session token expires, starting from the time it was obtained. Omitting it will result in the session token being refreshed for every request.", 2466 examples=["PT1H", "P1D"], 2467 title="Expiration Duration", 2468 ) 2469 request_authentication: Union[ 2470 SessionTokenRequestApiKeyAuthenticator, SessionTokenRequestBearerAuthenticator 2471 ] = Field( 2472 ..., 2473 description="Authentication method to use for requests sent to the API, specifying how to inject the session token.", 2474 title="Data Request Authentication", 2475 ) 2476 decoder: Optional[Union[JsonDecoder, XmlDecoder]] = Field( 2477 None, description="Component used to decode the response.", title="Decoder" 2478 ) 2479 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters") 2480 2481 2482class HttpRequester(BaseModelWithDeprecations): 2483 type: Literal["HttpRequester"] 2484 url_base: Optional[str] = Field( 2485 None, 2486 deprecated=True, 2487 deprecation_message="Use `url` field instead.", 2488 description="Deprecated, use the `url` instead. Base URL of the API source. Do not put sensitive information (e.g. API tokens) into this field - Use the Authenticator component for this.", 2489 examples=[ 2490 "https://connect.squareup.com/v2", 2491 "{{ config['base_url'] or 'https://app.posthog.com'}}/api", 2492 "https://connect.squareup.com/v2/quotes/{{ stream_partition['id'] }}/quote_line_groups", 2493 "https://example.com/api/v1/resource/{{ next_page_token['id'] }}", 2494 ], 2495 title="API Base URL", 2496 ) 2497 url: Optional[str] = Field( 2498 None, 2499 description="The URL of the source API endpoint. Do not put sensitive information (e.g. API tokens) into this field - Use the Authenticator component for this.", 2500 examples=[ 2501 "https://connect.squareup.com/v2", 2502 "{{ config['url'] or 'https://app.posthog.com'}}/api", 2503 "https://connect.squareup.com/v2/quotes/{{ stream_partition['id'] }}/quote_line_groups", 2504 "https://example.com/api/v1/resource/{{ next_page_token['id'] }}", 2505 ], 2506 title="API Endpoint URL", 2507 ) 2508 path: Optional[str] = Field( 2509 None, 2510 deprecated=True, 2511 deprecation_message="Use `url` field instead.", 2512 description="Deprecated, use the `url` instead. Path the specific API endpoint that this stream represents. Do not put sensitive information (e.g. API tokens) into this field - Use the Authenticator component for this.", 2513 examples=[ 2514 "/products", 2515 "/quotes/{{ stream_partition['id'] }}/quote_line_groups", 2516 "/trades/{{ config['symbol_id'] }}/history", 2517 ], 2518 title="URL Path", 2519 ) 2520 http_method: Optional[HttpMethod] = Field( 2521 HttpMethod.GET, 2522 description="The HTTP method used to fetch data from the source (can be GET or POST).", 2523 examples=["GET", "POST"], 2524 title="HTTP Method", 2525 ) 2526 authenticator: Optional[ 2527 Union[ 2528 ApiKeyAuthenticator, 2529 BasicHttpAuthenticator, 2530 BearerAuthenticator, 2531 OAuthAuthenticator, 2532 JwtAuthenticator, 2533 SessionTokenAuthenticator, 2534 SelectiveAuthenticator, 2535 CustomAuthenticator, 2536 NoAuth, 2537 LegacySessionTokenAuthenticator, 2538 ] 2539 ] = Field( 2540 None, 2541 description="Authentication method to use for requests sent to the API.", 2542 title="Authenticator", 2543 ) 2544 fetch_properties_from_endpoint: Optional[PropertiesFromEndpoint] = Field( 2545 None, 2546 deprecated=True, 2547 deprecation_message="Use `query_properties` field instead.", 2548 description="Allows for retrieving a dynamic set of properties from an API endpoint which can be injected into outbound request using the stream_partition.extra_fields.", 2549 title="Fetch Properties from Endpoint", 2550 ) 2551 query_properties: Optional[QueryProperties] = Field( 2552 None, 2553 description="For APIs that require explicit specification of the properties to query for, this component will take a static or dynamic set of properties (which can be optionally split into chunks) and allow them to be injected into an outbound request by accessing stream_partition.extra_fields.", 2554 title="Query Properties", 2555 ) 2556 request_parameters: Optional[Union[Dict[str, Union[str, QueryProperties]], str]] = Field( 2557 None, 2558 description="Specifies the query parameters that should be set on an outgoing HTTP request given the inputs.", 2559 examples=[ 2560 {"unit": "day"}, 2561 { 2562 "query": 'last_event_time BETWEEN TIMESTAMP "{{ stream_interval.start_time }}" AND TIMESTAMP "{{ stream_interval.end_time }}"' 2563 }, 2564 {"searchIn": "{{ ','.join(config.get('search_in', [])) }}"}, 2565 {"sort_by[asc]": "updated_at"}, 2566 ], 2567 title="Query Parameters", 2568 ) 2569 request_headers: Optional[Union[Dict[str, str], str]] = Field( 2570 None, 2571 description="Return any non-auth headers. Authentication headers will overwrite any overlapping headers returned from this method.", 2572 examples=[{"Output-Format": "JSON"}, {"Version": "{{ config['version'] }}"}], 2573 title="Request Headers", 2574 ) 2575 request_body_data: Optional[Union[Dict[str, str], str]] = Field( 2576 None, 2577 deprecated=True, 2578 deprecation_message="Use `request_body` field instead.", 2579 description="Specifies how to populate the body of the request with a non-JSON payload. Plain text will be sent as is, whereas objects will be converted to a urlencoded form.", 2580 examples=[ 2581 '[{"clause": {"type": "timestamp", "operator": 10, "parameters":\n [{"value": {{ stream_interval[\'start_time\'] | int * 1000 }} }]\n }, "orderBy": 1, "columnName": "Timestamp"}]/\n' 2582 ], 2583 title="Request Body Payload (Non-JSON)", 2584 ) 2585 request_body_json: Optional[Union[Dict[str, Any], str]] = Field( 2586 None, 2587 deprecated=True, 2588 deprecation_message="Use `request_body` field instead.", 2589 description="Specifies how to populate the body of the request with a JSON payload. Can contain nested objects.", 2590 examples=[ 2591 {"sort_order": "ASC", "sort_field": "CREATED_AT"}, 2592 {"key": "{{ config['value'] }}"}, 2593 {"sort": {"field": "updated_at", "order": "ascending"}}, 2594 ], 2595 title="Request Body JSON Payload", 2596 ) 2597 request_body: Optional[ 2598 Union[ 2599 RequestBodyPlainText, 2600 RequestBodyUrlEncodedForm, 2601 RequestBodyJsonObject, 2602 RequestBodyGraphQL, 2603 ] 2604 ] = Field( 2605 None, 2606 description="Specifies how to populate the body of the request with a payload. Can contain nested objects.", 2607 title="Request Body", 2608 ) 2609 error_handler: Optional[ 2610 Union[DefaultErrorHandler, CompositeErrorHandler, CustomErrorHandler] 2611 ] = Field( 2612 None, 2613 description="Error handler component that defines how to handle errors.", 2614 title="Error Handler", 2615 ) 2616 use_cache: Optional[bool] = Field( 2617 False, 2618 description="Enables stream requests caching. This field is automatically set by the CDK.", 2619 title="Use Cache", 2620 ) 2621 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters") 2622 2623 2624class DynamicSchemaLoader(BaseModel): 2625 type: Literal["DynamicSchemaLoader"] 2626 retriever: Union[SimpleRetriever, AsyncRetriever, CustomRetriever] = Field( 2627 ..., 2628 description="Component used to coordinate how records are extracted across stream slices and request pages.", 2629 title="Retriever", 2630 ) 2631 schema_filter: Optional[Union[RecordFilter, CustomRecordFilter]] = Field( 2632 None, 2633 description="Responsible for filtering fields to be added to json schema.", 2634 title="Schema Filter", 2635 ) 2636 schema_transformations: Optional[ 2637 List[ 2638 Union[ 2639 AddFields, 2640 RemoveFields, 2641 KeysToLower, 2642 KeysToSnakeCase, 2643 FlattenFields, 2644 DpathFlattenFields, 2645 KeysReplace, 2646 CustomTransformation, 2647 ] 2648 ] 2649 ] = Field( 2650 None, 2651 description="A list of transformations to be applied to the schema.", 2652 title="Schema Transformations", 2653 ) 2654 schema_type_identifier: SchemaTypeIdentifier 2655 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters") 2656 2657 2658class ParentStreamConfig(BaseModel): 2659 type: Literal["ParentStreamConfig"] 2660 stream: Union[DeclarativeStream, StateDelegatingStream] = Field( 2661 ..., description="Reference to the parent stream.", title="Parent Stream" 2662 ) 2663 parent_key: str = Field( 2664 ..., 2665 description="The primary key of records from the parent stream that will be used during the retrieval of records for the current substream. This parent identifier field is typically a characteristic of the child records being extracted from the source API.", 2666 examples=["id", "{{ config['parent_record_id'] }}"], 2667 title="Parent Key", 2668 ) 2669 partition_field: str = Field( 2670 ..., 2671 description="While iterating over parent records during a sync, the parent_key value can be referenced by using this field.", 2672 examples=["parent_id", "{{ config['parent_partition_field'] }}"], 2673 title="Current Parent Key Value Identifier", 2674 ) 2675 request_option: Optional[RequestOption] = Field( 2676 None, 2677 description="A request option describing where the parent key value should be injected into and under what field name if applicable.", 2678 title="Request Option", 2679 ) 2680 incremental_dependency: Optional[bool] = Field( 2681 False, 2682 description="Indicates whether the parent stream should be read incrementally based on updates in the child stream.", 2683 title="Incremental Dependency", 2684 ) 2685 lazy_read_pointer: Optional[List[str]] = Field( 2686 [], 2687 description="If set, this will enable lazy reading, using the initial read of parent records to extract child records.", 2688 title="Lazy Read Pointer", 2689 ) 2690 extra_fields: Optional[List[List[str]]] = Field( 2691 None, 2692 description="Array of field paths to include as additional fields in the stream slice. Each path is an array of strings representing keys to access fields in the respective parent record. Accessible via `stream_slice.extra_fields`. Missing fields are set to `None`.", 2693 title="Extra Fields", 2694 ) 2695 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters") 2696 2697 2698class PropertiesFromEndpoint(BaseModel): 2699 type: Literal["PropertiesFromEndpoint"] 2700 property_field_path: List[str] = Field( 2701 ..., 2702 description="Describes the path to the field that should be extracted", 2703 examples=[["name"]], 2704 ) 2705 retriever: Union[SimpleRetriever, CustomRetriever] = Field( 2706 ..., 2707 description="Requester component that describes how to fetch the properties to query from a remote API endpoint.", 2708 ) 2709 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters") 2710 2711 2712class QueryProperties(BaseModel): 2713 type: Literal["QueryProperties"] 2714 property_list: Union[List[str], PropertiesFromEndpoint] = Field( 2715 ..., 2716 description="The set of properties that will be queried for in the outbound request. This can either be statically defined or dynamic based on an API endpoint", 2717 title="Property List", 2718 ) 2719 always_include_properties: Optional[List[str]] = Field( 2720 None, 2721 description="The list of properties that should be included in every set of properties when multiple chunks of properties are being requested.", 2722 title="Always Include Properties", 2723 ) 2724 property_chunking: Optional[PropertyChunking] = Field( 2725 None, 2726 description="Defines how query properties will be grouped into smaller sets for APIs with limitations on the number of properties fetched per API request.", 2727 title="Property Chunking", 2728 ) 2729 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters") 2730 2731 2732class StateDelegatingStream(BaseModel): 2733 type: Literal["StateDelegatingStream"] 2734 name: str = Field(..., description="The stream name.", example=["Users"], title="Name") 2735 full_refresh_stream: DeclarativeStream = Field( 2736 ..., 2737 description="Component used to coordinate how records are extracted across stream slices and request pages when the state is empty or not provided.", 2738 title="Full Refresh Stream", 2739 ) 2740 incremental_stream: DeclarativeStream = Field( 2741 ..., 2742 description="Component used to coordinate how records are extracted across stream slices and request pages when the state provided.", 2743 title="Incremental Stream", 2744 ) 2745 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters") 2746 2747 2748class SimpleRetriever(BaseModel): 2749 type: Literal["SimpleRetriever"] 2750 requester: Union[HttpRequester, CustomRequester] = Field( 2751 ..., 2752 description="Requester component that describes how to prepare HTTP requests to send to the source API.", 2753 ) 2754 decoder: Optional[ 2755 Union[ 2756 JsonDecoder, 2757 XmlDecoder, 2758 CsvDecoder, 2759 JsonlDecoder, 2760 GzipDecoder, 2761 IterableDecoder, 2762 ZipfileDecoder, 2763 CustomDecoder, 2764 ] 2765 ] = Field( 2766 None, 2767 description="Component decoding the response so records can be extracted.", 2768 title="HTTP Response Format", 2769 ) 2770 record_selector: RecordSelector = Field( 2771 ..., 2772 description="Component that describes how to extract records from a HTTP response.", 2773 ) 2774 paginator: Optional[Union[DefaultPaginator, NoPagination]] = Field( 2775 None, 2776 description="Paginator component that describes how to navigate through the API's pages.", 2777 ) 2778 ignore_stream_slicer_parameters_on_paginated_requests: Optional[bool] = Field( 2779 False, 2780 description="If true, the partition router and incremental request options will be ignored when paginating requests. Request options set directly on the requester will not be ignored.", 2781 ) 2782 partition_router: Optional[ 2783 Union[ 2784 SubstreamPartitionRouter, 2785 ListPartitionRouter, 2786 GroupingPartitionRouter, 2787 CustomPartitionRouter, 2788 List[ 2789 Union[ 2790 SubstreamPartitionRouter, 2791 ListPartitionRouter, 2792 GroupingPartitionRouter, 2793 CustomPartitionRouter, 2794 ] 2795 ], 2796 ] 2797 ] = Field( 2798 None, 2799 description="Used to iteratively execute requests over a set of values, such as a parent stream's records or a list of constant values.", 2800 title="Partition Router", 2801 ) 2802 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters") 2803 2804 2805class AsyncRetriever(BaseModel): 2806 type: Literal["AsyncRetriever"] 2807 record_selector: RecordSelector = Field( 2808 ..., 2809 description="Component that describes how to extract records from a HTTP response.", 2810 ) 2811 status_mapping: AsyncJobStatusMap = Field( 2812 ..., description="Async Job Status to Airbyte CDK Async Job Status mapping." 2813 ) 2814 status_extractor: Union[DpathExtractor, CustomRecordExtractor] = Field( 2815 ..., description="Responsible for fetching the actual status of the async job." 2816 ) 2817 download_target_extractor: Union[DpathExtractor, CustomRecordExtractor] = Field( 2818 ..., 2819 description="Responsible for fetching the final result `urls` provided by the completed / finished / ready async job.", 2820 ) 2821 download_extractor: Optional[ 2822 Union[DpathExtractor, CustomRecordExtractor, ResponseToFileExtractor] 2823 ] = Field(None, description="Responsible for fetching the records from provided urls.") 2824 creation_requester: Union[HttpRequester, CustomRequester] = Field( 2825 ..., 2826 description="Requester component that describes how to prepare HTTP requests to send to the source API to create the async server-side job.", 2827 ) 2828 polling_requester: Union[HttpRequester, CustomRequester] = Field( 2829 ..., 2830 description="Requester component that describes how to prepare HTTP requests to send to the source API to fetch the status of the running async job.", 2831 ) 2832 polling_job_timeout: Optional[Union[int, str]] = Field( 2833 None, 2834 description="The time in minutes after which the single Async Job should be considered as Timed Out.", 2835 ) 2836 download_target_requester: Optional[Union[HttpRequester, CustomRequester]] = Field( 2837 None, 2838 description="Requester component that describes how to prepare HTTP requests to send to the source API to extract the url from polling response by the completed async job.", 2839 ) 2840 download_requester: Union[HttpRequester, CustomRequester] = Field( 2841 ..., 2842 description="Requester component that describes how to prepare HTTP requests to send to the source API to download the data provided by the completed async job.", 2843 ) 2844 download_paginator: Optional[Union[DefaultPaginator, NoPagination]] = Field( 2845 None, 2846 description="Paginator component that describes how to navigate through the API's pages during download.", 2847 ) 2848 abort_requester: Optional[Union[HttpRequester, CustomRequester]] = Field( 2849 None, 2850 description="Requester component that describes how to prepare HTTP requests to send to the source API to abort a job once it is timed out from the source's perspective.", 2851 ) 2852 delete_requester: Optional[Union[HttpRequester, CustomRequester]] = Field( 2853 None, 2854 description="Requester component that describes how to prepare HTTP requests to send to the source API to delete a job once the records are extracted.", 2855 ) 2856 partition_router: Optional[ 2857 Union[ 2858 ListPartitionRouter, 2859 SubstreamPartitionRouter, 2860 GroupingPartitionRouter, 2861 CustomPartitionRouter, 2862 List[ 2863 Union[ 2864 ListPartitionRouter, 2865 SubstreamPartitionRouter, 2866 GroupingPartitionRouter, 2867 CustomPartitionRouter, 2868 ] 2869 ], 2870 ] 2871 ] = Field( 2872 [], 2873 description="PartitionRouter component that describes how to partition the stream, enabling incremental syncs and checkpointing.", 2874 title="Partition Router", 2875 ) 2876 decoder: Optional[ 2877 Union[ 2878 CsvDecoder, 2879 GzipDecoder, 2880 JsonDecoder, 2881 JsonlDecoder, 2882 IterableDecoder, 2883 XmlDecoder, 2884 ZipfileDecoder, 2885 CustomDecoder, 2886 ] 2887 ] = Field( 2888 None, 2889 description="Component decoding the response so records can be extracted.", 2890 title="HTTP Response Format", 2891 ) 2892 download_decoder: Optional[ 2893 Union[ 2894 CsvDecoder, 2895 GzipDecoder, 2896 JsonDecoder, 2897 JsonlDecoder, 2898 IterableDecoder, 2899 XmlDecoder, 2900 ZipfileDecoder, 2901 CustomDecoder, 2902 ] 2903 ] = Field( 2904 None, 2905 description="Component decoding the download response so records can be extracted.", 2906 title="Download HTTP Response Format", 2907 ) 2908 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters") 2909 2910 2911class SubstreamPartitionRouter(BaseModel): 2912 type: Literal["SubstreamPartitionRouter"] 2913 parent_stream_configs: List[ParentStreamConfig] = Field( 2914 ..., 2915 description="Specifies which parent streams are being iterated over and how parent records should be used to partition the child stream data set.", 2916 title="Parent Stream Configs", 2917 ) 2918 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters") 2919 2920 2921class GroupingPartitionRouter(BaseModel): 2922 type: Literal["GroupingPartitionRouter"] 2923 group_size: int = Field( 2924 ..., 2925 description="The number of partitions to include in each group. This determines how many partition values are batched together in a single slice.", 2926 examples=[10, 50], 2927 title="Group Size", 2928 ) 2929 underlying_partition_router: Union[ 2930 ListPartitionRouter, SubstreamPartitionRouter, CustomPartitionRouter 2931 ] = Field( 2932 ..., 2933 description="The partition router whose output will be grouped. This can be any valid partition router component.", 2934 title="Underlying Partition Router", 2935 ) 2936 deduplicate: Optional[bool] = Field( 2937 True, 2938 description="If true, ensures that partitions are unique within each group by removing duplicates based on the partition key.", 2939 title="Deduplicate Partitions", 2940 ) 2941 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters") 2942 2943 2944class HttpComponentsResolver(BaseModel): 2945 type: Literal["HttpComponentsResolver"] 2946 retriever: Union[SimpleRetriever, AsyncRetriever, CustomRetriever] = Field( 2947 ..., 2948 description="Component used to coordinate how records are extracted across stream slices and request pages.", 2949 title="Retriever", 2950 ) 2951 components_mapping: List[ComponentMappingDefinition] 2952 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters") 2953 2954 2955class DynamicDeclarativeStream(BaseModel): 2956 type: Literal["DynamicDeclarativeStream"] 2957 name: Optional[str] = Field( 2958 "", description="The dynamic stream name.", example=["Tables"], title="Name" 2959 ) 2960 stream_template: Union[DeclarativeStream, StateDelegatingStream] = Field( 2961 ..., description="Reference to the stream template.", title="Stream Template" 2962 ) 2963 components_resolver: Union[ 2964 HttpComponentsResolver, ConfigComponentsResolver, ParametrizedComponentsResolver 2965 ] = Field( 2966 ..., 2967 description="Component resolve and populates stream templates with components values.", 2968 title="Components Resolver", 2969 ) 2970 use_parent_parameters: Optional[bool] = Field( 2971 True, 2972 description="Whether or not to prioritize parent parameters over component parameters when constructing dynamic streams. Defaults to true for backward compatibility.", 2973 title="Use Parent Parameters", 2974 ) 2975 2976 2977ComplexFieldType.update_forward_refs() 2978GzipDecoder.update_forward_refs() 2979CompositeErrorHandler.update_forward_refs() 2980DeclarativeSource1.update_forward_refs() 2981DeclarativeSource2.update_forward_refs() 2982SelectiveAuthenticator.update_forward_refs() 2983ConditionalStreams.update_forward_refs() 2984FileUploader.update_forward_refs() 2985DeclarativeStream.update_forward_refs() 2986SessionTokenAuthenticator.update_forward_refs() 2987HttpRequester.update_forward_refs() 2988DynamicSchemaLoader.update_forward_refs() 2989ParentStreamConfig.update_forward_refs() 2990PropertiesFromEndpoint.update_forward_refs() 2991SimpleRetriever.update_forward_refs() 2992AsyncRetriever.update_forward_refs()
class
AuthFlowType(enum.Enum):
An enumeration.
oauth2_0 =
<AuthFlowType.oauth2_0: 'oauth2.0'>
oauth1_0 =
<AuthFlowType.oauth1_0: 'oauth1.0'>
class
BasicHttpAuthenticator(pydantic.v1.main.BaseModel):
24class BasicHttpAuthenticator(BaseModel): 25 type: Literal["BasicHttpAuthenticator"] 26 username: str = Field( 27 ..., 28 description="The username that will be combined with the password, base64 encoded and used to make requests. Fill it in the user inputs.", 29 examples=["{{ config['username'] }}", "{{ config['api_key'] }}"], 30 title="Username", 31 ) 32 password: Optional[str] = Field( 33 "", 34 description="The password that will be combined with the username, base64 encoded and used to make requests. Fill it in the user inputs.", 35 examples=["{{ config['password'] }}", ""], 36 title="Password", 37 ) 38 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters")
class
BearerAuthenticator(pydantic.v1.main.BaseModel):
41class BearerAuthenticator(BaseModel): 42 type: Literal["BearerAuthenticator"] 43 api_token: str = Field( 44 ..., 45 description="Token to inject as request header for authenticating with the API.", 46 examples=["{{ config['api_key'] }}", "{{ config['token'] }}"], 47 title="Bearer Token", 48 ) 49 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters")
class
DynamicStreamCheckConfig(pydantic.v1.main.BaseModel):
52class DynamicStreamCheckConfig(BaseModel): 53 type: Literal["DynamicStreamCheckConfig"] 54 dynamic_stream_name: str = Field( 55 ..., description="The dynamic stream name.", title="Dynamic Stream Name" 56 ) 57 stream_count: Optional[int] = Field( 58 0, 59 description="The number of streams to attempt reading from during a check operation. If `stream_count` exceeds the total number of available streams, the minimum of the two values will be used.", 60 title="Stream Count", 61 )
class
CheckDynamicStream(pydantic.v1.main.BaseModel):
64class CheckDynamicStream(BaseModel): 65 type: Literal["CheckDynamicStream"] 66 stream_count: int = Field( 67 ..., 68 description="Numbers of the streams to try reading from when running a check operation.", 69 title="Stream Count", 70 ) 71 use_check_availability: Optional[bool] = Field( 72 True, 73 description="Enables stream check availability. This field is automatically set by the CDK.", 74 title="Use Check Availability", 75 )
class
ConcurrencyLevel(pydantic.v1.main.BaseModel):
78class ConcurrencyLevel(BaseModel): 79 type: Optional[Literal["ConcurrencyLevel"]] = None 80 default_concurrency: Union[int, str] = Field( 81 ..., 82 description="The amount of concurrency that will applied during a sync. This value can be hardcoded or user-defined in the config if different users have varying volume thresholds in the target API.", 83 examples=[10, "{{ config['num_workers'] or 10 }}"], 84 title="Default Concurrency", 85 ) 86 max_concurrency: Optional[int] = Field( 87 None, 88 description="The maximum level of concurrency that will be used during a sync. This becomes a required field when the default_concurrency derives from the config, because it serves as a safeguard against a user-defined threshold that is too high.", 89 examples=[20, 100], 90 title="Max Concurrency", 91 ) 92 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters")
class
ConstantBackoffStrategy(pydantic.v1.main.BaseModel):
95class ConstantBackoffStrategy(BaseModel): 96 type: Literal["ConstantBackoffStrategy"] 97 backoff_time_in_seconds: Union[float, str] = Field( 98 ..., 99 description="Backoff time in seconds.", 100 examples=[30, 30.5, "{{ config['backoff_time'] }}"], 101 title="Backoff Time", 102 ) 103 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters")
class
CursorPagination(pydantic.v1.main.BaseModel):
106class CursorPagination(BaseModel): 107 type: Literal["CursorPagination"] 108 cursor_value: str = Field( 109 ..., 110 description="Value of the cursor defining the next page to fetch.", 111 examples=[ 112 "{{ headers.link.next.cursor }}", 113 "{{ last_record['key'] }}", 114 "{{ response['nextPage'] }}", 115 ], 116 title="Cursor Value", 117 ) 118 page_size: Optional[int] = Field( 119 None, 120 description="The number of records to include in each pages.", 121 examples=[100], 122 title="Page Size", 123 ) 124 stop_condition: Optional[str] = Field( 125 None, 126 description="Template string evaluating when to stop paginating.", 127 examples=[ 128 "{{ response.data.has_more is false }}", 129 "{{ 'next' not in headers['link'] }}", 130 ], 131 title="Stop Condition", 132 ) 133 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters")
class
CustomAuthenticator(pydantic.v1.main.BaseModel):
136class CustomAuthenticator(BaseModel): 137 class Config: 138 extra = Extra.allow 139 140 type: Literal["CustomAuthenticator"] 141 class_name: str = Field( 142 ..., 143 description="Fully-qualified name of the class that will be implementing the custom authentication strategy. Has to be a sub class of DeclarativeAuthenticator. The format is `source_<name>.<package>.<class_name>`.", 144 examples=["source_railz.components.ShortLivedTokenAuthenticator"], 145 title="Class Name", 146 ) 147 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters")
class
CustomAuthenticator.Config:
class
CustomBackoffStrategy(pydantic.v1.main.BaseModel):
150class CustomBackoffStrategy(BaseModel): 151 class Config: 152 extra = Extra.allow 153 154 type: Literal["CustomBackoffStrategy"] 155 class_name: str = Field( 156 ..., 157 description="Fully-qualified name of the class that will be implementing the custom backoff strategy. The format is `source_<name>.<package>.<class_name>`.", 158 examples=["source_railz.components.MyCustomBackoffStrategy"], 159 title="Class Name", 160 ) 161 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters")
class
CustomBackoffStrategy.Config:
class
CustomErrorHandler(pydantic.v1.main.BaseModel):
164class CustomErrorHandler(BaseModel): 165 class Config: 166 extra = Extra.allow 167 168 type: Literal["CustomErrorHandler"] 169 class_name: str = Field( 170 ..., 171 description="Fully-qualified name of the class that will be implementing the custom error handler. The format is `source_<name>.<package>.<class_name>`.", 172 examples=["source_railz.components.MyCustomErrorHandler"], 173 title="Class Name", 174 ) 175 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters")
class
CustomErrorHandler.Config:
class
CustomIncrementalSync(pydantic.v1.main.BaseModel):
178class CustomIncrementalSync(BaseModel): 179 class Config: 180 extra = Extra.allow 181 182 type: Literal["CustomIncrementalSync"] 183 class_name: str = Field( 184 ..., 185 description="Fully-qualified name of the class that will be implementing the custom incremental sync. The format is `source_<name>.<package>.<class_name>`.", 186 examples=["source_railz.components.MyCustomIncrementalSync"], 187 title="Class Name", 188 ) 189 cursor_field: str = Field( 190 ..., 191 description="The location of the value on a record that will be used as a bookmark during sync.", 192 ) 193 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters")
class
CustomIncrementalSync.Config:
class
CustomPaginationStrategy(pydantic.v1.main.BaseModel):
196class CustomPaginationStrategy(BaseModel): 197 class Config: 198 extra = Extra.allow 199 200 type: Literal["CustomPaginationStrategy"] 201 class_name: str = Field( 202 ..., 203 description="Fully-qualified name of the class that will be implementing the custom pagination strategy. The format is `source_<name>.<package>.<class_name>`.", 204 examples=["source_railz.components.MyCustomPaginationStrategy"], 205 title="Class Name", 206 ) 207 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters")
class
CustomPaginationStrategy.Config:
class
CustomRecordExtractor(pydantic.v1.main.BaseModel):
210class CustomRecordExtractor(BaseModel): 211 class Config: 212 extra = Extra.allow 213 214 type: Literal["CustomRecordExtractor"] 215 class_name: str = Field( 216 ..., 217 description="Fully-qualified name of the class that will be implementing the custom record extraction strategy. The format is `source_<name>.<package>.<class_name>`.", 218 examples=["source_railz.components.MyCustomRecordExtractor"], 219 title="Class Name", 220 ) 221 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters")
class
CustomRecordExtractor.Config:
class
CustomRecordFilter(pydantic.v1.main.BaseModel):
224class CustomRecordFilter(BaseModel): 225 class Config: 226 extra = Extra.allow 227 228 type: Literal["CustomRecordFilter"] 229 class_name: str = Field( 230 ..., 231 description="Fully-qualified name of the class that will be implementing the custom record filter strategy. The format is `source_<name>.<package>.<class_name>`.", 232 examples=["source_railz.components.MyCustomCustomRecordFilter"], 233 title="Class Name", 234 ) 235 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters")
class
CustomRecordFilter.Config:
class
CustomRequester(pydantic.v1.main.BaseModel):
238class CustomRequester(BaseModel): 239 class Config: 240 extra = Extra.allow 241 242 type: Literal["CustomRequester"] 243 class_name: str = Field( 244 ..., 245 description="Fully-qualified name of the class that will be implementing the custom requester strategy. The format is `source_<name>.<package>.<class_name>`.", 246 examples=["source_railz.components.MyCustomRecordExtractor"], 247 title="Class Name", 248 ) 249 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters")
class
CustomRequester.Config:
class
CustomRetriever(pydantic.v1.main.BaseModel):
252class CustomRetriever(BaseModel): 253 class Config: 254 extra = Extra.allow 255 256 type: Literal["CustomRetriever"] 257 class_name: str = Field( 258 ..., 259 description="Fully-qualified name of the class that will be implementing the custom retriever strategy. The format is `source_<name>.<package>.<class_name>`.", 260 examples=["source_railz.components.MyCustomRetriever"], 261 title="Class Name", 262 ) 263 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters")
class
CustomRetriever.Config:
class
CustomPartitionRouter(pydantic.v1.main.BaseModel):
266class CustomPartitionRouter(BaseModel): 267 class Config: 268 extra = Extra.allow 269 270 type: Literal["CustomPartitionRouter"] 271 class_name: str = Field( 272 ..., 273 description="Fully-qualified name of the class that will be implementing the custom partition router. The format is `source_<name>.<package>.<class_name>`.", 274 examples=["source_railz.components.MyCustomPartitionRouter"], 275 title="Class Name", 276 ) 277 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters")
class
CustomPartitionRouter.Config:
class
CustomSchemaLoader(pydantic.v1.main.BaseModel):
280class CustomSchemaLoader(BaseModel): 281 class Config: 282 extra = Extra.allow 283 284 type: Literal["CustomSchemaLoader"] 285 class_name: str = Field( 286 ..., 287 description="Fully-qualified name of the class that will be implementing the custom schema loader. The format is `source_<name>.<package>.<class_name>`.", 288 examples=["source_railz.components.MyCustomSchemaLoader"], 289 title="Class Name", 290 ) 291 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters")
class
CustomSchemaLoader.Config:
class
CustomSchemaNormalization(pydantic.v1.main.BaseModel):
294class CustomSchemaNormalization(BaseModel): 295 class Config: 296 extra = Extra.allow 297 298 type: Literal["CustomSchemaNormalization"] 299 class_name: str = Field( 300 ..., 301 description="Fully-qualified name of the class that will be implementing the custom normalization. The format is `source_<name>.<package>.<class_name>`.", 302 examples=[ 303 "source_amazon_seller_partner.components.LedgerDetailedViewReportsTypeTransformer" 304 ], 305 title="Class Name", 306 ) 307 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters")
class
CustomSchemaNormalization.Config:
class
CustomStateMigration(pydantic.v1.main.BaseModel):
310class CustomStateMigration(BaseModel): 311 class Config: 312 extra = Extra.allow 313 314 type: Literal["CustomStateMigration"] 315 class_name: str = Field( 316 ..., 317 description="Fully-qualified name of the class that will be implementing the custom state migration. The format is `source_<name>.<package>.<class_name>`.", 318 examples=["source_railz.components.MyCustomStateMigration"], 319 title="Class Name", 320 ) 321 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters")
class
CustomStateMigration.Config:
class
CustomTransformation(pydantic.v1.main.BaseModel):
324class CustomTransformation(BaseModel): 325 class Config: 326 extra = Extra.allow 327 328 type: Literal["CustomTransformation"] 329 class_name: str = Field( 330 ..., 331 description="Fully-qualified name of the class that will be implementing the custom transformation. The format is `source_<name>.<package>.<class_name>`.", 332 examples=["source_railz.components.MyCustomTransformation"], 333 title="Class Name", 334 ) 335 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters")
class
CustomTransformation.Config:
class
LegacyToPerPartitionStateMigration(pydantic.v1.main.BaseModel):
class
LegacyToPerPartitionStateMigration.Config:
class
Clamping(pydantic.v1.main.BaseModel):
class
Algorithm(enum.Enum):
355class Algorithm(Enum): 356 HS256 = "HS256" 357 HS384 = "HS384" 358 HS512 = "HS512" 359 ES256 = "ES256" 360 ES256K = "ES256K" 361 ES384 = "ES384" 362 ES512 = "ES512" 363 RS256 = "RS256" 364 RS384 = "RS384" 365 RS512 = "RS512" 366 PS256 = "PS256" 367 PS384 = "PS384" 368 PS512 = "PS512" 369 EdDSA = "EdDSA"
An enumeration.
HS256 =
<Algorithm.HS256: 'HS256'>
HS384 =
<Algorithm.HS384: 'HS384'>
HS512 =
<Algorithm.HS512: 'HS512'>
ES256 =
<Algorithm.ES256: 'ES256'>
ES256K =
<Algorithm.ES256K: 'ES256K'>
ES384 =
<Algorithm.ES384: 'ES384'>
ES512 =
<Algorithm.ES512: 'ES512'>
RS256 =
<Algorithm.RS256: 'RS256'>
RS384 =
<Algorithm.RS384: 'RS384'>
RS512 =
<Algorithm.RS512: 'RS512'>
PS256 =
<Algorithm.PS256: 'PS256'>
PS384 =
<Algorithm.PS384: 'PS384'>
PS512 =
<Algorithm.PS512: 'PS512'>
EdDSA =
<Algorithm.EdDSA: 'EdDSA'>
class
JwtHeaders(pydantic.v1.main.BaseModel):
372class JwtHeaders(BaseModel): 373 class Config: 374 extra = Extra.forbid 375 376 kid: Optional[str] = Field( 377 None, 378 description="Private key ID for user account.", 379 examples=["{{ config['kid'] }}"], 380 title="Key Identifier", 381 ) 382 typ: Optional[str] = Field( 383 "JWT", 384 description="The media type of the complete JWT.", 385 examples=["JWT"], 386 title="Type", 387 ) 388 cty: Optional[str] = Field( 389 None, 390 description="Content type of JWT header.", 391 examples=["JWT"], 392 title="Content Type", 393 )
class
JwtHeaders.Config:
class
JwtPayload(pydantic.v1.main.BaseModel):
396class JwtPayload(BaseModel): 397 class Config: 398 extra = Extra.forbid 399 400 iss: Optional[str] = Field( 401 None, 402 description="The user/principal that issued the JWT. Commonly a value unique to the user.", 403 examples=["{{ config['iss'] }}"], 404 title="Issuer", 405 ) 406 sub: Optional[str] = Field( 407 None, 408 description="The subject of the JWT. Commonly defined by the API.", 409 title="Subject", 410 ) 411 aud: Optional[str] = Field( 412 None, 413 description="The recipient that the JWT is intended for. Commonly defined by the API.", 414 examples=["appstoreconnect-v1"], 415 title="Audience", 416 )
class
JwtPayload.Config:
class
JwtAuthenticator(pydantic.v1.main.BaseModel):
419class JwtAuthenticator(BaseModel): 420 type: Literal["JwtAuthenticator"] 421 secret_key: str = Field( 422 ..., 423 description="Secret used to sign the JSON web token.", 424 examples=["{{ config['secret_key'] }}"], 425 title="Secret Key", 426 ) 427 base64_encode_secret_key: Optional[bool] = Field( 428 False, 429 description='When set to true, the secret key will be base64 encoded prior to being encoded as part of the JWT. Only set to "true" when required by the API.', 430 title="Base64-encode Secret Key", 431 ) 432 algorithm: Algorithm = Field( 433 ..., 434 description="Algorithm used to sign the JSON web token.", 435 examples=["ES256", "HS256", "RS256", "{{ config['algorithm'] }}"], 436 title="Algorithm", 437 ) 438 token_duration: Optional[int] = Field( 439 1200, 440 description="The amount of time in seconds a JWT token can be valid after being issued.", 441 examples=[1200, 3600], 442 title="Token Duration", 443 ) 444 header_prefix: Optional[str] = Field( 445 None, 446 description="The prefix to be used within the Authentication header.", 447 examples=["Bearer", "Basic"], 448 title="Header Prefix", 449 ) 450 jwt_headers: Optional[JwtHeaders] = Field( 451 None, 452 description="JWT headers used when signing JSON web token.", 453 title="JWT Headers", 454 ) 455 additional_jwt_headers: Optional[Dict[str, Any]] = Field( 456 None, 457 description="Additional headers to be included with the JWT headers object.", 458 title="Additional JWT Headers", 459 ) 460 jwt_payload: Optional[JwtPayload] = Field( 461 None, 462 description="JWT Payload used when signing JSON web token.", 463 title="JWT Payload", 464 ) 465 additional_jwt_payload: Optional[Dict[str, Any]] = Field( 466 None, 467 description="Additional properties to be added to the JWT payload.", 468 title="Additional JWT Payload Properties", 469 ) 470 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters")
algorithm: Algorithm
jwt_headers: Optional[JwtHeaders]
jwt_payload: Optional[JwtPayload]
class
RefreshTokenUpdater(pydantic.v1.main.BaseModel):
473class RefreshTokenUpdater(BaseModel): 474 refresh_token_name: Optional[str] = Field( 475 "refresh_token", 476 description="The name of the property which contains the updated refresh token in the response from the token refresh endpoint.", 477 examples=["refresh_token"], 478 title="Refresh Token Property Name", 479 ) 480 access_token_config_path: Optional[List[str]] = Field( 481 ["credentials", "access_token"], 482 description="Config path to the access token. Make sure the field actually exists in the config.", 483 examples=[["credentials", "access_token"], ["access_token"]], 484 title="Config Path To Access Token", 485 ) 486 refresh_token_config_path: Optional[List[str]] = Field( 487 ["credentials", "refresh_token"], 488 description="Config path to the access token. Make sure the field actually exists in the config.", 489 examples=[["credentials", "refresh_token"], ["refresh_token"]], 490 title="Config Path To Refresh Token", 491 ) 492 token_expiry_date_config_path: Optional[List[str]] = Field( 493 ["credentials", "token_expiry_date"], 494 description="Config path to the expiry date. Make sure actually exists in the config.", 495 examples=[["credentials", "token_expiry_date"]], 496 title="Config Path To Expiry Date", 497 ) 498 refresh_token_error_status_codes: Optional[List[int]] = Field( 499 [], 500 description="Status Codes to Identify refresh token error in response (Refresh Token Error Key and Refresh Token Error Values should be also specified). Responses with one of the error status code and containing an error value will be flagged as a config error", 501 examples=[[400, 500]], 502 title="Refresh Token Error Status Codes", 503 ) 504 refresh_token_error_key: Optional[str] = Field( 505 "", 506 description="Key to Identify refresh token error in response (Refresh Token Error Status Codes and Refresh Token Error Values should be also specified).", 507 examples=["error"], 508 title="Refresh Token Error Key", 509 ) 510 refresh_token_error_values: Optional[List[str]] = Field( 511 [], 512 description='List of values to check for exception during token refresh process. Used to check if the error found in the response matches the key from the Refresh Token Error Key field (e.g. response={"error": "invalid_grant"}). Only responses with one of the error status code and containing an error value will be flagged as a config error', 513 examples=[["invalid_grant", "invalid_permissions"]], 514 title="Refresh Token Error Values", 515 )
class
OAuthAuthenticator(pydantic.v1.main.BaseModel):
518class OAuthAuthenticator(BaseModel): 519 type: Literal["OAuthAuthenticator"] 520 client_id_name: Optional[str] = Field( 521 "client_id", 522 description="The name of the property to use to refresh the `access_token`.", 523 examples=["custom_app_id"], 524 title="Client ID Property Name", 525 ) 526 client_id: Optional[str] = Field( 527 None, 528 description="The OAuth client ID. Fill it in the user inputs.", 529 examples=["{{ config['client_id }}", "{{ config['credentials']['client_id }}"], 530 title="Client ID", 531 ) 532 client_secret_name: Optional[str] = Field( 533 "client_secret", 534 description="The name of the property to use to refresh the `access_token`.", 535 examples=["custom_app_secret"], 536 title="Client Secret Property Name", 537 ) 538 client_secret: Optional[str] = Field( 539 None, 540 description="The OAuth client secret. Fill it in the user inputs.", 541 examples=[ 542 "{{ config['client_secret }}", 543 "{{ config['credentials']['client_secret }}", 544 ], 545 title="Client Secret", 546 ) 547 refresh_token_name: Optional[str] = Field( 548 "refresh_token", 549 description="The name of the property to use to refresh the `access_token`.", 550 examples=["custom_app_refresh_value"], 551 title="Refresh Token Property Name", 552 ) 553 refresh_token: Optional[str] = Field( 554 None, 555 description="Credential artifact used to get a new access token.", 556 examples=[ 557 "{{ config['refresh_token'] }}", 558 "{{ config['credentials]['refresh_token'] }}", 559 ], 560 title="Refresh Token", 561 ) 562 token_refresh_endpoint: Optional[str] = Field( 563 None, 564 description="The full URL to call to obtain a new access token.", 565 examples=["https://connect.squareup.com/oauth2/token"], 566 title="Token Refresh Endpoint", 567 ) 568 access_token_name: Optional[str] = Field( 569 "access_token", 570 description="The name of the property which contains the access token in the response from the token refresh endpoint.", 571 examples=["access_token"], 572 title="Access Token Property Name", 573 ) 574 access_token_value: Optional[str] = Field( 575 None, 576 description="The value of the access_token to bypass the token refreshing using `refresh_token`.", 577 examples=["secret_access_token_value"], 578 title="Access Token Value", 579 ) 580 expires_in_name: Optional[str] = Field( 581 "expires_in", 582 description="The name of the property which contains the expiry date in the response from the token refresh endpoint.", 583 examples=["expires_in"], 584 title="Token Expiry Property Name", 585 ) 586 grant_type_name: Optional[str] = Field( 587 "grant_type", 588 description="The name of the property to use to refresh the `access_token`.", 589 examples=["custom_grant_type"], 590 title="Grant Type Property Name", 591 ) 592 grant_type: Optional[str] = Field( 593 "refresh_token", 594 description="Specifies the OAuth2 grant type. If set to refresh_token, the refresh_token needs to be provided as well. For client_credentials, only client id and secret are required. Other grant types are not officially supported.", 595 examples=["refresh_token", "client_credentials"], 596 title="Grant Type", 597 ) 598 refresh_request_body: Optional[Dict[str, Any]] = Field( 599 None, 600 description="Body of the request sent to get a new access token.", 601 examples=[ 602 { 603 "applicationId": "{{ config['application_id'] }}", 604 "applicationSecret": "{{ config['application_secret'] }}", 605 "token": "{{ config['token'] }}", 606 } 607 ], 608 title="Refresh Request Body", 609 ) 610 refresh_request_headers: Optional[Dict[str, Any]] = Field( 611 None, 612 description="Headers of the request sent to get a new access token.", 613 examples=[ 614 { 615 "Authorization": "<AUTH_TOKEN>", 616 "Content-Type": "application/x-www-form-urlencoded", 617 } 618 ], 619 title="Refresh Request Headers", 620 ) 621 scopes: Optional[List[str]] = Field( 622 None, 623 description="List of scopes that should be granted to the access token.", 624 examples=[["crm.list.read", "crm.objects.contacts.read", "crm.schema.contacts.read"]], 625 title="Scopes", 626 ) 627 token_expiry_date: Optional[str] = Field( 628 None, 629 description="The access token expiry date.", 630 examples=["2023-04-06T07:12:10.421833+00:00", 1680842386], 631 title="Token Expiry Date", 632 ) 633 token_expiry_date_format: Optional[str] = Field( 634 None, 635 description="The format of the time to expiration datetime. Provide it if the time is returned as a date-time string instead of seconds.", 636 examples=["%Y-%m-%d %H:%M:%S.%f+00:00"], 637 title="Token Expiry Date Format", 638 ) 639 refresh_token_updater: Optional[RefreshTokenUpdater] = Field( 640 None, 641 description="When the token updater is defined, new refresh tokens, access tokens and the access token expiry date are written back from the authentication response to the config object. This is important if the refresh token can only used once.", 642 title="Token Updater", 643 ) 644 profile_assertion: Optional[JwtAuthenticator] = Field( 645 None, 646 description="The authenticator being used to authenticate the client authenticator.", 647 title="Profile Assertion", 648 ) 649 use_profile_assertion: Optional[bool] = Field( 650 False, 651 description="Enable using profile assertion as a flow for OAuth authorization.", 652 title="Use Profile Assertion", 653 ) 654 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters")
refresh_token_updater: Optional[RefreshTokenUpdater]
profile_assertion: Optional[JwtAuthenticator]
class
Rate(pydantic.v1.main.BaseModel):
657class Rate(BaseModel): 658 class Config: 659 extra = Extra.allow 660 661 limit: Union[int, str] = Field( 662 ..., 663 description="The maximum number of calls allowed within the interval.", 664 title="Limit", 665 ) 666 interval: str = Field( 667 ..., 668 description="The time interval for the rate limit.", 669 examples=["PT1H", "P1D"], 670 title="Interval", 671 )
class
Rate.Config:
class
HttpRequestRegexMatcher(pydantic.v1.main.BaseModel):
674class HttpRequestRegexMatcher(BaseModel): 675 class Config: 676 extra = Extra.allow 677 678 method: Optional[str] = Field( 679 None, description="The HTTP method to match (e.g., GET, POST).", title="Method" 680 ) 681 url_base: Optional[str] = Field( 682 None, 683 description='The base URL (scheme and host, e.g. "https://api.example.com") to match.', 684 title="URL Base", 685 ) 686 url_path_pattern: Optional[str] = Field( 687 None, 688 description="A regular expression pattern to match the URL path.", 689 title="URL Path Pattern", 690 ) 691 params: Optional[Dict[str, Any]] = Field( 692 None, description="The query parameters to match.", title="Parameters" 693 ) 694 headers: Optional[Dict[str, Any]] = Field( 695 None, description="The headers to match.", title="Headers" 696 )
class
HttpRequestRegexMatcher.Config:
class
DpathExtractor(pydantic.v1.main.BaseModel):
699class DpathExtractor(BaseModel): 700 type: Literal["DpathExtractor"] 701 field_path: List[str] = Field( 702 ..., 703 description='List of potentially nested fields describing the full path of the field to extract. Use "*" to extract all values from an array. See more info in the [docs](https://docs.airbyte.com/connector-development/config-based/understanding-the-yaml-file/record-selector).', 704 examples=[ 705 ["data"], 706 ["data", "records"], 707 ["data", "{{ parameters.name }}"], 708 ["data", "*", "record"], 709 ], 710 title="Field Path", 711 ) 712 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters")
class
ResponseToFileExtractor(pydantic.v1.main.BaseModel):
class
ExponentialBackoffStrategy(pydantic.v1.main.BaseModel):
720class ExponentialBackoffStrategy(BaseModel): 721 type: Literal["ExponentialBackoffStrategy"] 722 factor: Optional[Union[float, str]] = Field( 723 5, 724 description="Multiplicative constant applied on each retry.", 725 examples=[5, 5.5, "10"], 726 title="Factor", 727 ) 728 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters")
class
GroupByKeyMergeStrategy(pydantic.v1.main.BaseModel):
731class GroupByKeyMergeStrategy(BaseModel): 732 type: Literal["GroupByKeyMergeStrategy"] 733 key: Union[str, List[str]] = Field( 734 ..., 735 description="The name of the field on the record whose value will be used to group properties that were retrieved through multiple API requests.", 736 examples=["id", ["parent_id", "end_date"]], 737 title="Key", 738 ) 739 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters")
class
SessionTokenRequestBearerAuthenticator(pydantic.v1.main.BaseModel):
class
HttpMethod(enum.Enum):
An enumeration.
GET =
<HttpMethod.GET: 'GET'>
POST =
<HttpMethod.POST: 'POST'>
class
Action(enum.Enum):
751class Action(Enum): 752 SUCCESS = "SUCCESS" 753 FAIL = "FAIL" 754 RETRY = "RETRY" 755 IGNORE = "IGNORE" 756 RATE_LIMITED = "RATE_LIMITED"
An enumeration.
SUCCESS =
<Action.SUCCESS: 'SUCCESS'>
FAIL =
<Action.FAIL: 'FAIL'>
RETRY =
<Action.RETRY: 'RETRY'>
IGNORE =
<Action.IGNORE: 'IGNORE'>
RATE_LIMITED =
<Action.RATE_LIMITED: 'RATE_LIMITED'>
class
FailureType(enum.Enum):
759class FailureType(Enum): 760 system_error = "system_error" 761 config_error = "config_error" 762 transient_error = "transient_error"
An enumeration.
system_error =
<FailureType.system_error: 'system_error'>
config_error =
<FailureType.config_error: 'config_error'>
transient_error =
<FailureType.transient_error: 'transient_error'>
class
HttpResponseFilter(pydantic.v1.main.BaseModel):
765class HttpResponseFilter(BaseModel): 766 type: Literal["HttpResponseFilter"] 767 action: Optional[Action] = Field( 768 None, 769 description="Action to execute if a response matches the filter.", 770 examples=["SUCCESS", "FAIL", "RETRY", "IGNORE", "RATE_LIMITED"], 771 title="Action", 772 ) 773 failure_type: Optional[FailureType] = Field( 774 None, 775 description="Failure type of traced exception if a response matches the filter.", 776 examples=["system_error", "config_error", "transient_error"], 777 title="Failure Type", 778 ) 779 error_message: Optional[str] = Field( 780 None, 781 description="Error Message to display if the response matches the filter.", 782 title="Error Message", 783 ) 784 error_message_contains: Optional[str] = Field( 785 None, 786 description="Match the response if its error message contains the substring.", 787 example=["This API operation is not enabled for this site"], 788 title="Error Message Substring", 789 ) 790 http_codes: Optional[List[int]] = Field( 791 None, 792 description="Match the response if its HTTP code is included in this list.", 793 examples=[[420, 429], [500]], 794 title="HTTP Codes", 795 unique_items=True, 796 ) 797 predicate: Optional[str] = Field( 798 None, 799 description="Match the response if the predicate evaluates to true.", 800 examples=[ 801 "{{ 'Too much requests' in response }}", 802 "{{ 'error_code' in response and response['error_code'] == 'ComplexityException' }}", 803 ], 804 title="Predicate", 805 ) 806 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters")
action: Optional[Action]
failure_type: Optional[FailureType]
class
ComplexFieldType(pydantic.v1.main.BaseModel):
809class ComplexFieldType(BaseModel): 810 field_type: str 811 items: Optional[Union[str, ComplexFieldType]] = None
items: Union[str, ComplexFieldType, NoneType]
class
TypesMap(pydantic.v1.main.BaseModel):
814class TypesMap(BaseModel): 815 target_type: Union[str, List[str], ComplexFieldType] 816 current_type: Union[str, List[str]] 817 condition: Optional[str] = None
target_type: Union[str, List[str], ComplexFieldType]
class
SchemaTypeIdentifier(pydantic.v1.main.BaseModel):
820class SchemaTypeIdentifier(BaseModel): 821 type: Optional[Literal["SchemaTypeIdentifier"]] = None 822 schema_pointer: Optional[List[str]] = Field( 823 [], 824 description="List of nested fields defining the schema field path to extract. Defaults to [].", 825 title="Schema Path", 826 ) 827 key_pointer: List[str] = Field( 828 ..., 829 description="List of potentially nested fields describing the full path of the field key to extract.", 830 title="Key Path", 831 ) 832 type_pointer: Optional[List[str]] = Field( 833 None, 834 description="List of potentially nested fields describing the full path of the field type to extract.", 835 title="Type Path", 836 ) 837 types_mapping: Optional[List[TypesMap]] = None 838 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters")
types_mapping: Optional[List[TypesMap]]
class
InlineSchemaLoader(pydantic.v1.main.BaseModel):
841class InlineSchemaLoader(BaseModel): 842 type: Literal["InlineSchemaLoader"] 843 schema_: Optional[Dict[str, Any]] = Field( 844 None, 845 alias="schema", 846 description='Describes a streams\' schema. Refer to the <a href="https://docs.airbyte.com/understanding-airbyte/supported-data-types/">Data Types documentation</a> for more details on which types are valid.', 847 title="Schema", 848 )
class
JsonFileSchemaLoader(pydantic.v1.main.BaseModel):
851class JsonFileSchemaLoader(BaseModel): 852 type: Literal["JsonFileSchemaLoader"] 853 file_path: Optional[str] = Field( 854 None, 855 description="Path to the JSON file defining the schema. The path is relative to the connector module's root.", 856 example=["./schemas/users.json"], 857 title="File Path", 858 ) 859 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters")
class
JsonDecoder(pydantic.v1.main.BaseModel):
class
JsonlDecoder(pydantic.v1.main.BaseModel):
class
KeysToLower(pydantic.v1.main.BaseModel):
class
KeysToSnakeCase(pydantic.v1.main.BaseModel):
class
FlattenFields(pydantic.v1.main.BaseModel):
880class FlattenFields(BaseModel): 881 type: Literal["FlattenFields"] 882 flatten_lists: Optional[bool] = Field( 883 True, 884 description="Whether to flatten lists or leave it as is. Default is True.", 885 title="Flatten Lists", 886 ) 887 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters")
class
KeyTransformation(pydantic.v1.main.BaseModel):
890class KeyTransformation(BaseModel): 891 type: Literal["KeyTransformation"] 892 prefix: Optional[str] = Field( 893 None, 894 description="Prefix to add for object keys. If not provided original keys remain unchanged.", 895 examples=["flattened_"], 896 title="Key Prefix", 897 ) 898 suffix: Optional[str] = Field( 899 None, 900 description="Suffix to add for object keys. If not provided original keys remain unchanged.", 901 examples=["_flattened"], 902 title="Key Suffix", 903 )
class
DpathFlattenFields(pydantic.v1.main.BaseModel):
906class DpathFlattenFields(BaseModel): 907 type: Literal["DpathFlattenFields"] 908 field_path: List[str] = Field( 909 ..., 910 description="A path to field that needs to be flattened.", 911 examples=[["data"], ["data", "*", "field"]], 912 title="Field Path", 913 ) 914 delete_origin_value: Optional[bool] = Field( 915 None, 916 description="Whether to delete the origin value or keep it. Default is False.", 917 title="Delete Origin Value", 918 ) 919 replace_record: Optional[bool] = Field( 920 None, 921 description="Whether to replace the origin record or not. Default is False.", 922 title="Replace Origin Record", 923 ) 924 key_transformation: Optional[KeyTransformation] = Field( 925 None, 926 description="Transformation for object keys. If not provided, original key will be used.", 927 title="Key transformation", 928 ) 929 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters")
key_transformation: Optional[KeyTransformation]
class
KeysReplace(pydantic.v1.main.BaseModel):
932class KeysReplace(BaseModel): 933 type: Literal["KeysReplace"] 934 old: str = Field( 935 ..., 936 description="Old value to replace.", 937 examples=[ 938 " ", 939 "{{ record.id }}", 940 "{{ config['id'] }}", 941 "{{ stream_slice['id'] }}", 942 ], 943 title="Old value", 944 ) 945 new: str = Field( 946 ..., 947 description="New value to set.", 948 examples=[ 949 "_", 950 "{{ record.id }}", 951 "{{ config['id'] }}", 952 "{{ stream_slice['id'] }}", 953 ], 954 title="New value", 955 ) 956 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters")
class
IterableDecoder(pydantic.v1.main.BaseModel):
class
XmlDecoder(pydantic.v1.main.BaseModel):
class
CustomDecoder(pydantic.v1.main.BaseModel):
967class CustomDecoder(BaseModel): 968 class Config: 969 extra = Extra.allow 970 971 type: Literal["CustomDecoder"] 972 class_name: str = Field( 973 ..., 974 description="Fully-qualified name of the class that will be implementing the custom decoding. Has to be a sub class of Decoder. The format is `source_<name>.<package>.<class_name>`.", 975 examples=["source_amazon_ads.components.GzipJsonlDecoder"], 976 title="Class Name", 977 ) 978 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters")
class
CustomDecoder.Config:
class
MinMaxDatetime(pydantic.v1.main.BaseModel):
981class MinMaxDatetime(BaseModel): 982 type: Literal["MinMaxDatetime"] 983 datetime: str = Field( 984 ..., 985 description="Datetime value.", 986 examples=["2021-01-01", "2021-01-01T00:00:00Z", "{{ config['start_time'] }}"], 987 title="Datetime", 988 ) 989 datetime_format: Optional[str] = Field( 990 "", 991 description='Format of the datetime value. Defaults to "%Y-%m-%dT%H:%M:%S.%f%z" if left empty. Use placeholders starting with "%" to describe the format the API is using. The following placeholders are available:\n * **%s**: Epoch unix timestamp - `1686218963`\n * **%s_as_float**: Epoch unix timestamp in seconds as float with microsecond precision - `1686218963.123456`\n * **%ms**: Epoch unix timestamp - `1686218963123`\n * **%a**: Weekday (abbreviated) - `Sun`\n * **%A**: Weekday (full) - `Sunday`\n * **%w**: Weekday (decimal) - `0` (Sunday), `6` (Saturday)\n * **%d**: Day of the month (zero-padded) - `01`, `02`, ..., `31`\n * **%b**: Month (abbreviated) - `Jan`\n * **%B**: Month (full) - `January`\n * **%m**: Month (zero-padded) - `01`, `02`, ..., `12`\n * **%y**: Year (without century, zero-padded) - `00`, `01`, ..., `99`\n * **%Y**: Year (with century) - `0001`, `0002`, ..., `9999`\n * **%H**: Hour (24-hour, zero-padded) - `00`, `01`, ..., `23`\n * **%I**: Hour (12-hour, zero-padded) - `01`, `02`, ..., `12`\n * **%p**: AM/PM indicator\n * **%M**: Minute (zero-padded) - `00`, `01`, ..., `59`\n * **%S**: Second (zero-padded) - `00`, `01`, ..., `59`\n * **%f**: Microsecond (zero-padded to 6 digits) - `000000`, `000001`, ..., `999999`\n * **%_ms**: Millisecond (zero-padded to 3 digits) - `000`, `001`, ..., `999`\n * **%z**: UTC offset - `(empty)`, `+0000`, `-04:00`\n * **%Z**: Time zone name - `(empty)`, `UTC`, `GMT`\n * **%j**: Day of the year (zero-padded) - `001`, `002`, ..., `366`\n * **%U**: Week number of the year (Sunday as first day) - `00`, `01`, ..., `53`\n * **%W**: Week number of the year (Monday as first day) - `00`, `01`, ..., `53`\n * **%c**: Date and time representation - `Tue Aug 16 21:30:00 1988`\n * **%x**: Date representation - `08/16/1988`\n * **%X**: Time representation - `21:30:00`\n * **%%**: Literal \'%\' character\n\n Some placeholders depend on the locale of the underlying system - in most cases this locale is configured as en/US. For more information see the [Python documentation](https://docs.python.org/3/library/datetime.html#strftime-and-strptime-format-codes).\n', 992 examples=["%Y-%m-%dT%H:%M:%S.%f%z", "%Y-%m-%d", "%s"], 993 title="Datetime Format", 994 ) 995 max_datetime: Optional[str] = Field( 996 None, 997 description="Ceiling applied on the datetime value. Must be formatted with the datetime_format field.", 998 examples=["2021-01-01T00:00:00Z", "2021-01-01"], 999 title="Max Datetime", 1000 ) 1001 min_datetime: Optional[str] = Field( 1002 None, 1003 description="Floor applied on the datetime value. Must be formatted with the datetime_format field.", 1004 examples=["2010-01-01T00:00:00Z", "2010-01-01"], 1005 title="Min Datetime", 1006 ) 1007 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters")
class
NoAuth(pydantic.v1.main.BaseModel):
class
NoPagination(pydantic.v1.main.BaseModel):
class
State(pydantic.v1.main.BaseModel):
class
State.Config:
class
OauthConnectorInputSpecification(pydantic.v1.main.BaseModel):
1027class OauthConnectorInputSpecification(BaseModel): 1028 class Config: 1029 extra = Extra.allow 1030 1031 consent_url: str = Field( 1032 ..., 1033 description="The DeclarativeOAuth Specific string URL string template to initiate the authentication.\nThe placeholders are replaced during the processing to provide neccessary values.", 1034 examples=[ 1035 "https://domain.host.com/marketing_api/auth?{{client_id_key}}={{client_id_value}}&{{redirect_uri_key}}={{{{redirect_uri_value}} | urlEncoder}}&{{state_key}}={{state_value}}", 1036 "https://endpoint.host.com/oauth2/authorize?{{client_id_key}}={{client_id_value}}&{{redirect_uri_key}}={{{{redirect_uri_value}} | urlEncoder}}&{{scope_key}}={{{{scope_value}} | urlEncoder}}&{{state_key}}={{state_value}}&subdomain={{subdomain}}", 1037 ], 1038 title="Consent URL", 1039 ) 1040 scope: Optional[str] = Field( 1041 None, 1042 description="The DeclarativeOAuth Specific string of the scopes needed to be grant for authenticated user.", 1043 examples=["user:read user:read_orders workspaces:read"], 1044 title="Scopes", 1045 ) 1046 access_token_url: str = Field( 1047 ..., 1048 description="The DeclarativeOAuth Specific URL templated string to obtain the `access_token`, `refresh_token` etc.\nThe placeholders are replaced during the processing to provide neccessary values.", 1049 examples=[ 1050 "https://auth.host.com/oauth2/token?{{client_id_key}}={{client_id_value}}&{{client_secret_key}}={{client_secret_value}}&{{auth_code_key}}={{auth_code_value}}&{{redirect_uri_key}}={{{{redirect_uri_value}} | urlEncoder}}" 1051 ], 1052 title="Access Token URL", 1053 ) 1054 access_token_headers: Optional[Dict[str, Any]] = Field( 1055 None, 1056 description="The DeclarativeOAuth Specific optional headers to inject while exchanging the `auth_code` to `access_token` during `completeOAuthFlow` step.", 1057 examples=[ 1058 { 1059 "Authorization": "Basic {{ {{ client_id_value }}:{{ client_secret_value }} | base64Encoder }}" 1060 } 1061 ], 1062 title="Access Token Headers", 1063 ) 1064 access_token_params: Optional[Dict[str, Any]] = Field( 1065 None, 1066 description="The DeclarativeOAuth Specific optional query parameters to inject while exchanging the `auth_code` to `access_token` during `completeOAuthFlow` step.\nWhen this property is provided, the query params will be encoded as `Json` and included in the outgoing API request.", 1067 examples=[ 1068 { 1069 "{{ auth_code_key }}": "{{ auth_code_value }}", 1070 "{{ client_id_key }}": "{{ client_id_value }}", 1071 "{{ client_secret_key }}": "{{ client_secret_value }}", 1072 } 1073 ], 1074 title="Access Token Query Params (Json Encoded)", 1075 ) 1076 extract_output: Optional[List[str]] = Field( 1077 None, 1078 description="The DeclarativeOAuth Specific list of strings to indicate which keys should be extracted and returned back to the input config.", 1079 examples=[["access_token", "refresh_token", "other_field"]], 1080 title="Extract Output", 1081 ) 1082 state: Optional[State] = Field( 1083 None, 1084 description="The DeclarativeOAuth Specific object to provide the criteria of how the `state` query param should be constructed,\nincluding length and complexity.", 1085 examples=[{"min": 7, "max": 128}], 1086 title="Configurable State Query Param", 1087 ) 1088 client_id_key: Optional[str] = Field( 1089 None, 1090 description="The DeclarativeOAuth Specific optional override to provide the custom `client_id` key name, if required by data-provider.", 1091 examples=["my_custom_client_id_key_name"], 1092 title="Client ID Key Override", 1093 ) 1094 client_secret_key: Optional[str] = Field( 1095 None, 1096 description="The DeclarativeOAuth Specific optional override to provide the custom `client_secret` key name, if required by data-provider.", 1097 examples=["my_custom_client_secret_key_name"], 1098 title="Client Secret Key Override", 1099 ) 1100 scope_key: Optional[str] = Field( 1101 None, 1102 description="The DeclarativeOAuth Specific optional override to provide the custom `scope` key name, if required by data-provider.", 1103 examples=["my_custom_scope_key_key_name"], 1104 title="Scopes Key Override", 1105 ) 1106 state_key: Optional[str] = Field( 1107 None, 1108 description="The DeclarativeOAuth Specific optional override to provide the custom `state` key name, if required by data-provider.", 1109 examples=["my_custom_state_key_key_name"], 1110 title="State Key Override", 1111 ) 1112 auth_code_key: Optional[str] = Field( 1113 None, 1114 description="The DeclarativeOAuth Specific optional override to provide the custom `code` key name to something like `auth_code` or `custom_auth_code`, if required by data-provider.", 1115 examples=["my_custom_auth_code_key_name"], 1116 title="Auth Code Key Override", 1117 ) 1118 redirect_uri_key: Optional[str] = Field( 1119 None, 1120 description="The DeclarativeOAuth Specific optional override to provide the custom `redirect_uri` key name to something like `callback_uri`, if required by data-provider.", 1121 examples=["my_custom_redirect_uri_key_name"], 1122 title="Redirect URI Key Override", 1123 )
state: Optional[State]
class
OauthConnectorInputSpecification.Config:
class
OAuthConfigSpecification(pydantic.v1.main.BaseModel):
1126class OAuthConfigSpecification(BaseModel): 1127 class Config: 1128 extra = Extra.allow 1129 1130 oauth_user_input_from_connector_config_specification: Optional[Dict[str, Any]] = Field( 1131 None, 1132 description="OAuth specific blob. This is a Json Schema used to validate Json configurations used as input to OAuth.\nMust be a valid non-nested JSON that refers to properties from ConnectorSpecification.connectionSpecification\nusing special annotation 'path_in_connector_config'.\nThese are input values the user is entering through the UI to authenticate to the connector, that might also shared\nas inputs for syncing data via the connector.\nExamples:\nif no connector values is shared during oauth flow, oauth_user_input_from_connector_config_specification=[]\nif connector values such as 'app_id' inside the top level are used to generate the API url for the oauth flow,\n oauth_user_input_from_connector_config_specification={\n app_id: {\n type: string\n path_in_connector_config: ['app_id']\n }\n }\nif connector values such as 'info.app_id' nested inside another object are used to generate the API url for the oauth flow,\n oauth_user_input_from_connector_config_specification={\n app_id: {\n type: string\n path_in_connector_config: ['info', 'app_id']\n }\n }", 1133 examples=[ 1134 {"app_id": {"type": "string", "path_in_connector_config": ["app_id"]}}, 1135 { 1136 "app_id": { 1137 "type": "string", 1138 "path_in_connector_config": ["info", "app_id"], 1139 } 1140 }, 1141 ], 1142 title="OAuth user input", 1143 ) 1144 oauth_connector_input_specification: Optional[OauthConnectorInputSpecification] = Field( 1145 None, 1146 description='The DeclarativeOAuth specific blob.\nPertains to the fields defined by the connector relating to the OAuth flow.\n\nInterpolation capabilities:\n- The variables placeholders are declared as `{{my_var}}`.\n- The nested resolution variables like `{{ {{my_nested_var}} }}` is allowed as well.\n\n- The allowed interpolation context is:\n + base64Encoder - encode to `base64`, {{ {{my_var_a}}:{{my_var_b}} | base64Encoder }}\n + base64Decorer - decode from `base64` encoded string, {{ {{my_string_variable_or_string_value}} | base64Decoder }}\n + urlEncoder - encode the input string to URL-like format, {{ https://test.host.com/endpoint | urlEncoder}}\n + urlDecorer - decode the input url-encoded string into text format, {{ urlDecoder:https%3A%2F%2Fairbyte.io | urlDecoder}}\n + codeChallengeS256 - get the `codeChallenge` encoded value to provide additional data-provider specific authorisation values, {{ {{state_value}} | codeChallengeS256 }}\n\nExamples:\n - The TikTok Marketing DeclarativeOAuth spec:\n {\n "oauth_connector_input_specification": {\n "type": "object",\n "additionalProperties": false,\n "properties": {\n "consent_url": "https://ads.tiktok.com/marketing_api/auth?{{client_id_key}}={{client_id_value}}&{{redirect_uri_key}}={{ {{redirect_uri_value}} | urlEncoder}}&{{state_key}}={{state_value}}",\n "access_token_url": "https://business-api.tiktok.com/open_api/v1.3/oauth2/access_token/",\n "access_token_params": {\n "{{ auth_code_key }}": "{{ auth_code_value }}",\n "{{ client_id_key }}": "{{ client_id_value }}",\n "{{ client_secret_key }}": "{{ client_secret_value }}"\n },\n "access_token_headers": {\n "Content-Type": "application/json",\n "Accept": "application/json"\n },\n "extract_output": ["data.access_token"],\n "client_id_key": "app_id",\n "client_secret_key": "secret",\n "auth_code_key": "auth_code"\n }\n }\n }', 1147 title="DeclarativeOAuth Connector Specification", 1148 ) 1149 complete_oauth_output_specification: Optional[Dict[str, Any]] = Field( 1150 None, 1151 description="OAuth specific blob. This is a Json Schema used to validate Json configurations produced by the OAuth flows as they are\nreturned by the distant OAuth APIs.\nMust be a valid JSON describing the fields to merge back to `ConnectorSpecification.connectionSpecification`.\nFor each field, a special annotation `path_in_connector_config` can be specified to determine where to merge it,\nExamples:\n complete_oauth_output_specification={\n refresh_token: {\n type: string,\n path_in_connector_config: ['credentials', 'refresh_token']\n }\n }", 1152 examples=[ 1153 { 1154 "refresh_token": { 1155 "type": "string,", 1156 "path_in_connector_config": ["credentials", "refresh_token"], 1157 } 1158 } 1159 ], 1160 title="OAuth output specification", 1161 ) 1162 complete_oauth_server_input_specification: Optional[Dict[str, Any]] = Field( 1163 None, 1164 description="OAuth specific blob. This is a Json Schema used to validate Json configurations persisted as Airbyte Server configurations.\nMust be a valid non-nested JSON describing additional fields configured by the Airbyte Instance or Workspace Admins to be used by the\nserver when completing an OAuth flow (typically exchanging an auth code for refresh token).\nExamples:\n complete_oauth_server_input_specification={\n client_id: {\n type: string\n },\n client_secret: {\n type: string\n }\n }", 1165 examples=[{"client_id": {"type": "string"}, "client_secret": {"type": "string"}}], 1166 title="OAuth input specification", 1167 ) 1168 complete_oauth_server_output_specification: Optional[Dict[str, Any]] = Field( 1169 None, 1170 description="OAuth specific blob. This is a Json Schema used to validate Json configurations persisted as Airbyte Server configurations that\nalso need to be merged back into the connector configuration at runtime.\nThis is a subset configuration of `complete_oauth_server_input_specification` that filters fields out to retain only the ones that\nare necessary for the connector to function with OAuth. (some fields could be used during oauth flows but not needed afterwards, therefore\nthey would be listed in the `complete_oauth_server_input_specification` but not `complete_oauth_server_output_specification`)\nMust be a valid non-nested JSON describing additional fields configured by the Airbyte Instance or Workspace Admins to be used by the\nconnector when using OAuth flow APIs.\nThese fields are to be merged back to `ConnectorSpecification.connectionSpecification`.\nFor each field, a special annotation `path_in_connector_config` can be specified to determine where to merge it,\nExamples:\n complete_oauth_server_output_specification={\n client_id: {\n type: string,\n path_in_connector_config: ['credentials', 'client_id']\n },\n client_secret: {\n type: string,\n path_in_connector_config: ['credentials', 'client_secret']\n }\n }", 1171 examples=[ 1172 { 1173 "client_id": { 1174 "type": "string,", 1175 "path_in_connector_config": ["credentials", "client_id"], 1176 }, 1177 "client_secret": { 1178 "type": "string,", 1179 "path_in_connector_config": ["credentials", "client_secret"], 1180 }, 1181 } 1182 ], 1183 title="OAuth server output specification", 1184 )
oauth_connector_input_specification: Optional[OauthConnectorInputSpecification]
class
OAuthConfigSpecification.Config:
class
OffsetIncrement(pydantic.v1.main.BaseModel):
1187class OffsetIncrement(BaseModel): 1188 type: Literal["OffsetIncrement"] 1189 page_size: Optional[Union[int, str]] = Field( 1190 None, 1191 description="The number of records to include in each pages.", 1192 examples=[100, "{{ config['page_size'] }}"], 1193 title="Limit", 1194 ) 1195 inject_on_first_request: Optional[bool] = Field( 1196 False, 1197 description="Using the `offset` with value `0` during the first request", 1198 title="Inject Offset", 1199 ) 1200 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters")
class
PageIncrement(pydantic.v1.main.BaseModel):
1203class PageIncrement(BaseModel): 1204 type: Literal["PageIncrement"] 1205 page_size: Optional[Union[int, str]] = Field( 1206 None, 1207 description="The number of records to include in each pages.", 1208 examples=[100, "100", "{{ config['page_size'] }}"], 1209 title="Page Size", 1210 ) 1211 start_from_page: Optional[int] = Field( 1212 0, 1213 description="Index of the first page to request.", 1214 examples=[0, 1], 1215 title="Start From Page", 1216 ) 1217 inject_on_first_request: Optional[bool] = Field( 1218 False, 1219 description="Using the `page number` with value defined by `start_from_page` during the first request", 1220 title="Inject Page Number", 1221 ) 1222 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters")
class
PrimaryKey(pydantic.v1.main.BaseModel):
1225class PrimaryKey(BaseModel): 1226 __root__: Union[str, List[str], List[List[str]]] = Field( 1227 ..., 1228 description="The stream field to be used to distinguish unique records. Can either be a single field, an array of fields representing a composite key, or an array of arrays representing a composite key where the fields are nested fields.", 1229 examples=["id", ["code", "type"]], 1230 title="Primary Key", 1231 )
class
PropertyLimitType(enum.Enum):
1234class PropertyLimitType(Enum): 1235 characters = "characters" 1236 property_count = "property_count"
An enumeration.
characters =
<PropertyLimitType.characters: 'characters'>
property_count =
<PropertyLimitType.property_count: 'property_count'>
class
PropertyChunking(pydantic.v1.main.BaseModel):
1239class PropertyChunking(BaseModel): 1240 type: Literal["PropertyChunking"] 1241 property_limit_type: PropertyLimitType = Field( 1242 ..., 1243 description="The type used to determine the maximum number of properties per chunk", 1244 title="Property Limit Type", 1245 ) 1246 property_limit: Optional[int] = Field( 1247 None, 1248 description="The maximum amount of properties that can be retrieved per request according to the limit type.", 1249 title="Property Limit", 1250 ) 1251 record_merge_strategy: Optional[GroupByKeyMergeStrategy] = Field( 1252 None, 1253 description="Dictates how to records that require multiple requests to get all properties should be emitted to the destination", 1254 title="Record Merge Strategy", 1255 ) 1256 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters")
property_limit_type: PropertyLimitType
record_merge_strategy: Optional[GroupByKeyMergeStrategy]
class
RecordFilter(pydantic.v1.main.BaseModel):
1259class RecordFilter(BaseModel): 1260 type: Literal["RecordFilter"] 1261 condition: Optional[str] = Field( 1262 "", 1263 description="The predicate to filter a record. Records will be removed if evaluated to False.", 1264 examples=[ 1265 "{{ record['created_at'] >= stream_interval['start_time'] }}", 1266 "{{ record.status in ['active', 'expired'] }}", 1267 ], 1268 title="Condition", 1269 ) 1270 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters")
class
SchemaNormalization(enum.Enum):
An enumeration.
Default =
<SchemaNormalization.Default: 'Default'>
None_ =
<SchemaNormalization.None_: 'None'>
class
RemoveFields(pydantic.v1.main.BaseModel):
1278class RemoveFields(BaseModel): 1279 type: Literal["RemoveFields"] 1280 condition: Optional[str] = Field( 1281 "", 1282 description="The predicate to filter a property by a property value. Property will be removed if it is empty OR expression is evaluated to True.,", 1283 examples=[ 1284 "{{ property|string == '' }}", 1285 "{{ property is integer }}", 1286 "{{ property|length > 5 }}", 1287 "{{ property == 'some_string_to_match' }}", 1288 ], 1289 ) 1290 field_pointers: List[List[str]] = Field( 1291 ..., 1292 description="Array of paths defining the field to remove. Each item is an array whose field describe the path of a field to remove.", 1293 examples=[["tags"], [["content", "html"], ["content", "plain_text"]]], 1294 title="Field Paths", 1295 )
class
RequestPath(pydantic.v1.main.BaseModel):
class
InjectInto(enum.Enum):
1302class InjectInto(Enum): 1303 request_parameter = "request_parameter" 1304 header = "header" 1305 body_data = "body_data" 1306 body_json = "body_json"
An enumeration.
request_parameter =
<InjectInto.request_parameter: 'request_parameter'>
header =
<InjectInto.header: 'header'>
body_data =
<InjectInto.body_data: 'body_data'>
body_json =
<InjectInto.body_json: 'body_json'>
class
RequestOption(pydantic.v1.main.BaseModel):
1309class RequestOption(BaseModel): 1310 type: Literal["RequestOption"] 1311 field_name: Optional[str] = Field( 1312 None, 1313 description="Configures which key should be used in the location that the descriptor is being injected into. We hope to eventually deprecate this field in favor of `field_path` for all request_options, but must currently maintain it for backwards compatibility in the Builder.", 1314 examples=["segment_id"], 1315 title="Field Name", 1316 ) 1317 field_path: Optional[List[str]] = Field( 1318 None, 1319 description="Configures a path to be used for nested structures in JSON body requests (e.g. GraphQL queries)", 1320 examples=[["data", "viewer", "id"]], 1321 title="Field Path", 1322 ) 1323 inject_into: InjectInto = Field( 1324 ..., 1325 description="Configures where the descriptor should be set on the HTTP requests. Note that request parameters that are already encoded in the URL path will not be duplicated.", 1326 examples=["request_parameter", "header", "body_data", "body_json"], 1327 title="Inject Into", 1328 )
inject_into: InjectInto
class
Schemas(pydantic.v1.main.BaseModel):
class
Schemas.Config:
class
LegacySessionTokenAuthenticator(pydantic.v1.main.BaseModel):
1338class LegacySessionTokenAuthenticator(BaseModel): 1339 type: Literal["LegacySessionTokenAuthenticator"] 1340 header: str = Field( 1341 ..., 1342 description="The name of the session token header that will be injected in the request", 1343 examples=["X-Session"], 1344 title="Session Request Header", 1345 ) 1346 login_url: str = Field( 1347 ..., 1348 description="Path of the login URL (do not include the base URL)", 1349 examples=["session"], 1350 title="Login Path", 1351 ) 1352 session_token: Optional[str] = Field( 1353 None, 1354 description="Session token to use if using a pre-defined token. Not needed if authenticating with username + password pair", 1355 example=["{{ config['session_token'] }}"], 1356 title="Session Token", 1357 ) 1358 session_token_response_key: str = Field( 1359 ..., 1360 description="Name of the key of the session token to be extracted from the response", 1361 examples=["id"], 1362 title="Response Token Response Key", 1363 ) 1364 username: Optional[str] = Field( 1365 None, 1366 description="Username used to authenticate and obtain a session token", 1367 examples=[" {{ config['username'] }}"], 1368 title="Username", 1369 ) 1370 password: Optional[str] = Field( 1371 "", 1372 description="Password used to authenticate and obtain a session token", 1373 examples=["{{ config['password'] }}", ""], 1374 title="Password", 1375 ) 1376 validate_session_url: str = Field( 1377 ..., 1378 description="Path of the URL to use to validate that the session token is valid (do not include the base URL)", 1379 examples=["user/current"], 1380 title="Validate Session Path", 1381 ) 1382 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters")
class
CsvDecoder(pydantic.v1.main.BaseModel):
class
AsyncJobStatusMap(pydantic.v1.main.BaseModel):
class
ValueType(enum.Enum):
1400class ValueType(Enum): 1401 string = "string" 1402 number = "number" 1403 integer = "integer" 1404 boolean = "boolean"
An enumeration.
string =
<ValueType.string: 'string'>
number =
<ValueType.number: 'number'>
integer =
<ValueType.integer: 'integer'>
boolean =
<ValueType.boolean: 'boolean'>
class
WaitTimeFromHeader(pydantic.v1.main.BaseModel):
1407class WaitTimeFromHeader(BaseModel): 1408 type: Literal["WaitTimeFromHeader"] 1409 header: str = Field( 1410 ..., 1411 description="The name of the response header defining how long to wait before retrying.", 1412 examples=["Retry-After"], 1413 title="Response Header Name", 1414 ) 1415 regex: Optional[str] = Field( 1416 None, 1417 description="Optional regex to apply on the header to extract its value. The regex should define a capture group defining the wait time.", 1418 examples=["([-+]?\\d+)"], 1419 title="Extraction Regex", 1420 ) 1421 max_waiting_time_in_seconds: Optional[float] = Field( 1422 None, 1423 description="Given the value extracted from the header is greater than this value, stop the stream.", 1424 examples=[3600], 1425 title="Max Waiting Time in Seconds", 1426 ) 1427 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters")
class
WaitUntilTimeFromHeader(pydantic.v1.main.BaseModel):
1430class WaitUntilTimeFromHeader(BaseModel): 1431 type: Literal["WaitUntilTimeFromHeader"] 1432 header: str = Field( 1433 ..., 1434 description="The name of the response header defining how long to wait before retrying.", 1435 examples=["wait_time"], 1436 title="Response Header", 1437 ) 1438 min_wait: Optional[Union[float, str]] = Field( 1439 None, 1440 description="Minimum time to wait before retrying.", 1441 examples=[10, "60"], 1442 title="Minimum Wait Time", 1443 ) 1444 regex: Optional[str] = Field( 1445 None, 1446 description="Optional regex to apply on the header to extract its value. The regex should define a capture group defining the wait time.", 1447 examples=["([-+]?\\d+)"], 1448 title="Extraction Regex", 1449 ) 1450 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters")
class
ComponentMappingDefinition(pydantic.v1.main.BaseModel):
1453class ComponentMappingDefinition(BaseModel): 1454 type: Literal["ComponentMappingDefinition"] 1455 field_path: List[str] = Field( 1456 ..., 1457 description="A list of potentially nested fields indicating the full path where value will be added or updated.", 1458 examples=[ 1459 ["data"], 1460 ["data", "records"], 1461 ["data", 1, "name"], 1462 ["data", "{{ components_values.name }}"], 1463 ["data", "*", "record"], 1464 ["*", "**", "name"], 1465 ], 1466 title="Field Path", 1467 ) 1468 value: str = Field( 1469 ..., 1470 description="The dynamic or static value to assign to the key. Interpolated values can be used to dynamically determine the value during runtime.", 1471 examples=[ 1472 "{{ components_values['updates'] }}", 1473 "{{ components_values['MetaData']['LastUpdatedTime'] }}", 1474 "{{ config['segment_id'] }}", 1475 "{{ stream_slice['parent_id'] }}", 1476 "{{ stream_slice['extra_fields']['name'] }}", 1477 ], 1478 title="Value", 1479 ) 1480 value_type: Optional[ValueType] = Field( 1481 None, 1482 description="The expected data type of the value. If omitted, the type will be inferred from the value provided.", 1483 title="Value Type", 1484 ) 1485 create_or_update: Optional[bool] = Field( 1486 False, 1487 description="Determines whether to create a new path if it doesn't exist (true) or only update existing paths (false). When set to true, the resolver will create new paths in the stream template if they don't exist. When false (default), it will only update existing paths.", 1488 title="Create or Update", 1489 ) 1490 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters")
value_type: Optional[ValueType]
class
StreamConfig(pydantic.v1.main.BaseModel):
1493class StreamConfig(BaseModel): 1494 type: Literal["StreamConfig"] 1495 configs_pointer: List[str] = Field( 1496 ..., 1497 description="A list of potentially nested fields indicating the full path in source config file where streams configs located.", 1498 examples=[["data"], ["data", "streams"], ["data", "{{ parameters.name }}"]], 1499 title="Configs Pointer", 1500 ) 1501 default_values: Optional[List[Dict[str, Any]]] = Field( 1502 None, 1503 description="A list of default values, each matching the structure expected from the parsed component value.", 1504 title="Default Values", 1505 ) 1506 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters")
class
ConfigComponentsResolver(pydantic.v1.main.BaseModel):
1509class ConfigComponentsResolver(BaseModel): 1510 type: Literal["ConfigComponentsResolver"] 1511 stream_config: Union[List[StreamConfig], StreamConfig] 1512 components_mapping: List[ComponentMappingDefinition] 1513 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters")
stream_config: Union[List[StreamConfig], StreamConfig]
components_mapping: List[ComponentMappingDefinition]
class
StreamParametersDefinition(pydantic.v1.main.BaseModel):
1516class StreamParametersDefinition(BaseModel): 1517 type: Literal["StreamParametersDefinition"] 1518 list_of_parameters_for_stream: List[Dict[str, Any]] = Field( 1519 ..., 1520 description="A list of object of parameters for stream, each object in the list represents params for one stream.", 1521 examples=[ 1522 [ 1523 { 1524 "name": "test stream", 1525 "$parameters": {"entity": "test entity"}, 1526 "primary_key": "test key", 1527 } 1528 ] 1529 ], 1530 title="Stream Parameters", 1531 )
class
ParametrizedComponentsResolver(pydantic.v1.main.BaseModel):
1534class ParametrizedComponentsResolver(BaseModel): 1535 type: Literal["ParametrizedComponentsResolver"] 1536 stream_parameters: StreamParametersDefinition 1537 components_mapping: List[ComponentMappingDefinition] 1538 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters")
stream_parameters: StreamParametersDefinition
components_mapping: List[ComponentMappingDefinition]
class
RequestBodyPlainText(pydantic.v1.main.BaseModel):
class
RequestBodyUrlEncodedForm(pydantic.v1.main.BaseModel):
class
RequestBodyJsonObject(pydantic.v1.main.BaseModel):
class
RequestBodyGraphQlQuery(pydantic.v1.main.BaseModel):
class
RequestBodyGraphQlQuery.Config:
class
ValidateAdheresToSchema(pydantic.v1.main.BaseModel):
1563class ValidateAdheresToSchema(BaseModel): 1564 type: Literal["ValidateAdheresToSchema"] 1565 base_schema: Union[str, Dict[str, Any]] = Field( 1566 ..., 1567 description="The base JSON schema against which the user-provided schema will be validated.", 1568 examples=[ 1569 "{{ config['report_validation_schema'] }}", 1570 '\'{\n "$schema": "http://json-schema.org/draft-07/schema#",\n "title": "Person",\n "type": "object",\n "properties": {\n "name": {\n "type": "string",\n "description": "The person\'s name"\n },\n "age": {\n "type": "integer",\n "minimum": 0,\n "description": "The person\'s age"\n }\n },\n "required": ["name", "age"]\n}\'\n', 1571 { 1572 "$schema": "http://json-schema.org/draft-07/schema#", 1573 "title": "Person", 1574 "type": "object", 1575 "properties": { 1576 "name": {"type": "string", "description": "The person's name"}, 1577 "age": { 1578 "type": "integer", 1579 "minimum": 0, 1580 "description": "The person's age", 1581 }, 1582 }, 1583 "required": ["name", "age"], 1584 }, 1585 ], 1586 title="Base JSON Schema", 1587 )
class
CustomValidationStrategy(pydantic.v1.main.BaseModel):
1590class CustomValidationStrategy(BaseModel): 1591 class Config: 1592 extra = Extra.allow 1593 1594 type: Literal["CustomValidationStrategy"] 1595 class_name: str = Field( 1596 ..., 1597 description="Fully-qualified name of the class that will be implementing the custom validation strategy. Has to be a sub class of ValidationStrategy. The format is `source_<name>.<package>.<class_name>`.", 1598 examples=["source_declarative_manifest.components.MyCustomValidationStrategy"], 1599 title="Class Name", 1600 )
class
CustomValidationStrategy.Config:
class
ConfigRemapField(pydantic.v1.main.BaseModel):
1603class ConfigRemapField(BaseModel): 1604 type: Literal["ConfigRemapField"] 1605 map: Union[Dict[str, Any], str] = Field( 1606 ..., 1607 description="A mapping of original values to new values. When a field value matches a key in this map, it will be replaced with the corresponding value.", 1608 examples=[ 1609 {"pending": "in_progress", "done": "completed", "cancelled": "terminated"}, 1610 "{{ config['status_mapping'] }}", 1611 ], 1612 title="Value Mapping", 1613 ) 1614 field_path: List[str] = Field( 1615 ..., 1616 description="The path to the field whose value should be remapped. Specified as a list of path components to navigate through nested objects.", 1617 examples=[ 1618 ["status"], 1619 ["data", "status"], 1620 ["data", "{{ config.name }}", "status"], 1621 ["data", "*", "status"], 1622 ], 1623 title="Field Path", 1624 )
class
ConfigRemoveFields(pydantic.v1.main.BaseModel):
1627class ConfigRemoveFields(BaseModel): 1628 type: Literal["ConfigRemoveFields"] 1629 field_pointers: List[List[str]] = Field( 1630 ..., 1631 description="A list of field pointers to be removed from the config.", 1632 examples=[["tags"], [["content", "html"], ["content", "plain_text"]]], 1633 title="Field Pointers", 1634 ) 1635 condition: Optional[str] = Field( 1636 "", 1637 description="Fields will be removed if expression is evaluated to True.", 1638 examples=[ 1639 "{{ config['environemnt'] == 'sandbox' }}", 1640 "{{ property is integer }}", 1641 "{{ property|length > 5 }}", 1642 "{{ property == 'some_string_to_match' }}", 1643 ], 1644 )
class
AddedFieldDefinition(pydantic.v1.main.BaseModel):
1647class AddedFieldDefinition(BaseModel): 1648 type: Literal["AddedFieldDefinition"] 1649 path: List[str] = Field( 1650 ..., 1651 description="List of strings defining the path where to add the value on the record.", 1652 examples=[["segment_id"], ["metadata", "segment_id"]], 1653 title="Path", 1654 ) 1655 value: str = Field( 1656 ..., 1657 description="Value of the new field. Use {{ record['existing_field'] }} syntax to refer to other fields in the record.", 1658 examples=[ 1659 "{{ record['updates'] }}", 1660 "{{ record['MetaData']['LastUpdatedTime'] }}", 1661 "{{ stream_partition['segment_id'] }}", 1662 ], 1663 title="Value", 1664 ) 1665 value_type: Optional[ValueType] = Field( 1666 None, 1667 description="Type of the value. If not specified, the type will be inferred from the value.", 1668 title="Value Type", 1669 ) 1670 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters")
value_type: Optional[ValueType]
class
AddFields(pydantic.v1.main.BaseModel):
1673class AddFields(BaseModel): 1674 type: Literal["AddFields"] 1675 fields: List[AddedFieldDefinition] = Field( 1676 ..., 1677 description="List of transformations (path and corresponding value) that will be added to the record.", 1678 title="Fields", 1679 ) 1680 condition: Optional[str] = Field( 1681 "", 1682 description="Fields will be added if expression is evaluated to True.", 1683 examples=[ 1684 "{{ property|string == '' }}", 1685 "{{ property is integer }}", 1686 "{{ property|length > 5 }}", 1687 "{{ property == 'some_string_to_match' }}", 1688 ], 1689 ) 1690 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters")
fields: List[AddedFieldDefinition]
class
ApiKeyAuthenticator(pydantic.v1.main.BaseModel):
1693class ApiKeyAuthenticator(BaseModel): 1694 type: Literal["ApiKeyAuthenticator"] 1695 api_token: Optional[str] = Field( 1696 None, 1697 description="The API key to inject in the request. Fill it in the user inputs.", 1698 examples=["{{ config['api_key'] }}", "Token token={{ config['api_key'] }}"], 1699 title="API Key", 1700 ) 1701 header: Optional[str] = Field( 1702 None, 1703 description="The name of the HTTP header that will be set to the API key. This setting is deprecated, use inject_into instead. Header and inject_into can not be defined at the same time.", 1704 examples=["Authorization", "Api-Token", "X-Auth-Token"], 1705 title="Header Name", 1706 ) 1707 inject_into: Optional[RequestOption] = Field( 1708 None, 1709 description="Configure how the API Key will be sent in requests to the source API. Either inject_into or header has to be defined.", 1710 examples=[ 1711 {"inject_into": "header", "field_name": "Authorization"}, 1712 {"inject_into": "request_parameter", "field_name": "authKey"}, 1713 ], 1714 title="Inject API Key Into Outgoing HTTP Request", 1715 ) 1716 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters")
inject_into: Optional[RequestOption]
class
AuthFlow(pydantic.v1.main.BaseModel):
1719class AuthFlow(BaseModel): 1720 auth_flow_type: Optional[AuthFlowType] = Field( 1721 None, description="The type of auth to use", title="Auth flow type" 1722 ) 1723 predicate_key: Optional[List[str]] = Field( 1724 None, 1725 description="JSON path to a field in the connectorSpecification that should exist for the advanced auth to be applicable.", 1726 examples=[["credentials", "auth_type"]], 1727 title="Predicate key", 1728 ) 1729 predicate_value: Optional[str] = Field( 1730 None, 1731 description="Value of the predicate_key fields for the advanced auth to be applicable.", 1732 examples=["Oauth"], 1733 title="Predicate value", 1734 ) 1735 oauth_config_specification: Optional[OAuthConfigSpecification] = None
auth_flow_type: Optional[AuthFlowType]
oauth_config_specification: Optional[OAuthConfigSpecification]
class
CheckStream(pydantic.v1.main.BaseModel):
1738class CheckStream(BaseModel): 1739 type: Literal["CheckStream"] 1740 stream_names: Optional[List[str]] = Field( 1741 None, 1742 description="Names of the streams to try reading from when running a check operation.", 1743 examples=[["users"], ["users", "contacts"]], 1744 title="Stream Names", 1745 ) 1746 dynamic_streams_check_configs: Optional[List[DynamicStreamCheckConfig]] = None
dynamic_streams_check_configs: Optional[List[DynamicStreamCheckConfig]]
class
IncrementingCountCursor(pydantic.v1.main.BaseModel):
1749class IncrementingCountCursor(BaseModel): 1750 type: Literal["IncrementingCountCursor"] 1751 cursor_field: str = Field( 1752 ..., 1753 description="The location of the value on a record that will be used as a bookmark during sync. To ensure no data loss, the API must return records in ascending order based on the cursor field. Nested fields are not supported, so the field must be at the top level of the record. You can use a combination of Add Field and Remove Field transformations to move the nested field to the top.", 1754 examples=["created_at", "{{ config['record_cursor'] }}"], 1755 title="Cursor Field", 1756 ) 1757 start_value: Optional[Union[str, int]] = Field( 1758 None, 1759 description="The value that determines the earliest record that should be synced.", 1760 examples=[0, "{{ config['start_value'] }}"], 1761 title="Start Value", 1762 ) 1763 start_value_option: Optional[RequestOption] = Field( 1764 None, 1765 description="Optionally configures how the start value will be sent in requests to the source API.", 1766 title="Inject Start Value Into Outgoing HTTP Request", 1767 ) 1768 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters")
start_value_option: Optional[RequestOption]
class
DatetimeBasedCursor(pydantic.v1.main.BaseModel):
1771class DatetimeBasedCursor(BaseModel): 1772 type: Literal["DatetimeBasedCursor"] 1773 clamping: Optional[Clamping] = Field( 1774 None, 1775 description="This option is used to adjust the upper and lower boundaries of each datetime window to beginning and end of the provided target period (day, week, month)", 1776 title="Date Range Clamping", 1777 ) 1778 cursor_field: str = Field( 1779 ..., 1780 description="The location of the value on a record that will be used as a bookmark during sync. To ensure no data loss, the API must return records in ascending order based on the cursor field. Nested fields are not supported, so the field must be at the top level of the record. You can use a combination of Add Field and Remove Field transformations to move the nested field to the top.", 1781 examples=["created_at", "{{ config['record_cursor'] }}"], 1782 title="Cursor Field", 1783 ) 1784 cursor_datetime_formats: Optional[List[str]] = Field( 1785 None, 1786 description="The possible formats for the cursor field, in order of preference. The first format that matches the cursor field value will be used to parse it. If not provided, the `datetime_format` will be used.", 1787 title="Cursor Datetime Formats", 1788 ) 1789 start_datetime: Union[MinMaxDatetime, str] = Field( 1790 ..., 1791 description="The datetime that determines the earliest record that should be synced.", 1792 examples=["2020-01-1T00:00:00Z", "{{ config['start_time'] }}"], 1793 title="Start Datetime", 1794 ) 1795 start_time_option: Optional[RequestOption] = Field( 1796 None, 1797 description="Optionally configures how the start datetime will be sent in requests to the source API.", 1798 title="Inject Start Time Into Outgoing HTTP Request", 1799 ) 1800 end_datetime: Optional[Union[MinMaxDatetime, str]] = Field( 1801 None, 1802 description="The datetime that determines the last record that should be synced. If not provided, `{{ now_utc() }}` will be used.", 1803 examples=["2021-01-1T00:00:00Z", "{{ now_utc() }}", "{{ day_delta(-1) }}"], 1804 title="End Datetime", 1805 ) 1806 end_time_option: Optional[RequestOption] = Field( 1807 None, 1808 description="Optionally configures how the end datetime will be sent in requests to the source API.", 1809 title="Inject End Time Into Outgoing HTTP Request", 1810 ) 1811 datetime_format: str = Field( 1812 ..., 1813 description="The datetime format used to format the datetime values that are sent in outgoing requests to the API. Use placeholders starting with \"%\" to describe the format the API is using. The following placeholders are available:\n * **%s**: Epoch unix timestamp - `1686218963`\n * **%s_as_float**: Epoch unix timestamp in seconds as float with microsecond precision - `1686218963.123456`\n * **%ms**: Epoch unix timestamp (milliseconds) - `1686218963123`\n * **%a**: Weekday (abbreviated) - `Sun`\n * **%A**: Weekday (full) - `Sunday`\n * **%w**: Weekday (decimal) - `0` (Sunday), `6` (Saturday)\n * **%d**: Day of the month (zero-padded) - `01`, `02`, ..., `31`\n * **%b**: Month (abbreviated) - `Jan`\n * **%B**: Month (full) - `January`\n * **%m**: Month (zero-padded) - `01`, `02`, ..., `12`\n * **%y**: Year (without century, zero-padded) - `00`, `01`, ..., `99`\n * **%Y**: Year (with century) - `0001`, `0002`, ..., `9999`\n * **%H**: Hour (24-hour, zero-padded) - `00`, `01`, ..., `23`\n * **%I**: Hour (12-hour, zero-padded) - `01`, `02`, ..., `12`\n * **%p**: AM/PM indicator\n * **%M**: Minute (zero-padded) - `00`, `01`, ..., `59`\n * **%S**: Second (zero-padded) - `00`, `01`, ..., `59`\n * **%f**: Microsecond (zero-padded to 6 digits) - `000000`\n * **%_ms**: Millisecond (zero-padded to 3 digits) - `000`\n * **%z**: UTC offset - `(empty)`, `+0000`, `-04:00`\n * **%Z**: Time zone name - `(empty)`, `UTC`, `GMT`\n * **%j**: Day of the year (zero-padded) - `001`, `002`, ..., `366`\n * **%U**: Week number of the year (starting Sunday) - `00`, ..., `53`\n * **%W**: Week number of the year (starting Monday) - `00`, ..., `53`\n * **%c**: Date and time - `Tue Aug 16 21:30:00 1988`\n * **%x**: Date standard format - `08/16/1988`\n * **%X**: Time standard format - `21:30:00`\n * **%%**: Literal '%' character\n\n Some placeholders depend on the locale of the underlying system - in most cases this locale is configured as en/US. For more information see the [Python documentation](https://docs.python.org/3/library/datetime.html#strftime-and-strptime-format-codes).\n", 1814 examples=["%Y-%m-%dT%H:%M:%S.%f%z", "%Y-%m-%d", "%s", "%ms", "%s_as_float"], 1815 title="Outgoing Datetime Format", 1816 ) 1817 cursor_granularity: Optional[str] = Field( 1818 None, 1819 description="Smallest increment the datetime_format has (ISO 8601 duration) that is used to ensure the start of a slice does not overlap with the end of the previous one, e.g. for %Y-%m-%d the granularity should be P1D, for %Y-%m-%dT%H:%M:%SZ the granularity should be PT1S. Given this field is provided, `step` needs to be provided as well.", 1820 examples=["PT1S"], 1821 title="Cursor Granularity", 1822 ) 1823 is_data_feed: Optional[bool] = Field( 1824 None, 1825 description="A data feed API is an API that does not allow filtering and paginates the content from the most recent to the least recent. Given this, the CDK needs to know when to stop paginating and this field will generate a stop condition for pagination.", 1826 title="Whether the target API is formatted as a data feed", 1827 ) 1828 is_client_side_incremental: Optional[bool] = Field( 1829 None, 1830 description="If the target API endpoint does not take cursor values to filter records and returns all records anyway, the connector with this cursor will filter out records locally, and only emit new records from the last sync, hence incremental. This means that all records would be read from the API, but only new records will be emitted to the destination.", 1831 title="Whether the target API does not support filtering and returns all data (the cursor filters records in the client instead of the API side)", 1832 ) 1833 is_compare_strictly: Optional[bool] = Field( 1834 False, 1835 description="Set to True if the target API does not accept queries where the start time equal the end time.", 1836 title="Whether to skip requests if the start time equals the end time", 1837 ) 1838 global_substream_cursor: Optional[bool] = Field( 1839 False, 1840 description="This setting optimizes performance when the parent stream has thousands of partitions by storing the cursor as a single value rather than per partition. Notably, the substream state is updated only at the end of the sync, which helps prevent data loss in case of a sync failure. See more info in the [docs](https://docs.airbyte.com/connector-development/config-based/understanding-the-yaml-file/incremental-syncs).", 1841 title="Whether to store cursor as one value instead of per partition", 1842 ) 1843 lookback_window: Optional[str] = Field( 1844 None, 1845 description="Time interval before the start_datetime to read data for, e.g. P1M for looking back one month.", 1846 examples=["P1D", "P{{ config['lookback_days'] }}D"], 1847 title="Lookback Window", 1848 ) 1849 partition_field_end: Optional[str] = Field( 1850 None, 1851 description="Name of the partition start time field.", 1852 examples=["ending_time"], 1853 title="Partition Field End", 1854 ) 1855 partition_field_start: Optional[str] = Field( 1856 None, 1857 description="Name of the partition end time field.", 1858 examples=["starting_time"], 1859 title="Partition Field Start", 1860 ) 1861 step: Optional[str] = Field( 1862 None, 1863 description="The size of the time window (ISO8601 duration). Given this field is provided, `cursor_granularity` needs to be provided as well.", 1864 examples=["P1W", "{{ config['step_increment'] }}"], 1865 title="Step", 1866 ) 1867 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters")
clamping: Optional[Clamping]
start_datetime: Union[MinMaxDatetime, str]
start_time_option: Optional[RequestOption]
end_datetime: Union[MinMaxDatetime, str, NoneType]
end_time_option: Optional[RequestOption]
class
FixedWindowCallRatePolicy(pydantic.v1.main.BaseModel):
1870class FixedWindowCallRatePolicy(BaseModel): 1871 class Config: 1872 extra = Extra.allow 1873 1874 type: Literal["FixedWindowCallRatePolicy"] 1875 period: str = Field( 1876 ..., description="The time interval for the rate limit window.", title="Period" 1877 ) 1878 call_limit: int = Field( 1879 ..., 1880 description="The maximum number of calls allowed within the period.", 1881 title="Call Limit", 1882 ) 1883 matchers: List[HttpRequestRegexMatcher] = Field( 1884 ..., 1885 description="List of matchers that define which requests this policy applies to.", 1886 title="Matchers", 1887 )
matchers: List[HttpRequestRegexMatcher]
class
FixedWindowCallRatePolicy.Config:
class
MovingWindowCallRatePolicy(pydantic.v1.main.BaseModel):
1890class MovingWindowCallRatePolicy(BaseModel): 1891 class Config: 1892 extra = Extra.allow 1893 1894 type: Literal["MovingWindowCallRatePolicy"] 1895 rates: List[Rate] = Field( 1896 ..., 1897 description="List of rates that define the call limits for different time intervals.", 1898 title="Rates", 1899 ) 1900 matchers: List[HttpRequestRegexMatcher] = Field( 1901 ..., 1902 description="List of matchers that define which requests this policy applies to.", 1903 title="Matchers", 1904 )
rates: List[Rate]
matchers: List[HttpRequestRegexMatcher]
class
MovingWindowCallRatePolicy.Config:
class
UnlimitedCallRatePolicy(pydantic.v1.main.BaseModel):
1907class UnlimitedCallRatePolicy(BaseModel): 1908 class Config: 1909 extra = Extra.allow 1910 1911 type: Literal["UnlimitedCallRatePolicy"] 1912 matchers: List[HttpRequestRegexMatcher] = Field( 1913 ..., 1914 description="List of matchers that define which requests this policy applies to.", 1915 title="Matchers", 1916 )
matchers: List[HttpRequestRegexMatcher]
class
UnlimitedCallRatePolicy.Config:
class
DefaultErrorHandler(pydantic.v1.main.BaseModel):
1919class DefaultErrorHandler(BaseModel): 1920 type: Literal["DefaultErrorHandler"] 1921 backoff_strategies: Optional[ 1922 List[ 1923 Union[ 1924 ConstantBackoffStrategy, 1925 ExponentialBackoffStrategy, 1926 WaitTimeFromHeader, 1927 WaitUntilTimeFromHeader, 1928 CustomBackoffStrategy, 1929 ] 1930 ] 1931 ] = Field( 1932 None, 1933 description="List of backoff strategies to use to determine how long to wait before retrying a retryable request.", 1934 title="Backoff Strategies", 1935 ) 1936 max_retries: Optional[int] = Field( 1937 5, 1938 description="The maximum number of time to retry a retryable request before giving up and failing.", 1939 examples=[5, 0, 10], 1940 title="Max Retry Count", 1941 ) 1942 response_filters: Optional[List[HttpResponseFilter]] = Field( 1943 None, 1944 description="List of response filters to iterate on when deciding how to handle an error. When using an array of multiple filters, the filters will be applied sequentially and the response will be selected if it matches any of the filter's predicate.", 1945 title="Response Filters", 1946 ) 1947 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters")
backoff_strategies: Optional[List[Union[ConstantBackoffStrategy, ExponentialBackoffStrategy, WaitTimeFromHeader, WaitUntilTimeFromHeader, CustomBackoffStrategy]]]
response_filters: Optional[List[HttpResponseFilter]]
class
DefaultPaginator(pydantic.v1.main.BaseModel):
1950class DefaultPaginator(BaseModel): 1951 type: Literal["DefaultPaginator"] 1952 pagination_strategy: Union[ 1953 PageIncrement, OffsetIncrement, CursorPagination, CustomPaginationStrategy 1954 ] = Field( 1955 ..., 1956 description="Strategy defining how records are paginated.", 1957 title="Pagination Strategy", 1958 ) 1959 page_size_option: Optional[RequestOption] = Field( 1960 None, title="Inject Page Size Into Outgoing HTTP Request" 1961 ) 1962 page_token_option: Optional[Union[RequestOption, RequestPath]] = Field( 1963 None, title="Inject Page Token Into Outgoing HTTP Request" 1964 ) 1965 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters")
pagination_strategy: Union[PageIncrement, OffsetIncrement, CursorPagination, CustomPaginationStrategy]
page_size_option: Optional[RequestOption]
page_token_option: Union[RequestOption, RequestPath, NoneType]
class
SessionTokenRequestApiKeyAuthenticator(pydantic.v1.main.BaseModel):
1968class SessionTokenRequestApiKeyAuthenticator(BaseModel): 1969 type: Literal["ApiKey"] 1970 inject_into: RequestOption = Field( 1971 ..., 1972 description="Configure how the API Key will be sent in requests to the source API.", 1973 examples=[ 1974 {"inject_into": "header", "field_name": "Authorization"}, 1975 {"inject_into": "request_parameter", "field_name": "authKey"}, 1976 ], 1977 title="Inject API Key Into Outgoing HTTP Request", 1978 )
inject_into: RequestOption
class
ListPartitionRouter(pydantic.v1.main.BaseModel):
1981class ListPartitionRouter(BaseModel): 1982 type: Literal["ListPartitionRouter"] 1983 cursor_field: str = Field( 1984 ..., 1985 description='While iterating over list values, the name of field used to reference a list value. The partition value can be accessed with string interpolation. e.g. "{{ stream_partition[\'my_key\'] }}" where "my_key" is the value of the cursor_field.', 1986 examples=["section", "{{ config['section_key'] }}"], 1987 title="Current Partition Value Identifier", 1988 ) 1989 values: Union[str, List[str]] = Field( 1990 ..., 1991 description="The list of attributes being iterated over and used as input for the requests made to the source API.", 1992 examples=[["section_a", "section_b", "section_c"], "{{ config['sections'] }}"], 1993 title="Partition Values", 1994 ) 1995 request_option: Optional[RequestOption] = Field( 1996 None, 1997 description="A request option describing where the list value should be injected into and under what field name if applicable.", 1998 title="Inject Partition Value Into Outgoing HTTP Request", 1999 ) 2000 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters")
request_option: Optional[RequestOption]
class
RecordSelector(pydantic.v1.main.BaseModel):
2003class RecordSelector(BaseModel): 2004 type: Literal["RecordSelector"] 2005 extractor: Union[DpathExtractor, CustomRecordExtractor] 2006 record_filter: Optional[Union[RecordFilter, CustomRecordFilter]] = Field( 2007 None, 2008 description="Responsible for filtering records to be emitted by the Source.", 2009 title="Record Filter", 2010 ) 2011 schema_normalization: Optional[Union[SchemaNormalization, CustomSchemaNormalization]] = Field( 2012 None, 2013 description="Responsible for normalization according to the schema.", 2014 title="Schema Normalization", 2015 ) 2016 transform_before_filtering: Optional[bool] = Field( 2017 None, 2018 description="If true, transformation will be applied before record filtering.", 2019 title="Transform Before Filtering", 2020 ) 2021 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters")
extractor: Union[DpathExtractor, CustomRecordExtractor]
record_filter: Union[RecordFilter, CustomRecordFilter, NoneType]
schema_normalization: Union[SchemaNormalization, CustomSchemaNormalization, NoneType]
class
GzipDecoder(pydantic.v1.main.BaseModel):
2024class GzipDecoder(BaseModel): 2025 type: Literal["GzipDecoder"] 2026 decoder: Union[CsvDecoder, GzipDecoder, JsonDecoder, JsonlDecoder]
decoder: Union[CsvDecoder, GzipDecoder, JsonDecoder, JsonlDecoder]
class
RequestBodyGraphQL(pydantic.v1.main.BaseModel):
2029class RequestBodyGraphQL(BaseModel): 2030 type: Literal["RequestBodyGraphQL"] 2031 value: RequestBodyGraphQlQuery
value: RequestBodyGraphQlQuery
class
DpathValidator(pydantic.v1.main.BaseModel):
2034class DpathValidator(BaseModel): 2035 type: Literal["DpathValidator"] 2036 field_path: List[str] = Field( 2037 ..., 2038 description='List of potentially nested fields describing the full path of the field to validate. Use "*" to validate all values from an array.', 2039 examples=[ 2040 ["data"], 2041 ["data", "records"], 2042 ["data", "{{ parameters.name }}"], 2043 ["data", "*", "record"], 2044 ], 2045 title="Field Path", 2046 ) 2047 validation_strategy: Union[ValidateAdheresToSchema, CustomValidationStrategy] = Field( 2048 ..., 2049 description="The condition that the specified config value will be evaluated against", 2050 title="Validation Strategy", 2051 )
validation_strategy: Union[ValidateAdheresToSchema, CustomValidationStrategy]
class
PredicateValidator(pydantic.v1.main.BaseModel):
2054class PredicateValidator(BaseModel): 2055 type: Literal["PredicateValidator"] 2056 value: Optional[Union[str, float, Dict[str, Any], List[Any], bool]] = Field( 2057 ..., 2058 description="The value to be validated. Can be a literal value or interpolated from configuration.", 2059 examples=[ 2060 "test-value", 2061 "{{ config['api_version'] }}", 2062 "{{ config['tenant_id'] }}", 2063 123, 2064 ], 2065 title="Value", 2066 ) 2067 validation_strategy: Union[ValidateAdheresToSchema, CustomValidationStrategy] = Field( 2068 ..., 2069 description="The validation strategy to apply to the value.", 2070 title="Validation Strategy", 2071 )
validation_strategy: Union[ValidateAdheresToSchema, CustomValidationStrategy]
class
ConfigAddFields(pydantic.v1.main.BaseModel):
2074class ConfigAddFields(BaseModel): 2075 type: Literal["ConfigAddFields"] 2076 fields: List[AddedFieldDefinition] = Field( 2077 ..., 2078 description="A list of transformations (path and corresponding value) that will be added to the config.", 2079 title="Fields", 2080 ) 2081 condition: Optional[str] = Field( 2082 "", 2083 description="Fields will be added if expression is evaluated to True.", 2084 examples=[ 2085 "{{ config['environemnt'] == 'sandbox' }}", 2086 "{{ property is integer }}", 2087 "{{ property|length > 5 }}", 2088 "{{ property == 'some_string_to_match' }}", 2089 ], 2090 )
fields: List[AddedFieldDefinition]
class
CompositeErrorHandler(pydantic.v1.main.BaseModel):
2093class CompositeErrorHandler(BaseModel): 2094 type: Literal["CompositeErrorHandler"] 2095 error_handlers: List[Union[CompositeErrorHandler, DefaultErrorHandler]] = Field( 2096 ..., 2097 description="List of error handlers to iterate on to determine how to handle a failed response.", 2098 title="Error Handlers", 2099 ) 2100 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters")
error_handlers: List[Union[CompositeErrorHandler, DefaultErrorHandler]]
class
HTTPAPIBudget(pydantic.v1.main.BaseModel):
2103class HTTPAPIBudget(BaseModel): 2104 class Config: 2105 extra = Extra.allow 2106 2107 type: Literal["HTTPAPIBudget"] 2108 policies: List[ 2109 Union[ 2110 FixedWindowCallRatePolicy, 2111 MovingWindowCallRatePolicy, 2112 UnlimitedCallRatePolicy, 2113 ] 2114 ] = Field( 2115 ..., 2116 description="List of call rate policies that define how many calls are allowed.", 2117 title="Policies", 2118 ) 2119 ratelimit_reset_header: Optional[str] = Field( 2120 "ratelimit-reset", 2121 description="The HTTP response header name that indicates when the rate limit resets.", 2122 title="Rate Limit Reset Header", 2123 ) 2124 ratelimit_remaining_header: Optional[str] = Field( 2125 "ratelimit-remaining", 2126 description="The HTTP response header name that indicates the number of remaining allowed calls.", 2127 title="Rate Limit Remaining Header", 2128 ) 2129 status_codes_for_ratelimit_hit: Optional[List[int]] = Field( 2130 [429], 2131 description="List of HTTP status codes that indicate a rate limit has been hit.", 2132 title="Status Codes for Rate Limit Hit", 2133 )
policies: List[Union[FixedWindowCallRatePolicy, MovingWindowCallRatePolicy, UnlimitedCallRatePolicy]]
class
HTTPAPIBudget.Config:
class
ZipfileDecoder(pydantic.v1.main.BaseModel):
2136class ZipfileDecoder(BaseModel): 2137 class Config: 2138 extra = Extra.allow 2139 2140 type: Literal["ZipfileDecoder"] 2141 decoder: Union[CsvDecoder, GzipDecoder, JsonDecoder, JsonlDecoder] = Field( 2142 ..., 2143 description="Parser to parse the decompressed data from the zipfile(s).", 2144 title="Parser", 2145 )
decoder: Union[CsvDecoder, GzipDecoder, JsonDecoder, JsonlDecoder]
class
ZipfileDecoder.Config:
class
ConfigMigration(pydantic.v1.main.BaseModel):
2148class ConfigMigration(BaseModel): 2149 type: Literal["ConfigMigration"] 2150 description: Optional[str] = Field( 2151 None, description="The description/purpose of the config migration." 2152 ) 2153 transformations: List[Union[ConfigRemapField, ConfigAddFields, ConfigRemoveFields]] = Field( 2154 ..., 2155 description="The list of transformations that will attempt to be applied on an incoming unmigrated config. The transformations will be applied in the order they are defined.", 2156 title="Transformations", 2157 )
transformations: List[Union[ConfigRemapField, ConfigAddFields, ConfigRemoveFields]]
class
ConfigNormalizationRules(pydantic.v1.main.BaseModel):
2160class ConfigNormalizationRules(BaseModel): 2161 class Config: 2162 extra = Extra.forbid 2163 2164 config_migrations: Optional[List[ConfigMigration]] = Field( 2165 [], 2166 description="The discrete migrations that will be applied on the incoming config. Each migration will be applied in the order they are defined.", 2167 title="Config Migrations", 2168 ) 2169 transformations: Optional[ 2170 List[Union[ConfigRemapField, ConfigAddFields, ConfigRemoveFields]] 2171 ] = Field( 2172 [], 2173 description="The list of transformations that will be applied on the incoming config at the start of each sync. The transformations will be applied in the order they are defined.", 2174 title="Transformations", 2175 ) 2176 validations: Optional[List[Union[DpathValidator, PredicateValidator]]] = Field( 2177 [], 2178 description="The list of validations that will be performed on the incoming config at the start of each sync.", 2179 title="Validations", 2180 )
config_migrations: Optional[List[ConfigMigration]]
transformations: Optional[List[Union[ConfigRemapField, ConfigAddFields, ConfigRemoveFields]]]
validations: Optional[List[Union[DpathValidator, PredicateValidator]]]
class
ConfigNormalizationRules.Config:
class
Spec(pydantic.v1.main.BaseModel):
2183class Spec(BaseModel): 2184 type: Literal["Spec"] 2185 connection_specification: Dict[str, Any] = Field( 2186 ..., 2187 description="A connection specification describing how a the connector can be configured.", 2188 title="Connection Specification", 2189 ) 2190 documentation_url: Optional[str] = Field( 2191 None, 2192 description="URL of the connector's documentation page.", 2193 examples=["https://docs.airbyte.com/integrations/sources/dremio"], 2194 title="Documentation URL", 2195 ) 2196 advanced_auth: Optional[AuthFlow] = Field( 2197 None, 2198 description="Advanced specification for configuring the authentication flow.", 2199 title="Advanced Auth", 2200 ) 2201 config_normalization_rules: Optional[ConfigNormalizationRules] = Field( 2202 None, title="Config Normalization Rules" 2203 )
advanced_auth: Optional[AuthFlow]
config_normalization_rules: Optional[ConfigNormalizationRules]
class
DeclarativeSource1(pydantic.v1.main.BaseModel):
2206class DeclarativeSource1(BaseModel): 2207 class Config: 2208 extra = Extra.forbid 2209 2210 type: Literal["DeclarativeSource"] 2211 check: Union[CheckStream, CheckDynamicStream] 2212 streams: List[Union[ConditionalStreams, DeclarativeStream, StateDelegatingStream]] 2213 dynamic_streams: Optional[List[DynamicDeclarativeStream]] = None 2214 version: str = Field( 2215 ..., 2216 description="The version of the Airbyte CDK used to build and test the source.", 2217 ) 2218 schemas: Optional[Schemas] = None 2219 definitions: Optional[Dict[str, Any]] = None 2220 spec: Optional[Spec] = None 2221 concurrency_level: Optional[ConcurrencyLevel] = None 2222 api_budget: Optional[HTTPAPIBudget] = None 2223 max_concurrent_async_job_count: Optional[Union[int, str]] = Field( 2224 None, 2225 description="Maximum number of concurrent asynchronous jobs to run. This property is only relevant for sources/streams that support asynchronous job execution through the AsyncRetriever (e.g. a report-based stream that initiates a job, polls the job status, and then fetches the job results). This is often set by the API's maximum number of concurrent jobs on the account level. Refer to the API's documentation for this information.", 2226 examples=[3, "{{ config['max_concurrent_async_job_count'] }}"], 2227 title="Maximum Concurrent Asynchronous Jobs", 2228 ) 2229 metadata: Optional[Dict[str, Any]] = Field( 2230 None, 2231 description="For internal Airbyte use only - DO NOT modify manually. Used by consumers of declarative manifests for storing related metadata.", 2232 ) 2233 description: Optional[str] = Field( 2234 None, 2235 description="A description of the connector. It will be presented on the Source documentation page.", 2236 )
check: Union[CheckStream, CheckDynamicStream]
streams: List[Union[ConditionalStreams, DeclarativeStream, StateDelegatingStream]]
dynamic_streams: Optional[List[DynamicDeclarativeStream]]
schemas: Optional[Schemas]
spec: Optional[Spec]
concurrency_level: Optional[ConcurrencyLevel]
api_budget: Optional[HTTPAPIBudget]
class
DeclarativeSource1.Config:
class
DeclarativeSource2(pydantic.v1.main.BaseModel):
2239class DeclarativeSource2(BaseModel): 2240 class Config: 2241 extra = Extra.forbid 2242 2243 type: Literal["DeclarativeSource"] 2244 check: Union[CheckStream, CheckDynamicStream] 2245 streams: Optional[List[Union[ConditionalStreams, DeclarativeStream, StateDelegatingStream]]] = ( 2246 None 2247 ) 2248 dynamic_streams: List[DynamicDeclarativeStream] 2249 version: str = Field( 2250 ..., 2251 description="The version of the Airbyte CDK used to build and test the source.", 2252 ) 2253 schemas: Optional[Schemas] = None 2254 definitions: Optional[Dict[str, Any]] = None 2255 spec: Optional[Spec] = None 2256 concurrency_level: Optional[ConcurrencyLevel] = None 2257 api_budget: Optional[HTTPAPIBudget] = None 2258 max_concurrent_async_job_count: Optional[Union[int, str]] = Field( 2259 None, 2260 description="Maximum number of concurrent asynchronous jobs to run. This property is only relevant for sources/streams that support asynchronous job execution through the AsyncRetriever (e.g. a report-based stream that initiates a job, polls the job status, and then fetches the job results). This is often set by the API's maximum number of concurrent jobs on the account level. Refer to the API's documentation for this information.", 2261 examples=[3, "{{ config['max_concurrent_async_job_count'] }}"], 2262 title="Maximum Concurrent Asynchronous Jobs", 2263 ) 2264 metadata: Optional[Dict[str, Any]] = Field( 2265 None, 2266 description="For internal Airbyte use only - DO NOT modify manually. Used by consumers of declarative manifests for storing related metadata.", 2267 ) 2268 description: Optional[str] = Field( 2269 None, 2270 description="A description of the connector. It will be presented on the Source documentation page.", 2271 )
check: Union[CheckStream, CheckDynamicStream]
streams: Optional[List[Union[ConditionalStreams, DeclarativeStream, StateDelegatingStream]]]
dynamic_streams: List[DynamicDeclarativeStream]
schemas: Optional[Schemas]
spec: Optional[Spec]
concurrency_level: Optional[ConcurrencyLevel]
api_budget: Optional[HTTPAPIBudget]
class
DeclarativeSource2.Config:
class
DeclarativeSource(pydantic.v1.main.BaseModel):
2274class DeclarativeSource(BaseModel): 2275 class Config: 2276 extra = Extra.forbid 2277 2278 __root__: Union[DeclarativeSource1, DeclarativeSource2] = Field( 2279 ..., 2280 description="An API source that extracts data according to its declarative components.", 2281 title="DeclarativeSource", 2282 )
class
DeclarativeSource.Config:
class
SelectiveAuthenticator(pydantic.v1.main.BaseModel):
2285class SelectiveAuthenticator(BaseModel): 2286 class Config: 2287 extra = Extra.allow 2288 2289 type: Literal["SelectiveAuthenticator"] 2290 authenticator_selection_path: List[str] = Field( 2291 ..., 2292 description="Path of the field in config with selected authenticator name", 2293 examples=[["auth"], ["auth", "type"]], 2294 title="Authenticator Selection Path", 2295 ) 2296 authenticators: Dict[ 2297 str, 2298 Union[ 2299 ApiKeyAuthenticator, 2300 BasicHttpAuthenticator, 2301 BearerAuthenticator, 2302 OAuthAuthenticator, 2303 JwtAuthenticator, 2304 SessionTokenAuthenticator, 2305 LegacySessionTokenAuthenticator, 2306 CustomAuthenticator, 2307 NoAuth, 2308 ], 2309 ] = Field( 2310 ..., 2311 description="Authenticators to select from.", 2312 examples=[ 2313 { 2314 "authenticators": { 2315 "token": "#/definitions/ApiKeyAuthenticator", 2316 "oauth": "#/definitions/OAuthAuthenticator", 2317 "jwt": "#/definitions/JwtAuthenticator", 2318 } 2319 } 2320 ], 2321 title="Authenticators", 2322 ) 2323 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters")
authenticators: Dict[str, Union[ApiKeyAuthenticator, BasicHttpAuthenticator, BearerAuthenticator, OAuthAuthenticator, JwtAuthenticator, SessionTokenAuthenticator, LegacySessionTokenAuthenticator, CustomAuthenticator, NoAuth]]
class
SelectiveAuthenticator.Config:
class
ConditionalStreams(pydantic.v1.main.BaseModel):
2326class ConditionalStreams(BaseModel): 2327 type: Literal["ConditionalStreams"] 2328 condition: str = Field( 2329 ..., 2330 description="Condition that will be evaluated to determine if a set of streams should be available.", 2331 examples=["{{ config['is_sandbox'] }}"], 2332 title="Condition", 2333 ) 2334 streams: List[DeclarativeStream] = Field( 2335 ..., 2336 description="Streams that will be used during an operation based on the condition.", 2337 title="Streams", 2338 ) 2339 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters")
streams: List[DeclarativeStream]
class
FileUploader(pydantic.v1.main.BaseModel):
2342class FileUploader(BaseModel): 2343 type: Literal["FileUploader"] 2344 requester: Union[HttpRequester, CustomRequester] = Field( 2345 ..., 2346 description="Requester component that describes how to prepare HTTP requests to send to the source API.", 2347 ) 2348 download_target_extractor: Union[DpathExtractor, CustomRecordExtractor] = Field( 2349 ..., 2350 description="Responsible for fetching the url where the file is located. This is applied on each records and not on the HTTP response", 2351 ) 2352 file_extractor: Optional[Union[DpathExtractor, CustomRecordExtractor]] = Field( 2353 None, 2354 description="Responsible for fetching the content of the file. If not defined, the assumption is that the whole response body is the file content", 2355 ) 2356 filename_extractor: Optional[str] = Field( 2357 None, 2358 description="Defines the name to store the file. Stream name is automatically added to the file path. File unique ID can be used to avoid overwriting files. Random UUID will be used if the extractor is not provided.", 2359 examples=[ 2360 "{{ record.id }}/{{ record.file_name }}/", 2361 "{{ record.id }}_{{ record.file_name }}/", 2362 ], 2363 ) 2364 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters")
requester: Union[HttpRequester, CustomRequester]
download_target_extractor: Union[DpathExtractor, CustomRecordExtractor]
file_extractor: Union[DpathExtractor, CustomRecordExtractor, NoneType]
class
DeclarativeStream(pydantic.v1.main.BaseModel):
2367class DeclarativeStream(BaseModel): 2368 class Config: 2369 extra = Extra.allow 2370 2371 type: Literal["DeclarativeStream"] 2372 name: Optional[str] = Field("", description="The stream name.", example=["Users"], title="Name") 2373 retriever: Union[SimpleRetriever, AsyncRetriever, CustomRetriever] = Field( 2374 ..., 2375 description="Component used to coordinate how records are extracted across stream slices and request pages.", 2376 title="Retriever", 2377 ) 2378 incremental_sync: Optional[ 2379 Union[DatetimeBasedCursor, IncrementingCountCursor, CustomIncrementalSync] 2380 ] = Field( 2381 None, 2382 description="Component used to fetch data incrementally based on a time field in the data.", 2383 title="Incremental Sync", 2384 ) 2385 primary_key: Optional[PrimaryKey] = Field("", title="Primary Key") 2386 schema_loader: Optional[ 2387 Union[ 2388 InlineSchemaLoader, 2389 DynamicSchemaLoader, 2390 JsonFileSchemaLoader, 2391 List[ 2392 Union[ 2393 InlineSchemaLoader, 2394 DynamicSchemaLoader, 2395 JsonFileSchemaLoader, 2396 CustomSchemaLoader, 2397 ] 2398 ], 2399 CustomSchemaLoader, 2400 ] 2401 ] = Field( 2402 None, 2403 description="One or many schema loaders can be used to retrieve the schema for the current stream. When multiple schema loaders are defined, schema properties will be merged together. Schema loaders defined first taking precedence in the event of a conflict.", 2404 title="Schema Loader", 2405 ) 2406 transformations: Optional[ 2407 List[ 2408 Union[ 2409 AddFields, 2410 RemoveFields, 2411 KeysToLower, 2412 KeysToSnakeCase, 2413 FlattenFields, 2414 DpathFlattenFields, 2415 KeysReplace, 2416 CustomTransformation, 2417 ] 2418 ] 2419 ] = Field( 2420 None, 2421 description="A list of transformations to be applied to each output record.", 2422 title="Transformations", 2423 ) 2424 state_migrations: Optional[ 2425 List[Union[LegacyToPerPartitionStateMigration, CustomStateMigration]] 2426 ] = Field( 2427 [], 2428 description="Array of state migrations to be applied on the input state", 2429 title="State Migrations", 2430 ) 2431 file_uploader: Optional[FileUploader] = Field( 2432 None, 2433 description="(experimental) Describes how to fetch a file", 2434 title="File Uploader", 2435 ) 2436 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters")
retriever: Union[SimpleRetriever, AsyncRetriever, CustomRetriever]
incremental_sync: Union[DatetimeBasedCursor, IncrementingCountCursor, CustomIncrementalSync, NoneType]
primary_key: Optional[PrimaryKey]
schema_loader: Union[InlineSchemaLoader, DynamicSchemaLoader, JsonFileSchemaLoader, List[Union[InlineSchemaLoader, DynamicSchemaLoader, JsonFileSchemaLoader, CustomSchemaLoader]], CustomSchemaLoader, NoneType]
transformations: Optional[List[Union[AddFields, RemoveFields, KeysToLower, KeysToSnakeCase, FlattenFields, DpathFlattenFields, KeysReplace, CustomTransformation]]]
state_migrations: Optional[List[Union[LegacyToPerPartitionStateMigration, CustomStateMigration]]]
file_uploader: Optional[FileUploader]
class
DeclarativeStream.Config:
class
SessionTokenAuthenticator(pydantic.v1.main.BaseModel):
2439class SessionTokenAuthenticator(BaseModel): 2440 type: Literal["SessionTokenAuthenticator"] 2441 login_requester: HttpRequester = Field( 2442 ..., 2443 description="Description of the request to perform to obtain a session token to perform data requests. The response body is expected to be a JSON object with a session token property.", 2444 examples=[ 2445 { 2446 "type": "HttpRequester", 2447 "url_base": "https://my_api.com", 2448 "path": "/login", 2449 "authenticator": { 2450 "type": "BasicHttpAuthenticator", 2451 "username": "{{ config.username }}", 2452 "password": "{{ config.password }}", 2453 }, 2454 } 2455 ], 2456 title="Login Requester", 2457 ) 2458 session_token_path: List[str] = Field( 2459 ..., 2460 description="The path in the response body returned from the login requester to the session token.", 2461 examples=[["access_token"], ["result", "token"]], 2462 title="Session Token Path", 2463 ) 2464 expiration_duration: Optional[str] = Field( 2465 None, 2466 description="The duration in ISO 8601 duration notation after which the session token expires, starting from the time it was obtained. Omitting it will result in the session token being refreshed for every request.", 2467 examples=["PT1H", "P1D"], 2468 title="Expiration Duration", 2469 ) 2470 request_authentication: Union[ 2471 SessionTokenRequestApiKeyAuthenticator, SessionTokenRequestBearerAuthenticator 2472 ] = Field( 2473 ..., 2474 description="Authentication method to use for requests sent to the API, specifying how to inject the session token.", 2475 title="Data Request Authentication", 2476 ) 2477 decoder: Optional[Union[JsonDecoder, XmlDecoder]] = Field( 2478 None, description="Component used to decode the response.", title="Decoder" 2479 ) 2480 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters")
login_requester: HttpRequester
request_authentication: Union[SessionTokenRequestApiKeyAuthenticator, SessionTokenRequestBearerAuthenticator]
decoder: Union[JsonDecoder, XmlDecoder, NoneType]
class
HttpRequester(airbyte_cdk.sources.declarative.models.base_model_with_deprecations.BaseModelWithDeprecations):
2483class HttpRequester(BaseModelWithDeprecations): 2484 type: Literal["HttpRequester"] 2485 url_base: Optional[str] = Field( 2486 None, 2487 deprecated=True, 2488 deprecation_message="Use `url` field instead.", 2489 description="Deprecated, use the `url` instead. Base URL of the API source. Do not put sensitive information (e.g. API tokens) into this field - Use the Authenticator component for this.", 2490 examples=[ 2491 "https://connect.squareup.com/v2", 2492 "{{ config['base_url'] or 'https://app.posthog.com'}}/api", 2493 "https://connect.squareup.com/v2/quotes/{{ stream_partition['id'] }}/quote_line_groups", 2494 "https://example.com/api/v1/resource/{{ next_page_token['id'] }}", 2495 ], 2496 title="API Base URL", 2497 ) 2498 url: Optional[str] = Field( 2499 None, 2500 description="The URL of the source API endpoint. Do not put sensitive information (e.g. API tokens) into this field - Use the Authenticator component for this.", 2501 examples=[ 2502 "https://connect.squareup.com/v2", 2503 "{{ config['url'] or 'https://app.posthog.com'}}/api", 2504 "https://connect.squareup.com/v2/quotes/{{ stream_partition['id'] }}/quote_line_groups", 2505 "https://example.com/api/v1/resource/{{ next_page_token['id'] }}", 2506 ], 2507 title="API Endpoint URL", 2508 ) 2509 path: Optional[str] = Field( 2510 None, 2511 deprecated=True, 2512 deprecation_message="Use `url` field instead.", 2513 description="Deprecated, use the `url` instead. Path the specific API endpoint that this stream represents. Do not put sensitive information (e.g. API tokens) into this field - Use the Authenticator component for this.", 2514 examples=[ 2515 "/products", 2516 "/quotes/{{ stream_partition['id'] }}/quote_line_groups", 2517 "/trades/{{ config['symbol_id'] }}/history", 2518 ], 2519 title="URL Path", 2520 ) 2521 http_method: Optional[HttpMethod] = Field( 2522 HttpMethod.GET, 2523 description="The HTTP method used to fetch data from the source (can be GET or POST).", 2524 examples=["GET", "POST"], 2525 title="HTTP Method", 2526 ) 2527 authenticator: Optional[ 2528 Union[ 2529 ApiKeyAuthenticator, 2530 BasicHttpAuthenticator, 2531 BearerAuthenticator, 2532 OAuthAuthenticator, 2533 JwtAuthenticator, 2534 SessionTokenAuthenticator, 2535 SelectiveAuthenticator, 2536 CustomAuthenticator, 2537 NoAuth, 2538 LegacySessionTokenAuthenticator, 2539 ] 2540 ] = Field( 2541 None, 2542 description="Authentication method to use for requests sent to the API.", 2543 title="Authenticator", 2544 ) 2545 fetch_properties_from_endpoint: Optional[PropertiesFromEndpoint] = Field( 2546 None, 2547 deprecated=True, 2548 deprecation_message="Use `query_properties` field instead.", 2549 description="Allows for retrieving a dynamic set of properties from an API endpoint which can be injected into outbound request using the stream_partition.extra_fields.", 2550 title="Fetch Properties from Endpoint", 2551 ) 2552 query_properties: Optional[QueryProperties] = Field( 2553 None, 2554 description="For APIs that require explicit specification of the properties to query for, this component will take a static or dynamic set of properties (which can be optionally split into chunks) and allow them to be injected into an outbound request by accessing stream_partition.extra_fields.", 2555 title="Query Properties", 2556 ) 2557 request_parameters: Optional[Union[Dict[str, Union[str, QueryProperties]], str]] = Field( 2558 None, 2559 description="Specifies the query parameters that should be set on an outgoing HTTP request given the inputs.", 2560 examples=[ 2561 {"unit": "day"}, 2562 { 2563 "query": 'last_event_time BETWEEN TIMESTAMP "{{ stream_interval.start_time }}" AND TIMESTAMP "{{ stream_interval.end_time }}"' 2564 }, 2565 {"searchIn": "{{ ','.join(config.get('search_in', [])) }}"}, 2566 {"sort_by[asc]": "updated_at"}, 2567 ], 2568 title="Query Parameters", 2569 ) 2570 request_headers: Optional[Union[Dict[str, str], str]] = Field( 2571 None, 2572 description="Return any non-auth headers. Authentication headers will overwrite any overlapping headers returned from this method.", 2573 examples=[{"Output-Format": "JSON"}, {"Version": "{{ config['version'] }}"}], 2574 title="Request Headers", 2575 ) 2576 request_body_data: Optional[Union[Dict[str, str], str]] = Field( 2577 None, 2578 deprecated=True, 2579 deprecation_message="Use `request_body` field instead.", 2580 description="Specifies how to populate the body of the request with a non-JSON payload. Plain text will be sent as is, whereas objects will be converted to a urlencoded form.", 2581 examples=[ 2582 '[{"clause": {"type": "timestamp", "operator": 10, "parameters":\n [{"value": {{ stream_interval[\'start_time\'] | int * 1000 }} }]\n }, "orderBy": 1, "columnName": "Timestamp"}]/\n' 2583 ], 2584 title="Request Body Payload (Non-JSON)", 2585 ) 2586 request_body_json: Optional[Union[Dict[str, Any], str]] = Field( 2587 None, 2588 deprecated=True, 2589 deprecation_message="Use `request_body` field instead.", 2590 description="Specifies how to populate the body of the request with a JSON payload. Can contain nested objects.", 2591 examples=[ 2592 {"sort_order": "ASC", "sort_field": "CREATED_AT"}, 2593 {"key": "{{ config['value'] }}"}, 2594 {"sort": {"field": "updated_at", "order": "ascending"}}, 2595 ], 2596 title="Request Body JSON Payload", 2597 ) 2598 request_body: Optional[ 2599 Union[ 2600 RequestBodyPlainText, 2601 RequestBodyUrlEncodedForm, 2602 RequestBodyJsonObject, 2603 RequestBodyGraphQL, 2604 ] 2605 ] = Field( 2606 None, 2607 description="Specifies how to populate the body of the request with a payload. Can contain nested objects.", 2608 title="Request Body", 2609 ) 2610 error_handler: Optional[ 2611 Union[DefaultErrorHandler, CompositeErrorHandler, CustomErrorHandler] 2612 ] = Field( 2613 None, 2614 description="Error handler component that defines how to handle errors.", 2615 title="Error Handler", 2616 ) 2617 use_cache: Optional[bool] = Field( 2618 False, 2619 description="Enables stream requests caching. This field is automatically set by the CDK.", 2620 title="Use Cache", 2621 ) 2622 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters")
Pydantic BaseModel that warns when deprecated fields are accessed. The deprecation message is stored in the field's extra attributes. This class is used to create models that can have deprecated fields and show warnings when those fields are accessed or initialized.
The _deprecation_logs attribute is stored in the model itself.
The collected deprecation warnings are further propagated to the Airbyte log messages,
during the component creation process, in model_to_component._collect_model_deprecations().
The component implementation is not responsible for handling the deprecation warnings, since the deprecation warnings are already handled in the model itself.
http_method: Optional[HttpMethod]
authenticator: Union[ApiKeyAuthenticator, BasicHttpAuthenticator, BearerAuthenticator, OAuthAuthenticator, JwtAuthenticator, SessionTokenAuthenticator, SelectiveAuthenticator, CustomAuthenticator, NoAuth, LegacySessionTokenAuthenticator, NoneType]
fetch_properties_from_endpoint: Optional[PropertiesFromEndpoint]
query_properties: Optional[QueryProperties]
request_parameters: Union[Dict[str, Union[str, QueryProperties]], str, NoneType]
request_body: Union[RequestBodyPlainText, RequestBodyUrlEncodedForm, RequestBodyJsonObject, RequestBodyGraphQL, NoneType]
error_handler: Union[DefaultErrorHandler, CompositeErrorHandler, CustomErrorHandler, NoneType]
class
DynamicSchemaLoader(pydantic.v1.main.BaseModel):
2625class DynamicSchemaLoader(BaseModel): 2626 type: Literal["DynamicSchemaLoader"] 2627 retriever: Union[SimpleRetriever, AsyncRetriever, CustomRetriever] = Field( 2628 ..., 2629 description="Component used to coordinate how records are extracted across stream slices and request pages.", 2630 title="Retriever", 2631 ) 2632 schema_filter: Optional[Union[RecordFilter, CustomRecordFilter]] = Field( 2633 None, 2634 description="Responsible for filtering fields to be added to json schema.", 2635 title="Schema Filter", 2636 ) 2637 schema_transformations: Optional[ 2638 List[ 2639 Union[ 2640 AddFields, 2641 RemoveFields, 2642 KeysToLower, 2643 KeysToSnakeCase, 2644 FlattenFields, 2645 DpathFlattenFields, 2646 KeysReplace, 2647 CustomTransformation, 2648 ] 2649 ] 2650 ] = Field( 2651 None, 2652 description="A list of transformations to be applied to the schema.", 2653 title="Schema Transformations", 2654 ) 2655 schema_type_identifier: SchemaTypeIdentifier 2656 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters")
retriever: Union[SimpleRetriever, AsyncRetriever, CustomRetriever]
schema_filter: Union[RecordFilter, CustomRecordFilter, NoneType]
schema_transformations: Optional[List[Union[AddFields, RemoveFields, KeysToLower, KeysToSnakeCase, FlattenFields, DpathFlattenFields, KeysReplace, CustomTransformation]]]
schema_type_identifier: SchemaTypeIdentifier
class
ParentStreamConfig(pydantic.v1.main.BaseModel):
2659class ParentStreamConfig(BaseModel): 2660 type: Literal["ParentStreamConfig"] 2661 stream: Union[DeclarativeStream, StateDelegatingStream] = Field( 2662 ..., description="Reference to the parent stream.", title="Parent Stream" 2663 ) 2664 parent_key: str = Field( 2665 ..., 2666 description="The primary key of records from the parent stream that will be used during the retrieval of records for the current substream. This parent identifier field is typically a characteristic of the child records being extracted from the source API.", 2667 examples=["id", "{{ config['parent_record_id'] }}"], 2668 title="Parent Key", 2669 ) 2670 partition_field: str = Field( 2671 ..., 2672 description="While iterating over parent records during a sync, the parent_key value can be referenced by using this field.", 2673 examples=["parent_id", "{{ config['parent_partition_field'] }}"], 2674 title="Current Parent Key Value Identifier", 2675 ) 2676 request_option: Optional[RequestOption] = Field( 2677 None, 2678 description="A request option describing where the parent key value should be injected into and under what field name if applicable.", 2679 title="Request Option", 2680 ) 2681 incremental_dependency: Optional[bool] = Field( 2682 False, 2683 description="Indicates whether the parent stream should be read incrementally based on updates in the child stream.", 2684 title="Incremental Dependency", 2685 ) 2686 lazy_read_pointer: Optional[List[str]] = Field( 2687 [], 2688 description="If set, this will enable lazy reading, using the initial read of parent records to extract child records.", 2689 title="Lazy Read Pointer", 2690 ) 2691 extra_fields: Optional[List[List[str]]] = Field( 2692 None, 2693 description="Array of field paths to include as additional fields in the stream slice. Each path is an array of strings representing keys to access fields in the respective parent record. Accessible via `stream_slice.extra_fields`. Missing fields are set to `None`.", 2694 title="Extra Fields", 2695 ) 2696 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters")
stream: Union[DeclarativeStream, StateDelegatingStream]
request_option: Optional[RequestOption]
class
PropertiesFromEndpoint(pydantic.v1.main.BaseModel):
2699class PropertiesFromEndpoint(BaseModel): 2700 type: Literal["PropertiesFromEndpoint"] 2701 property_field_path: List[str] = Field( 2702 ..., 2703 description="Describes the path to the field that should be extracted", 2704 examples=[["name"]], 2705 ) 2706 retriever: Union[SimpleRetriever, CustomRetriever] = Field( 2707 ..., 2708 description="Requester component that describes how to fetch the properties to query from a remote API endpoint.", 2709 ) 2710 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters")
retriever: Union[SimpleRetriever, CustomRetriever]
class
QueryProperties(pydantic.v1.main.BaseModel):
2713class QueryProperties(BaseModel): 2714 type: Literal["QueryProperties"] 2715 property_list: Union[List[str], PropertiesFromEndpoint] = Field( 2716 ..., 2717 description="The set of properties that will be queried for in the outbound request. This can either be statically defined or dynamic based on an API endpoint", 2718 title="Property List", 2719 ) 2720 always_include_properties: Optional[List[str]] = Field( 2721 None, 2722 description="The list of properties that should be included in every set of properties when multiple chunks of properties are being requested.", 2723 title="Always Include Properties", 2724 ) 2725 property_chunking: Optional[PropertyChunking] = Field( 2726 None, 2727 description="Defines how query properties will be grouped into smaller sets for APIs with limitations on the number of properties fetched per API request.", 2728 title="Property Chunking", 2729 ) 2730 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters")
property_list: Union[List[str], PropertiesFromEndpoint]
property_chunking: Optional[PropertyChunking]
class
StateDelegatingStream(pydantic.v1.main.BaseModel):
2733class StateDelegatingStream(BaseModel): 2734 type: Literal["StateDelegatingStream"] 2735 name: str = Field(..., description="The stream name.", example=["Users"], title="Name") 2736 full_refresh_stream: DeclarativeStream = Field( 2737 ..., 2738 description="Component used to coordinate how records are extracted across stream slices and request pages when the state is empty or not provided.", 2739 title="Full Refresh Stream", 2740 ) 2741 incremental_stream: DeclarativeStream = Field( 2742 ..., 2743 description="Component used to coordinate how records are extracted across stream slices and request pages when the state provided.", 2744 title="Incremental Stream", 2745 ) 2746 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters")
full_refresh_stream: DeclarativeStream
incremental_stream: DeclarativeStream
class
SimpleRetriever(pydantic.v1.main.BaseModel):
2749class SimpleRetriever(BaseModel): 2750 type: Literal["SimpleRetriever"] 2751 requester: Union[HttpRequester, CustomRequester] = Field( 2752 ..., 2753 description="Requester component that describes how to prepare HTTP requests to send to the source API.", 2754 ) 2755 decoder: Optional[ 2756 Union[ 2757 JsonDecoder, 2758 XmlDecoder, 2759 CsvDecoder, 2760 JsonlDecoder, 2761 GzipDecoder, 2762 IterableDecoder, 2763 ZipfileDecoder, 2764 CustomDecoder, 2765 ] 2766 ] = Field( 2767 None, 2768 description="Component decoding the response so records can be extracted.", 2769 title="HTTP Response Format", 2770 ) 2771 record_selector: RecordSelector = Field( 2772 ..., 2773 description="Component that describes how to extract records from a HTTP response.", 2774 ) 2775 paginator: Optional[Union[DefaultPaginator, NoPagination]] = Field( 2776 None, 2777 description="Paginator component that describes how to navigate through the API's pages.", 2778 ) 2779 ignore_stream_slicer_parameters_on_paginated_requests: Optional[bool] = Field( 2780 False, 2781 description="If true, the partition router and incremental request options will be ignored when paginating requests. Request options set directly on the requester will not be ignored.", 2782 ) 2783 partition_router: Optional[ 2784 Union[ 2785 SubstreamPartitionRouter, 2786 ListPartitionRouter, 2787 GroupingPartitionRouter, 2788 CustomPartitionRouter, 2789 List[ 2790 Union[ 2791 SubstreamPartitionRouter, 2792 ListPartitionRouter, 2793 GroupingPartitionRouter, 2794 CustomPartitionRouter, 2795 ] 2796 ], 2797 ] 2798 ] = Field( 2799 None, 2800 description="Used to iteratively execute requests over a set of values, such as a parent stream's records or a list of constant values.", 2801 title="Partition Router", 2802 ) 2803 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters")
requester: Union[HttpRequester, CustomRequester]
decoder: Union[JsonDecoder, XmlDecoder, CsvDecoder, JsonlDecoder, GzipDecoder, IterableDecoder, ZipfileDecoder, CustomDecoder, NoneType]
record_selector: RecordSelector
paginator: Union[DefaultPaginator, NoPagination, NoneType]
partition_router: Union[SubstreamPartitionRouter, ListPartitionRouter, GroupingPartitionRouter, CustomPartitionRouter, List[Union[SubstreamPartitionRouter, ListPartitionRouter, GroupingPartitionRouter, CustomPartitionRouter]], NoneType]
class
AsyncRetriever(pydantic.v1.main.BaseModel):
2806class AsyncRetriever(BaseModel): 2807 type: Literal["AsyncRetriever"] 2808 record_selector: RecordSelector = Field( 2809 ..., 2810 description="Component that describes how to extract records from a HTTP response.", 2811 ) 2812 status_mapping: AsyncJobStatusMap = Field( 2813 ..., description="Async Job Status to Airbyte CDK Async Job Status mapping." 2814 ) 2815 status_extractor: Union[DpathExtractor, CustomRecordExtractor] = Field( 2816 ..., description="Responsible for fetching the actual status of the async job." 2817 ) 2818 download_target_extractor: Union[DpathExtractor, CustomRecordExtractor] = Field( 2819 ..., 2820 description="Responsible for fetching the final result `urls` provided by the completed / finished / ready async job.", 2821 ) 2822 download_extractor: Optional[ 2823 Union[DpathExtractor, CustomRecordExtractor, ResponseToFileExtractor] 2824 ] = Field(None, description="Responsible for fetching the records from provided urls.") 2825 creation_requester: Union[HttpRequester, CustomRequester] = Field( 2826 ..., 2827 description="Requester component that describes how to prepare HTTP requests to send to the source API to create the async server-side job.", 2828 ) 2829 polling_requester: Union[HttpRequester, CustomRequester] = Field( 2830 ..., 2831 description="Requester component that describes how to prepare HTTP requests to send to the source API to fetch the status of the running async job.", 2832 ) 2833 polling_job_timeout: Optional[Union[int, str]] = Field( 2834 None, 2835 description="The time in minutes after which the single Async Job should be considered as Timed Out.", 2836 ) 2837 download_target_requester: Optional[Union[HttpRequester, CustomRequester]] = Field( 2838 None, 2839 description="Requester component that describes how to prepare HTTP requests to send to the source API to extract the url from polling response by the completed async job.", 2840 ) 2841 download_requester: Union[HttpRequester, CustomRequester] = Field( 2842 ..., 2843 description="Requester component that describes how to prepare HTTP requests to send to the source API to download the data provided by the completed async job.", 2844 ) 2845 download_paginator: Optional[Union[DefaultPaginator, NoPagination]] = Field( 2846 None, 2847 description="Paginator component that describes how to navigate through the API's pages during download.", 2848 ) 2849 abort_requester: Optional[Union[HttpRequester, CustomRequester]] = Field( 2850 None, 2851 description="Requester component that describes how to prepare HTTP requests to send to the source API to abort a job once it is timed out from the source's perspective.", 2852 ) 2853 delete_requester: Optional[Union[HttpRequester, CustomRequester]] = Field( 2854 None, 2855 description="Requester component that describes how to prepare HTTP requests to send to the source API to delete a job once the records are extracted.", 2856 ) 2857 partition_router: Optional[ 2858 Union[ 2859 ListPartitionRouter, 2860 SubstreamPartitionRouter, 2861 GroupingPartitionRouter, 2862 CustomPartitionRouter, 2863 List[ 2864 Union[ 2865 ListPartitionRouter, 2866 SubstreamPartitionRouter, 2867 GroupingPartitionRouter, 2868 CustomPartitionRouter, 2869 ] 2870 ], 2871 ] 2872 ] = Field( 2873 [], 2874 description="PartitionRouter component that describes how to partition the stream, enabling incremental syncs and checkpointing.", 2875 title="Partition Router", 2876 ) 2877 decoder: Optional[ 2878 Union[ 2879 CsvDecoder, 2880 GzipDecoder, 2881 JsonDecoder, 2882 JsonlDecoder, 2883 IterableDecoder, 2884 XmlDecoder, 2885 ZipfileDecoder, 2886 CustomDecoder, 2887 ] 2888 ] = Field( 2889 None, 2890 description="Component decoding the response so records can be extracted.", 2891 title="HTTP Response Format", 2892 ) 2893 download_decoder: Optional[ 2894 Union[ 2895 CsvDecoder, 2896 GzipDecoder, 2897 JsonDecoder, 2898 JsonlDecoder, 2899 IterableDecoder, 2900 XmlDecoder, 2901 ZipfileDecoder, 2902 CustomDecoder, 2903 ] 2904 ] = Field( 2905 None, 2906 description="Component decoding the download response so records can be extracted.", 2907 title="Download HTTP Response Format", 2908 ) 2909 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters")
record_selector: RecordSelector
status_mapping: AsyncJobStatusMap
status_extractor: Union[DpathExtractor, CustomRecordExtractor]
download_target_extractor: Union[DpathExtractor, CustomRecordExtractor]
download_extractor: Union[DpathExtractor, CustomRecordExtractor, ResponseToFileExtractor, NoneType]
creation_requester: Union[HttpRequester, CustomRequester]
polling_requester: Union[HttpRequester, CustomRequester]
download_target_requester: Union[HttpRequester, CustomRequester, NoneType]
download_requester: Union[HttpRequester, CustomRequester]
download_paginator: Union[DefaultPaginator, NoPagination, NoneType]
abort_requester: Union[HttpRequester, CustomRequester, NoneType]
delete_requester: Union[HttpRequester, CustomRequester, NoneType]
partition_router: Union[SubstreamPartitionRouter, ListPartitionRouter, GroupingPartitionRouter, CustomPartitionRouter, List[Union[SubstreamPartitionRouter, ListPartitionRouter, GroupingPartitionRouter, CustomPartitionRouter]], NoneType]
decoder: Union[JsonDecoder, XmlDecoder, CsvDecoder, JsonlDecoder, GzipDecoder, IterableDecoder, ZipfileDecoder, CustomDecoder, NoneType]
download_decoder: Union[JsonDecoder, XmlDecoder, CsvDecoder, JsonlDecoder, GzipDecoder, IterableDecoder, ZipfileDecoder, CustomDecoder, NoneType]
class
SubstreamPartitionRouter(pydantic.v1.main.BaseModel):
2912class SubstreamPartitionRouter(BaseModel): 2913 type: Literal["SubstreamPartitionRouter"] 2914 parent_stream_configs: List[ParentStreamConfig] = Field( 2915 ..., 2916 description="Specifies which parent streams are being iterated over and how parent records should be used to partition the child stream data set.", 2917 title="Parent Stream Configs", 2918 ) 2919 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters")
parent_stream_configs: List[ParentStreamConfig]
class
GroupingPartitionRouter(pydantic.v1.main.BaseModel):
2922class GroupingPartitionRouter(BaseModel): 2923 type: Literal["GroupingPartitionRouter"] 2924 group_size: int = Field( 2925 ..., 2926 description="The number of partitions to include in each group. This determines how many partition values are batched together in a single slice.", 2927 examples=[10, 50], 2928 title="Group Size", 2929 ) 2930 underlying_partition_router: Union[ 2931 ListPartitionRouter, SubstreamPartitionRouter, CustomPartitionRouter 2932 ] = Field( 2933 ..., 2934 description="The partition router whose output will be grouped. This can be any valid partition router component.", 2935 title="Underlying Partition Router", 2936 ) 2937 deduplicate: Optional[bool] = Field( 2938 True, 2939 description="If true, ensures that partitions are unique within each group by removing duplicates based on the partition key.", 2940 title="Deduplicate Partitions", 2941 ) 2942 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters")
underlying_partition_router: Union[ListPartitionRouter, SubstreamPartitionRouter, CustomPartitionRouter]
class
HttpComponentsResolver(pydantic.v1.main.BaseModel):
2945class HttpComponentsResolver(BaseModel): 2946 type: Literal["HttpComponentsResolver"] 2947 retriever: Union[SimpleRetriever, AsyncRetriever, CustomRetriever] = Field( 2948 ..., 2949 description="Component used to coordinate how records are extracted across stream slices and request pages.", 2950 title="Retriever", 2951 ) 2952 components_mapping: List[ComponentMappingDefinition] 2953 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters")
retriever: Union[SimpleRetriever, AsyncRetriever, CustomRetriever]
components_mapping: List[ComponentMappingDefinition]
class
DynamicDeclarativeStream(pydantic.v1.main.BaseModel):
2956class DynamicDeclarativeStream(BaseModel): 2957 type: Literal["DynamicDeclarativeStream"] 2958 name: Optional[str] = Field( 2959 "", description="The dynamic stream name.", example=["Tables"], title="Name" 2960 ) 2961 stream_template: Union[DeclarativeStream, StateDelegatingStream] = Field( 2962 ..., description="Reference to the stream template.", title="Stream Template" 2963 ) 2964 components_resolver: Union[ 2965 HttpComponentsResolver, ConfigComponentsResolver, ParametrizedComponentsResolver 2966 ] = Field( 2967 ..., 2968 description="Component resolve and populates stream templates with components values.", 2969 title="Components Resolver", 2970 ) 2971 use_parent_parameters: Optional[bool] = Field( 2972 True, 2973 description="Whether or not to prioritize parent parameters over component parameters when constructing dynamic streams. Defaults to true for backward compatibility.", 2974 title="Use Parent Parameters", 2975 )
stream_template: Union[DeclarativeStream, StateDelegatingStream]
components_resolver: Union[HttpComponentsResolver, ConfigComponentsResolver, ParametrizedComponentsResolver]