airbyte_cdk.sources.declarative.models.declarative_component_schema
1# generated by datamodel-codegen: 2# filename: declarative_component_schema.yaml 3 4from __future__ import annotations 5 6from enum import Enum 7from typing import Any, Dict, List, Literal, Optional, Union 8 9from pydantic.v1 import BaseModel, Extra, Field 10 11from airbyte_cdk.sources.declarative.models.base_model_with_deprecations import ( 12 BaseModelWithDeprecations, 13) 14 15 16class AuthFlowType(Enum): 17 oauth2_0 = "oauth2.0" 18 oauth1_0 = "oauth1.0" 19 20 21class BasicHttpAuthenticator(BaseModel): 22 type: Literal["BasicHttpAuthenticator"] 23 username: str = Field( 24 ..., 25 description="The username that will be combined with the password, base64 encoded and used to make requests. Fill it in the user inputs.", 26 examples=["{{ config['username'] }}", "{{ config['api_key'] }}"], 27 title="Username", 28 ) 29 password: Optional[str] = Field( 30 "", 31 description="The password that will be combined with the username, base64 encoded and used to make requests. Fill it in the user inputs.", 32 examples=["{{ config['password'] }}", ""], 33 title="Password", 34 ) 35 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters") 36 37 38class BearerAuthenticator(BaseModel): 39 type: Literal["BearerAuthenticator"] 40 api_token: str = Field( 41 ..., 42 description="Token to inject as request header for authenticating with the API.", 43 examples=["{{ config['api_key'] }}", "{{ config['token'] }}"], 44 title="Bearer Token", 45 ) 46 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters") 47 48 49class DynamicStreamCheckConfig(BaseModel): 50 type: Literal["DynamicStreamCheckConfig"] 51 dynamic_stream_name: str = Field( 52 ..., description="The dynamic stream name.", title="Dynamic Stream Name" 53 ) 54 stream_count: Optional[int] = Field( 55 0, 56 description="The number of streams to attempt reading from during a check operation. If `stream_count` exceeds the total number of available streams, the minimum of the two values will be used.", 57 title="Stream Count", 58 ) 59 60 61class CheckDynamicStream(BaseModel): 62 type: Literal["CheckDynamicStream"] 63 stream_count: int = Field( 64 ..., 65 description="Numbers of the streams to try reading from when running a check operation.", 66 title="Stream Count", 67 ) 68 use_check_availability: Optional[bool] = Field( 69 True, 70 description="Enables stream check availability. This field is automatically set by the CDK.", 71 title="Use Check Availability", 72 ) 73 74 75class ConcurrencyLevel(BaseModel): 76 type: Optional[Literal["ConcurrencyLevel"]] = None 77 default_concurrency: Union[int, str] = Field( 78 ..., 79 description="The amount of concurrency that will applied during a sync. This value can be hardcoded or user-defined in the config if different users have varying volume thresholds in the target API.", 80 examples=[10, "{{ config['num_workers'] or 10 }}"], 81 title="Default Concurrency", 82 ) 83 max_concurrency: Optional[int] = Field( 84 None, 85 description="The maximum level of concurrency that will be used during a sync. This becomes a required field when the default_concurrency derives from the config, because it serves as a safeguard against a user-defined threshold that is too high.", 86 examples=[20, 100], 87 title="Max Concurrency", 88 ) 89 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters") 90 91 92class ConstantBackoffStrategy(BaseModel): 93 type: Literal["ConstantBackoffStrategy"] 94 backoff_time_in_seconds: Union[float, str] = Field( 95 ..., 96 description="Backoff time in seconds.", 97 examples=[30, 30.5, "{{ config['backoff_time'] }}"], 98 title="Backoff Time", 99 ) 100 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters") 101 102 103class CursorPagination(BaseModel): 104 type: Literal["CursorPagination"] 105 cursor_value: str = Field( 106 ..., 107 description="Value of the cursor defining the next page to fetch.", 108 examples=[ 109 "{{ headers.link.next.cursor }}", 110 "{{ last_record['key'] }}", 111 "{{ response['nextPage'] }}", 112 ], 113 title="Cursor Value", 114 ) 115 page_size: Optional[int] = Field( 116 None, 117 description="The number of records to include in each pages.", 118 examples=[100], 119 title="Page Size", 120 ) 121 stop_condition: Optional[str] = Field( 122 None, 123 description="Template string evaluating when to stop paginating.", 124 examples=[ 125 "{{ response.data.has_more is false }}", 126 "{{ 'next' not in headers['link'] }}", 127 ], 128 title="Stop Condition", 129 ) 130 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters") 131 132 133class CustomAuthenticator(BaseModel): 134 class Config: 135 extra = Extra.allow 136 137 type: Literal["CustomAuthenticator"] 138 class_name: str = Field( 139 ..., 140 description="Fully-qualified name of the class that will be implementing the custom authentication strategy. Has to be a sub class of DeclarativeAuthenticator. The format is `source_<name>.<package>.<class_name>`.", 141 examples=["source_railz.components.ShortLivedTokenAuthenticator"], 142 title="Class Name", 143 ) 144 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters") 145 146 147class CustomBackoffStrategy(BaseModel): 148 class Config: 149 extra = Extra.allow 150 151 type: Literal["CustomBackoffStrategy"] 152 class_name: str = Field( 153 ..., 154 description="Fully-qualified name of the class that will be implementing the custom backoff strategy. The format is `source_<name>.<package>.<class_name>`.", 155 examples=["source_railz.components.MyCustomBackoffStrategy"], 156 title="Class Name", 157 ) 158 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters") 159 160 161class CustomErrorHandler(BaseModel): 162 class Config: 163 extra = Extra.allow 164 165 type: Literal["CustomErrorHandler"] 166 class_name: str = Field( 167 ..., 168 description="Fully-qualified name of the class that will be implementing the custom error handler. The format is `source_<name>.<package>.<class_name>`.", 169 examples=["source_railz.components.MyCustomErrorHandler"], 170 title="Class Name", 171 ) 172 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters") 173 174 175class CustomIncrementalSync(BaseModel): 176 class Config: 177 extra = Extra.allow 178 179 type: Literal["CustomIncrementalSync"] 180 class_name: str = Field( 181 ..., 182 description="Fully-qualified name of the class that will be implementing the custom incremental sync. The format is `source_<name>.<package>.<class_name>`.", 183 examples=["source_railz.components.MyCustomIncrementalSync"], 184 title="Class Name", 185 ) 186 cursor_field: str = Field( 187 ..., 188 description="The location of the value on a record that will be used as a bookmark during sync.", 189 ) 190 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters") 191 192 193class CustomPaginationStrategy(BaseModel): 194 class Config: 195 extra = Extra.allow 196 197 type: Literal["CustomPaginationStrategy"] 198 class_name: str = Field( 199 ..., 200 description="Fully-qualified name of the class that will be implementing the custom pagination strategy. The format is `source_<name>.<package>.<class_name>`.", 201 examples=["source_railz.components.MyCustomPaginationStrategy"], 202 title="Class Name", 203 ) 204 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters") 205 206 207class CustomRecordExtractor(BaseModel): 208 class Config: 209 extra = Extra.allow 210 211 type: Literal["CustomRecordExtractor"] 212 class_name: str = Field( 213 ..., 214 description="Fully-qualified name of the class that will be implementing the custom record extraction strategy. The format is `source_<name>.<package>.<class_name>`.", 215 examples=["source_railz.components.MyCustomRecordExtractor"], 216 title="Class Name", 217 ) 218 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters") 219 220 221class CustomRecordFilter(BaseModel): 222 class Config: 223 extra = Extra.allow 224 225 type: Literal["CustomRecordFilter"] 226 class_name: str = Field( 227 ..., 228 description="Fully-qualified name of the class that will be implementing the custom record filter strategy. The format is `source_<name>.<package>.<class_name>`.", 229 examples=["source_railz.components.MyCustomCustomRecordFilter"], 230 title="Class Name", 231 ) 232 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters") 233 234 235class CustomRequester(BaseModel): 236 class Config: 237 extra = Extra.allow 238 239 type: Literal["CustomRequester"] 240 class_name: str = Field( 241 ..., 242 description="Fully-qualified name of the class that will be implementing the custom requester strategy. The format is `source_<name>.<package>.<class_name>`.", 243 examples=["source_railz.components.MyCustomRecordExtractor"], 244 title="Class Name", 245 ) 246 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters") 247 248 249class CustomRetriever(BaseModel): 250 class Config: 251 extra = Extra.allow 252 253 type: Literal["CustomRetriever"] 254 class_name: str = Field( 255 ..., 256 description="Fully-qualified name of the class that will be implementing the custom retriever strategy. The format is `source_<name>.<package>.<class_name>`.", 257 examples=["source_railz.components.MyCustomRetriever"], 258 title="Class Name", 259 ) 260 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters") 261 262 263class CustomPartitionRouter(BaseModel): 264 class Config: 265 extra = Extra.allow 266 267 type: Literal["CustomPartitionRouter"] 268 class_name: str = Field( 269 ..., 270 description="Fully-qualified name of the class that will be implementing the custom partition router. The format is `source_<name>.<package>.<class_name>`.", 271 examples=["source_railz.components.MyCustomPartitionRouter"], 272 title="Class Name", 273 ) 274 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters") 275 276 277class CustomSchemaLoader(BaseModel): 278 class Config: 279 extra = Extra.allow 280 281 type: Literal["CustomSchemaLoader"] 282 class_name: str = Field( 283 ..., 284 description="Fully-qualified name of the class that will be implementing the custom schema loader. The format is `source_<name>.<package>.<class_name>`.", 285 examples=["source_railz.components.MyCustomSchemaLoader"], 286 title="Class Name", 287 ) 288 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters") 289 290 291class CustomSchemaNormalization(BaseModel): 292 class Config: 293 extra = Extra.allow 294 295 type: Literal["CustomSchemaNormalization"] 296 class_name: str = Field( 297 ..., 298 description="Fully-qualified name of the class that will be implementing the custom normalization. The format is `source_<name>.<package>.<class_name>`.", 299 examples=[ 300 "source_amazon_seller_partner.components.LedgerDetailedViewReportsTypeTransformer" 301 ], 302 title="Class Name", 303 ) 304 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters") 305 306 307class CustomStateMigration(BaseModel): 308 class Config: 309 extra = Extra.allow 310 311 type: Literal["CustomStateMigration"] 312 class_name: str = Field( 313 ..., 314 description="Fully-qualified name of the class that will be implementing the custom state migration. The format is `source_<name>.<package>.<class_name>`.", 315 examples=["source_railz.components.MyCustomStateMigration"], 316 title="Class Name", 317 ) 318 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters") 319 320 321class CustomTransformation(BaseModel): 322 class Config: 323 extra = Extra.allow 324 325 type: Literal["CustomTransformation"] 326 class_name: str = Field( 327 ..., 328 description="Fully-qualified name of the class that will be implementing the custom transformation. The format is `source_<name>.<package>.<class_name>`.", 329 examples=["source_railz.components.MyCustomTransformation"], 330 title="Class Name", 331 ) 332 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters") 333 334 335class LegacyToPerPartitionStateMigration(BaseModel): 336 class Config: 337 extra = Extra.allow 338 339 type: Optional[Literal["LegacyToPerPartitionStateMigration"]] = None 340 341 342class Clamping(BaseModel): 343 target: str = Field( 344 ..., 345 description="The period of time that datetime windows will be clamped by", 346 examples=["DAY", "WEEK", "MONTH", "{{ config['target'] }}"], 347 title="Target", 348 ) 349 target_details: Optional[Dict[str, Any]] = None 350 351 352class Algorithm(Enum): 353 HS256 = "HS256" 354 HS384 = "HS384" 355 HS512 = "HS512" 356 ES256 = "ES256" 357 ES256K = "ES256K" 358 ES384 = "ES384" 359 ES512 = "ES512" 360 RS256 = "RS256" 361 RS384 = "RS384" 362 RS512 = "RS512" 363 PS256 = "PS256" 364 PS384 = "PS384" 365 PS512 = "PS512" 366 EdDSA = "EdDSA" 367 368 369class JwtHeaders(BaseModel): 370 class Config: 371 extra = Extra.forbid 372 373 kid: Optional[str] = Field( 374 None, 375 description="Private key ID for user account.", 376 examples=["{{ config['kid'] }}"], 377 title="Key Identifier", 378 ) 379 typ: Optional[str] = Field( 380 "JWT", 381 description="The media type of the complete JWT.", 382 examples=["JWT"], 383 title="Type", 384 ) 385 cty: Optional[str] = Field( 386 None, 387 description="Content type of JWT header.", 388 examples=["JWT"], 389 title="Content Type", 390 ) 391 392 393class JwtPayload(BaseModel): 394 class Config: 395 extra = Extra.forbid 396 397 iss: Optional[str] = Field( 398 None, 399 description="The user/principal that issued the JWT. Commonly a value unique to the user.", 400 examples=["{{ config['iss'] }}"], 401 title="Issuer", 402 ) 403 sub: Optional[str] = Field( 404 None, 405 description="The subject of the JWT. Commonly defined by the API.", 406 title="Subject", 407 ) 408 aud: Optional[str] = Field( 409 None, 410 description="The recipient that the JWT is intended for. Commonly defined by the API.", 411 examples=["appstoreconnect-v1"], 412 title="Audience", 413 ) 414 415 416class JwtAuthenticator(BaseModel): 417 type: Literal["JwtAuthenticator"] 418 secret_key: str = Field( 419 ..., 420 description="Secret used to sign the JSON web token.", 421 examples=["{{ config['secret_key'] }}"], 422 title="Secret Key", 423 ) 424 base64_encode_secret_key: Optional[bool] = Field( 425 False, 426 description='When set to true, the secret key will be base64 encoded prior to being encoded as part of the JWT. Only set to "true" when required by the API.', 427 title="Base64-encode Secret Key", 428 ) 429 algorithm: Algorithm = Field( 430 ..., 431 description="Algorithm used to sign the JSON web token.", 432 examples=["ES256", "HS256", "RS256", "{{ config['algorithm'] }}"], 433 title="Algorithm", 434 ) 435 token_duration: Optional[int] = Field( 436 1200, 437 description="The amount of time in seconds a JWT token can be valid after being issued.", 438 examples=[1200, 3600], 439 title="Token Duration", 440 ) 441 header_prefix: Optional[str] = Field( 442 None, 443 description="The prefix to be used within the Authentication header.", 444 examples=["Bearer", "Basic"], 445 title="Header Prefix", 446 ) 447 jwt_headers: Optional[JwtHeaders] = Field( 448 None, 449 description="JWT headers used when signing JSON web token.", 450 title="JWT Headers", 451 ) 452 additional_jwt_headers: Optional[Dict[str, Any]] = Field( 453 None, 454 description="Additional headers to be included with the JWT headers object.", 455 title="Additional JWT Headers", 456 ) 457 jwt_payload: Optional[JwtPayload] = Field( 458 None, 459 description="JWT Payload used when signing JSON web token.", 460 title="JWT Payload", 461 ) 462 additional_jwt_payload: Optional[Dict[str, Any]] = Field( 463 None, 464 description="Additional properties to be added to the JWT payload.", 465 title="Additional JWT Payload Properties", 466 ) 467 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters") 468 469 470class RefreshTokenUpdater(BaseModel): 471 refresh_token_name: Optional[str] = Field( 472 "refresh_token", 473 description="The name of the property which contains the updated refresh token in the response from the token refresh endpoint.", 474 examples=["refresh_token"], 475 title="Refresh Token Property Name", 476 ) 477 access_token_config_path: Optional[List[str]] = Field( 478 ["credentials", "access_token"], 479 description="Config path to the access token. Make sure the field actually exists in the config.", 480 examples=[["credentials", "access_token"], ["access_token"]], 481 title="Config Path To Access Token", 482 ) 483 refresh_token_config_path: Optional[List[str]] = Field( 484 ["credentials", "refresh_token"], 485 description="Config path to the access token. Make sure the field actually exists in the config.", 486 examples=[["credentials", "refresh_token"], ["refresh_token"]], 487 title="Config Path To Refresh Token", 488 ) 489 token_expiry_date_config_path: Optional[List[str]] = Field( 490 ["credentials", "token_expiry_date"], 491 description="Config path to the expiry date. Make sure actually exists in the config.", 492 examples=[["credentials", "token_expiry_date"]], 493 title="Config Path To Expiry Date", 494 ) 495 refresh_token_error_status_codes: Optional[List[int]] = Field( 496 [], 497 description="Status Codes to Identify refresh token error in response (Refresh Token Error Key and Refresh Token Error Values should be also specified). Responses with one of the error status code and containing an error value will be flagged as a config error", 498 examples=[[400, 500]], 499 title="Refresh Token Error Status Codes", 500 ) 501 refresh_token_error_key: Optional[str] = Field( 502 "", 503 description="Key to Identify refresh token error in response (Refresh Token Error Status Codes and Refresh Token Error Values should be also specified).", 504 examples=["error"], 505 title="Refresh Token Error Key", 506 ) 507 refresh_token_error_values: Optional[List[str]] = Field( 508 [], 509 description='List of values to check for exception during token refresh process. Used to check if the error found in the response matches the key from the Refresh Token Error Key field (e.g. response={"error": "invalid_grant"}). Only responses with one of the error status code and containing an error value will be flagged as a config error', 510 examples=[["invalid_grant", "invalid_permissions"]], 511 title="Refresh Token Error Values", 512 ) 513 514 515class OAuthAuthenticator(BaseModel): 516 type: Literal["OAuthAuthenticator"] 517 client_id_name: Optional[str] = Field( 518 "client_id", 519 description="The name of the property to use to refresh the `access_token`.", 520 examples=["custom_app_id"], 521 title="Client ID Property Name", 522 ) 523 client_id: Optional[str] = Field( 524 None, 525 description="The OAuth client ID. Fill it in the user inputs.", 526 examples=["{{ config['client_id }}", "{{ config['credentials']['client_id }}"], 527 title="Client ID", 528 ) 529 client_secret_name: Optional[str] = Field( 530 "client_secret", 531 description="The name of the property to use to refresh the `access_token`.", 532 examples=["custom_app_secret"], 533 title="Client Secret Property Name", 534 ) 535 client_secret: Optional[str] = Field( 536 None, 537 description="The OAuth client secret. Fill it in the user inputs.", 538 examples=[ 539 "{{ config['client_secret }}", 540 "{{ config['credentials']['client_secret }}", 541 ], 542 title="Client Secret", 543 ) 544 refresh_token_name: Optional[str] = Field( 545 "refresh_token", 546 description="The name of the property to use to refresh the `access_token`.", 547 examples=["custom_app_refresh_value"], 548 title="Refresh Token Property Name", 549 ) 550 refresh_token: Optional[str] = Field( 551 None, 552 description="Credential artifact used to get a new access token.", 553 examples=[ 554 "{{ config['refresh_token'] }}", 555 "{{ config['credentials]['refresh_token'] }}", 556 ], 557 title="Refresh Token", 558 ) 559 token_refresh_endpoint: Optional[str] = Field( 560 None, 561 description="The full URL to call to obtain a new access token.", 562 examples=["https://connect.squareup.com/oauth2/token"], 563 title="Token Refresh Endpoint", 564 ) 565 access_token_name: Optional[str] = Field( 566 "access_token", 567 description="The name of the property which contains the access token in the response from the token refresh endpoint.", 568 examples=["access_token"], 569 title="Access Token Property Name", 570 ) 571 access_token_value: Optional[str] = Field( 572 None, 573 description="The value of the access_token to bypass the token refreshing using `refresh_token`.", 574 examples=["secret_access_token_value"], 575 title="Access Token Value", 576 ) 577 expires_in_name: Optional[str] = Field( 578 "expires_in", 579 description="The name of the property which contains the expiry date in the response from the token refresh endpoint.", 580 examples=["expires_in"], 581 title="Token Expiry Property Name", 582 ) 583 grant_type_name: Optional[str] = Field( 584 "grant_type", 585 description="The name of the property to use to refresh the `access_token`.", 586 examples=["custom_grant_type"], 587 title="Grant Type Property Name", 588 ) 589 grant_type: Optional[str] = Field( 590 "refresh_token", 591 description="Specifies the OAuth2 grant type. If set to refresh_token, the refresh_token needs to be provided as well. For client_credentials, only client id and secret are required. Other grant types are not officially supported.", 592 examples=["refresh_token", "client_credentials"], 593 title="Grant Type", 594 ) 595 refresh_request_body: Optional[Dict[str, Any]] = Field( 596 None, 597 description="Body of the request sent to get a new access token.", 598 examples=[ 599 { 600 "applicationId": "{{ config['application_id'] }}", 601 "applicationSecret": "{{ config['application_secret'] }}", 602 "token": "{{ config['token'] }}", 603 } 604 ], 605 title="Refresh Request Body", 606 ) 607 refresh_request_headers: Optional[Dict[str, Any]] = Field( 608 None, 609 description="Headers of the request sent to get a new access token.", 610 examples=[ 611 { 612 "Authorization": "<AUTH_TOKEN>", 613 "Content-Type": "application/x-www-form-urlencoded", 614 } 615 ], 616 title="Refresh Request Headers", 617 ) 618 scopes: Optional[List[str]] = Field( 619 None, 620 description="List of scopes that should be granted to the access token.", 621 examples=[["crm.list.read", "crm.objects.contacts.read", "crm.schema.contacts.read"]], 622 title="Scopes", 623 ) 624 token_expiry_date: Optional[str] = Field( 625 None, 626 description="The access token expiry date.", 627 examples=["2023-04-06T07:12:10.421833+00:00", 1680842386], 628 title="Token Expiry Date", 629 ) 630 token_expiry_date_format: Optional[str] = Field( 631 None, 632 description="The format of the time to expiration datetime. Provide it if the time is returned as a date-time string instead of seconds.", 633 examples=["%Y-%m-%d %H:%M:%S.%f+00:00"], 634 title="Token Expiry Date Format", 635 ) 636 refresh_token_updater: Optional[RefreshTokenUpdater] = Field( 637 None, 638 description="When the token updater is defined, new refresh tokens, access tokens and the access token expiry date are written back from the authentication response to the config object. This is important if the refresh token can only used once.", 639 title="Token Updater", 640 ) 641 profile_assertion: Optional[JwtAuthenticator] = Field( 642 None, 643 description="The authenticator being used to authenticate the client authenticator.", 644 title="Profile Assertion", 645 ) 646 use_profile_assertion: Optional[bool] = Field( 647 False, 648 description="Enable using profile assertion as a flow for OAuth authorization.", 649 title="Use Profile Assertion", 650 ) 651 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters") 652 653 654class Rate(BaseModel): 655 class Config: 656 extra = Extra.allow 657 658 limit: Union[int, str] = Field( 659 ..., 660 description="The maximum number of calls allowed within the interval.", 661 title="Limit", 662 ) 663 interval: str = Field( 664 ..., 665 description="The time interval for the rate limit.", 666 examples=["PT1H", "P1D"], 667 title="Interval", 668 ) 669 670 671class HttpRequestRegexMatcher(BaseModel): 672 class Config: 673 extra = Extra.allow 674 675 method: Optional[str] = Field( 676 None, description="The HTTP method to match (e.g., GET, POST).", title="Method" 677 ) 678 url_base: Optional[str] = Field( 679 None, 680 description='The base URL (scheme and host, e.g. "https://api.example.com") to match.', 681 title="URL Base", 682 ) 683 url_path_pattern: Optional[str] = Field( 684 None, 685 description="A regular expression pattern to match the URL path.", 686 title="URL Path Pattern", 687 ) 688 params: Optional[Dict[str, Any]] = Field( 689 None, description="The query parameters to match.", title="Parameters" 690 ) 691 headers: Optional[Dict[str, Any]] = Field( 692 None, description="The headers to match.", title="Headers" 693 ) 694 695 696class DpathExtractor(BaseModel): 697 type: Literal["DpathExtractor"] 698 field_path: List[str] = Field( 699 ..., 700 description='List of potentially nested fields describing the full path of the field to extract. Use "*" to extract all values from an array. See more info in the [docs](https://docs.airbyte.com/connector-development/config-based/understanding-the-yaml-file/record-selector).', 701 examples=[ 702 ["data"], 703 ["data", "records"], 704 ["data", "{{ parameters.name }}"], 705 ["data", "*", "record"], 706 ], 707 title="Field Path", 708 ) 709 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters") 710 711 712class ResponseToFileExtractor(BaseModel): 713 type: Literal["ResponseToFileExtractor"] 714 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters") 715 716 717class ExponentialBackoffStrategy(BaseModel): 718 type: Literal["ExponentialBackoffStrategy"] 719 factor: Optional[Union[float, str]] = Field( 720 5, 721 description="Multiplicative constant applied on each retry.", 722 examples=[5, 5.5, "10"], 723 title="Factor", 724 ) 725 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters") 726 727 728class GroupByKeyMergeStrategy(BaseModel): 729 type: Literal["GroupByKeyMergeStrategy"] 730 key: Union[str, List[str]] = Field( 731 ..., 732 description="The name of the field on the record whose value will be used to group properties that were retrieved through multiple API requests.", 733 examples=["id", ["parent_id", "end_date"]], 734 title="Key", 735 ) 736 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters") 737 738 739class SessionTokenRequestBearerAuthenticator(BaseModel): 740 type: Literal["Bearer"] 741 742 743class HttpMethod(Enum): 744 GET = "GET" 745 POST = "POST" 746 747 748class Action(Enum): 749 SUCCESS = "SUCCESS" 750 FAIL = "FAIL" 751 RETRY = "RETRY" 752 IGNORE = "IGNORE" 753 RATE_LIMITED = "RATE_LIMITED" 754 755 756class FailureType(Enum): 757 system_error = "system_error" 758 config_error = "config_error" 759 transient_error = "transient_error" 760 761 762class HttpResponseFilter(BaseModel): 763 type: Literal["HttpResponseFilter"] 764 action: Optional[Action] = Field( 765 None, 766 description="Action to execute if a response matches the filter.", 767 examples=["SUCCESS", "FAIL", "RETRY", "IGNORE", "RATE_LIMITED"], 768 title="Action", 769 ) 770 failure_type: Optional[FailureType] = Field( 771 None, 772 description="Failure type of traced exception if a response matches the filter.", 773 examples=["system_error", "config_error", "transient_error"], 774 title="Failure Type", 775 ) 776 error_message: Optional[str] = Field( 777 None, 778 description="Error Message to display if the response matches the filter.", 779 title="Error Message", 780 ) 781 error_message_contains: Optional[str] = Field( 782 None, 783 description="Match the response if its error message contains the substring.", 784 example=["This API operation is not enabled for this site"], 785 title="Error Message Substring", 786 ) 787 http_codes: Optional[List[int]] = Field( 788 None, 789 description="Match the response if its HTTP code is included in this list.", 790 examples=[[420, 429], [500]], 791 title="HTTP Codes", 792 unique_items=True, 793 ) 794 predicate: Optional[str] = Field( 795 None, 796 description="Match the response if the predicate evaluates to true.", 797 examples=[ 798 "{{ 'Too much requests' in response }}", 799 "{{ 'error_code' in response and response['error_code'] == 'ComplexityException' }}", 800 ], 801 title="Predicate", 802 ) 803 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters") 804 805 806class ComplexFieldType(BaseModel): 807 field_type: str 808 items: Optional[Union[str, ComplexFieldType]] = None 809 810 811class TypesMap(BaseModel): 812 target_type: Union[str, List[str], ComplexFieldType] 813 current_type: Union[str, List[str]] 814 condition: Optional[str] = None 815 816 817class SchemaTypeIdentifier(BaseModel): 818 type: Optional[Literal["SchemaTypeIdentifier"]] = None 819 schema_pointer: Optional[List[str]] = Field( 820 [], 821 description="List of nested fields defining the schema field path to extract. Defaults to [].", 822 title="Schema Path", 823 ) 824 key_pointer: List[str] = Field( 825 ..., 826 description="List of potentially nested fields describing the full path of the field key to extract.", 827 title="Key Path", 828 ) 829 type_pointer: Optional[List[str]] = Field( 830 None, 831 description="List of potentially nested fields describing the full path of the field type to extract.", 832 title="Type Path", 833 ) 834 types_mapping: Optional[List[TypesMap]] = None 835 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters") 836 837 838class InlineSchemaLoader(BaseModel): 839 type: Literal["InlineSchemaLoader"] 840 schema_: Optional[Dict[str, Any]] = Field( 841 None, 842 alias="schema", 843 description='Describes a streams\' schema. Refer to the <a href="https://docs.airbyte.com/understanding-airbyte/supported-data-types/">Data Types documentation</a> for more details on which types are valid.', 844 title="Schema", 845 ) 846 847 848class JsonFileSchemaLoader(BaseModel): 849 type: Literal["JsonFileSchemaLoader"] 850 file_path: Optional[str] = Field( 851 None, 852 description="Path to the JSON file defining the schema. The path is relative to the connector module's root.", 853 example=["./schemas/users.json"], 854 title="File Path", 855 ) 856 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters") 857 858 859class JsonDecoder(BaseModel): 860 type: Literal["JsonDecoder"] 861 862 863class JsonlDecoder(BaseModel): 864 type: Literal["JsonlDecoder"] 865 866 867class KeysToLower(BaseModel): 868 type: Literal["KeysToLower"] 869 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters") 870 871 872class KeysToSnakeCase(BaseModel): 873 type: Literal["KeysToSnakeCase"] 874 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters") 875 876 877class FlattenFields(BaseModel): 878 type: Literal["FlattenFields"] 879 flatten_lists: Optional[bool] = Field( 880 True, 881 description="Whether to flatten lists or leave it as is. Default is True.", 882 title="Flatten Lists", 883 ) 884 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters") 885 886 887class KeyTransformation(BaseModel): 888 type: Literal["KeyTransformation"] 889 prefix: Optional[str] = Field( 890 None, 891 description="Prefix to add for object keys. If not provided original keys remain unchanged.", 892 examples=["flattened_"], 893 title="Key Prefix", 894 ) 895 suffix: Optional[str] = Field( 896 None, 897 description="Suffix to add for object keys. If not provided original keys remain unchanged.", 898 examples=["_flattened"], 899 title="Key Suffix", 900 ) 901 902 903class DpathFlattenFields(BaseModel): 904 type: Literal["DpathFlattenFields"] 905 field_path: List[str] = Field( 906 ..., 907 description="A path to field that needs to be flattened.", 908 examples=[["data"], ["data", "*", "field"]], 909 title="Field Path", 910 ) 911 delete_origin_value: Optional[bool] = Field( 912 None, 913 description="Whether to delete the origin value or keep it. Default is False.", 914 title="Delete Origin Value", 915 ) 916 replace_record: Optional[bool] = Field( 917 None, 918 description="Whether to replace the origin record or not. Default is False.", 919 title="Replace Origin Record", 920 ) 921 key_transformation: Optional[KeyTransformation] = Field( 922 None, 923 description="Transformation for object keys. If not provided, original key will be used.", 924 title="Key transformation", 925 ) 926 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters") 927 928 929class KeysReplace(BaseModel): 930 type: Literal["KeysReplace"] 931 old: str = Field( 932 ..., 933 description="Old value to replace.", 934 examples=[ 935 " ", 936 "{{ record.id }}", 937 "{{ config['id'] }}", 938 "{{ stream_slice['id'] }}", 939 ], 940 title="Old value", 941 ) 942 new: str = Field( 943 ..., 944 description="New value to set.", 945 examples=[ 946 "_", 947 "{{ record.id }}", 948 "{{ config['id'] }}", 949 "{{ stream_slice['id'] }}", 950 ], 951 title="New value", 952 ) 953 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters") 954 955 956class IterableDecoder(BaseModel): 957 type: Literal["IterableDecoder"] 958 959 960class XmlDecoder(BaseModel): 961 type: Literal["XmlDecoder"] 962 963 964class CustomDecoder(BaseModel): 965 class Config: 966 extra = Extra.allow 967 968 type: Literal["CustomDecoder"] 969 class_name: str = Field( 970 ..., 971 description="Fully-qualified name of the class that will be implementing the custom decoding. Has to be a sub class of Decoder. The format is `source_<name>.<package>.<class_name>`.", 972 examples=["source_amazon_ads.components.GzipJsonlDecoder"], 973 title="Class Name", 974 ) 975 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters") 976 977 978class MinMaxDatetime(BaseModel): 979 type: Literal["MinMaxDatetime"] 980 datetime: str = Field( 981 ..., 982 description="Datetime value.", 983 examples=["2021-01-01", "2021-01-01T00:00:00Z", "{{ config['start_time'] }}"], 984 title="Datetime", 985 ) 986 datetime_format: Optional[str] = Field( 987 "", 988 description='Format of the datetime value. Defaults to "%Y-%m-%dT%H:%M:%S.%f%z" if left empty. Use placeholders starting with "%" to describe the format the API is using. The following placeholders are available:\n * **%s**: Epoch unix timestamp - `1686218963`\n * **%s_as_float**: Epoch unix timestamp in seconds as float with microsecond precision - `1686218963.123456`\n * **%ms**: Epoch unix timestamp - `1686218963123`\n * **%a**: Weekday (abbreviated) - `Sun`\n * **%A**: Weekday (full) - `Sunday`\n * **%w**: Weekday (decimal) - `0` (Sunday), `6` (Saturday)\n * **%d**: Day of the month (zero-padded) - `01`, `02`, ..., `31`\n * **%b**: Month (abbreviated) - `Jan`\n * **%B**: Month (full) - `January`\n * **%m**: Month (zero-padded) - `01`, `02`, ..., `12`\n * **%y**: Year (without century, zero-padded) - `00`, `01`, ..., `99`\n * **%Y**: Year (with century) - `0001`, `0002`, ..., `9999`\n * **%H**: Hour (24-hour, zero-padded) - `00`, `01`, ..., `23`\n * **%I**: Hour (12-hour, zero-padded) - `01`, `02`, ..., `12`\n * **%p**: AM/PM indicator\n * **%M**: Minute (zero-padded) - `00`, `01`, ..., `59`\n * **%S**: Second (zero-padded) - `00`, `01`, ..., `59`\n * **%f**: Microsecond (zero-padded to 6 digits) - `000000`, `000001`, ..., `999999`\n * **%_ms**: Millisecond (zero-padded to 3 digits) - `000`, `001`, ..., `999`\n * **%z**: UTC offset - `(empty)`, `+0000`, `-04:00`\n * **%Z**: Time zone name - `(empty)`, `UTC`, `GMT`\n * **%j**: Day of the year (zero-padded) - `001`, `002`, ..., `366`\n * **%U**: Week number of the year (Sunday as first day) - `00`, `01`, ..., `53`\n * **%W**: Week number of the year (Monday as first day) - `00`, `01`, ..., `53`\n * **%c**: Date and time representation - `Tue Aug 16 21:30:00 1988`\n * **%x**: Date representation - `08/16/1988`\n * **%X**: Time representation - `21:30:00`\n * **%%**: Literal \'%\' character\n\n Some placeholders depend on the locale of the underlying system - in most cases this locale is configured as en/US. For more information see the [Python documentation](https://docs.python.org/3/library/datetime.html#strftime-and-strptime-format-codes).\n', 989 examples=["%Y-%m-%dT%H:%M:%S.%f%z", "%Y-%m-%d", "%s"], 990 title="Datetime Format", 991 ) 992 max_datetime: Optional[str] = Field( 993 None, 994 description="Ceiling applied on the datetime value. Must be formatted with the datetime_format field.", 995 examples=["2021-01-01T00:00:00Z", "2021-01-01"], 996 title="Max Datetime", 997 ) 998 min_datetime: Optional[str] = Field( 999 None, 1000 description="Floor applied on the datetime value. Must be formatted with the datetime_format field.", 1001 examples=["2010-01-01T00:00:00Z", "2010-01-01"], 1002 title="Min Datetime", 1003 ) 1004 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters") 1005 1006 1007class NoAuth(BaseModel): 1008 type: Literal["NoAuth"] 1009 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters") 1010 1011 1012class NoPagination(BaseModel): 1013 type: Literal["NoPagination"] 1014 1015 1016class State(BaseModel): 1017 class Config: 1018 extra = Extra.allow 1019 1020 min: int 1021 max: int 1022 1023 1024class OauthConnectorInputSpecification(BaseModel): 1025 class Config: 1026 extra = Extra.allow 1027 1028 consent_url: str = Field( 1029 ..., 1030 description="The DeclarativeOAuth Specific string URL string template to initiate the authentication.\nThe placeholders are replaced during the processing to provide neccessary values.", 1031 examples=[ 1032 "https://domain.host.com/marketing_api/auth?{{client_id_key}}={{client_id_value}}&{{redirect_uri_key}}={{{{redirect_uri_value}} | urlEncoder}}&{{state_key}}={{state_value}}", 1033 "https://endpoint.host.com/oauth2/authorize?{{client_id_key}}={{client_id_value}}&{{redirect_uri_key}}={{{{redirect_uri_value}} | urlEncoder}}&{{scope_key}}={{{{scope_value}} | urlEncoder}}&{{state_key}}={{state_value}}&subdomain={{subdomain}}", 1034 ], 1035 title="Consent URL", 1036 ) 1037 scope: Optional[str] = Field( 1038 None, 1039 description="The DeclarativeOAuth Specific string of the scopes needed to be grant for authenticated user.", 1040 examples=["user:read user:read_orders workspaces:read"], 1041 title="Scopes", 1042 ) 1043 access_token_url: str = Field( 1044 ..., 1045 description="The DeclarativeOAuth Specific URL templated string to obtain the `access_token`, `refresh_token` etc.\nThe placeholders are replaced during the processing to provide neccessary values.", 1046 examples=[ 1047 "https://auth.host.com/oauth2/token?{{client_id_key}}={{client_id_value}}&{{client_secret_key}}={{client_secret_value}}&{{auth_code_key}}={{auth_code_value}}&{{redirect_uri_key}}={{{{redirect_uri_value}} | urlEncoder}}" 1048 ], 1049 title="Access Token URL", 1050 ) 1051 access_token_headers: Optional[Dict[str, Any]] = Field( 1052 None, 1053 description="The DeclarativeOAuth Specific optional headers to inject while exchanging the `auth_code` to `access_token` during `completeOAuthFlow` step.", 1054 examples=[ 1055 { 1056 "Authorization": "Basic {{ {{ client_id_value }}:{{ client_secret_value }} | base64Encoder }}" 1057 } 1058 ], 1059 title="Access Token Headers", 1060 ) 1061 access_token_params: Optional[Dict[str, Any]] = Field( 1062 None, 1063 description="The DeclarativeOAuth Specific optional query parameters to inject while exchanging the `auth_code` to `access_token` during `completeOAuthFlow` step.\nWhen this property is provided, the query params will be encoded as `Json` and included in the outgoing API request.", 1064 examples=[ 1065 { 1066 "{{ auth_code_key }}": "{{ auth_code_value }}", 1067 "{{ client_id_key }}": "{{ client_id_value }}", 1068 "{{ client_secret_key }}": "{{ client_secret_value }}", 1069 } 1070 ], 1071 title="Access Token Query Params (Json Encoded)", 1072 ) 1073 extract_output: Optional[List[str]] = Field( 1074 None, 1075 description="The DeclarativeOAuth Specific list of strings to indicate which keys should be extracted and returned back to the input config.", 1076 examples=[["access_token", "refresh_token", "other_field"]], 1077 title="Extract Output", 1078 ) 1079 state: Optional[State] = Field( 1080 None, 1081 description="The DeclarativeOAuth Specific object to provide the criteria of how the `state` query param should be constructed,\nincluding length and complexity.", 1082 examples=[{"min": 7, "max": 128}], 1083 title="Configurable State Query Param", 1084 ) 1085 client_id_key: Optional[str] = Field( 1086 None, 1087 description="The DeclarativeOAuth Specific optional override to provide the custom `client_id` key name, if required by data-provider.", 1088 examples=["my_custom_client_id_key_name"], 1089 title="Client ID Key Override", 1090 ) 1091 client_secret_key: Optional[str] = Field( 1092 None, 1093 description="The DeclarativeOAuth Specific optional override to provide the custom `client_secret` key name, if required by data-provider.", 1094 examples=["my_custom_client_secret_key_name"], 1095 title="Client Secret Key Override", 1096 ) 1097 scope_key: Optional[str] = Field( 1098 None, 1099 description="The DeclarativeOAuth Specific optional override to provide the custom `scope` key name, if required by data-provider.", 1100 examples=["my_custom_scope_key_key_name"], 1101 title="Scopes Key Override", 1102 ) 1103 state_key: Optional[str] = Field( 1104 None, 1105 description="The DeclarativeOAuth Specific optional override to provide the custom `state` key name, if required by data-provider.", 1106 examples=["my_custom_state_key_key_name"], 1107 title="State Key Override", 1108 ) 1109 auth_code_key: Optional[str] = Field( 1110 None, 1111 description="The DeclarativeOAuth Specific optional override to provide the custom `code` key name to something like `auth_code` or `custom_auth_code`, if required by data-provider.", 1112 examples=["my_custom_auth_code_key_name"], 1113 title="Auth Code Key Override", 1114 ) 1115 redirect_uri_key: Optional[str] = Field( 1116 None, 1117 description="The DeclarativeOAuth Specific optional override to provide the custom `redirect_uri` key name to something like `callback_uri`, if required by data-provider.", 1118 examples=["my_custom_redirect_uri_key_name"], 1119 title="Redirect URI Key Override", 1120 ) 1121 1122 1123class OAuthConfigSpecification(BaseModel): 1124 class Config: 1125 extra = Extra.allow 1126 1127 oauth_user_input_from_connector_config_specification: Optional[Dict[str, Any]] = Field( 1128 None, 1129 description="OAuth specific blob. This is a Json Schema used to validate Json configurations used as input to OAuth.\nMust be a valid non-nested JSON that refers to properties from ConnectorSpecification.connectionSpecification\nusing special annotation 'path_in_connector_config'.\nThese are input values the user is entering through the UI to authenticate to the connector, that might also shared\nas inputs for syncing data via the connector.\nExamples:\nif no connector values is shared during oauth flow, oauth_user_input_from_connector_config_specification=[]\nif connector values such as 'app_id' inside the top level are used to generate the API url for the oauth flow,\n oauth_user_input_from_connector_config_specification={\n app_id: {\n type: string\n path_in_connector_config: ['app_id']\n }\n }\nif connector values such as 'info.app_id' nested inside another object are used to generate the API url for the oauth flow,\n oauth_user_input_from_connector_config_specification={\n app_id: {\n type: string\n path_in_connector_config: ['info', 'app_id']\n }\n }", 1130 examples=[ 1131 {"app_id": {"type": "string", "path_in_connector_config": ["app_id"]}}, 1132 { 1133 "app_id": { 1134 "type": "string", 1135 "path_in_connector_config": ["info", "app_id"], 1136 } 1137 }, 1138 ], 1139 title="OAuth user input", 1140 ) 1141 oauth_connector_input_specification: Optional[OauthConnectorInputSpecification] = Field( 1142 None, 1143 description='The DeclarativeOAuth specific blob.\nPertains to the fields defined by the connector relating to the OAuth flow.\n\nInterpolation capabilities:\n- The variables placeholders are declared as `{{my_var}}`.\n- The nested resolution variables like `{{ {{my_nested_var}} }}` is allowed as well.\n\n- The allowed interpolation context is:\n + base64Encoder - encode to `base64`, {{ {{my_var_a}}:{{my_var_b}} | base64Encoder }}\n + base64Decorer - decode from `base64` encoded string, {{ {{my_string_variable_or_string_value}} | base64Decoder }}\n + urlEncoder - encode the input string to URL-like format, {{ https://test.host.com/endpoint | urlEncoder}}\n + urlDecorer - decode the input url-encoded string into text format, {{ urlDecoder:https%3A%2F%2Fairbyte.io | urlDecoder}}\n + codeChallengeS256 - get the `codeChallenge` encoded value to provide additional data-provider specific authorisation values, {{ {{state_value}} | codeChallengeS256 }}\n\nExamples:\n - The TikTok Marketing DeclarativeOAuth spec:\n {\n "oauth_connector_input_specification": {\n "type": "object",\n "additionalProperties": false,\n "properties": {\n "consent_url": "https://ads.tiktok.com/marketing_api/auth?{{client_id_key}}={{client_id_value}}&{{redirect_uri_key}}={{ {{redirect_uri_value}} | urlEncoder}}&{{state_key}}={{state_value}}",\n "access_token_url": "https://business-api.tiktok.com/open_api/v1.3/oauth2/access_token/",\n "access_token_params": {\n "{{ auth_code_key }}": "{{ auth_code_value }}",\n "{{ client_id_key }}": "{{ client_id_value }}",\n "{{ client_secret_key }}": "{{ client_secret_value }}"\n },\n "access_token_headers": {\n "Content-Type": "application/json",\n "Accept": "application/json"\n },\n "extract_output": ["data.access_token"],\n "client_id_key": "app_id",\n "client_secret_key": "secret",\n "auth_code_key": "auth_code"\n }\n }\n }', 1144 title="DeclarativeOAuth Connector Specification", 1145 ) 1146 complete_oauth_output_specification: Optional[Dict[str, Any]] = Field( 1147 None, 1148 description="OAuth specific blob. This is a Json Schema used to validate Json configurations produced by the OAuth flows as they are\nreturned by the distant OAuth APIs.\nMust be a valid JSON describing the fields to merge back to `ConnectorSpecification.connectionSpecification`.\nFor each field, a special annotation `path_in_connector_config` can be specified to determine where to merge it,\nExamples:\n complete_oauth_output_specification={\n refresh_token: {\n type: string,\n path_in_connector_config: ['credentials', 'refresh_token']\n }\n }", 1149 examples=[ 1150 { 1151 "refresh_token": { 1152 "type": "string,", 1153 "path_in_connector_config": ["credentials", "refresh_token"], 1154 } 1155 } 1156 ], 1157 title="OAuth output specification", 1158 ) 1159 complete_oauth_server_input_specification: Optional[Dict[str, Any]] = Field( 1160 None, 1161 description="OAuth specific blob. This is a Json Schema used to validate Json configurations persisted as Airbyte Server configurations.\nMust be a valid non-nested JSON describing additional fields configured by the Airbyte Instance or Workspace Admins to be used by the\nserver when completing an OAuth flow (typically exchanging an auth code for refresh token).\nExamples:\n complete_oauth_server_input_specification={\n client_id: {\n type: string\n },\n client_secret: {\n type: string\n }\n }", 1162 examples=[{"client_id": {"type": "string"}, "client_secret": {"type": "string"}}], 1163 title="OAuth input specification", 1164 ) 1165 complete_oauth_server_output_specification: Optional[Dict[str, Any]] = Field( 1166 None, 1167 description="OAuth specific blob. This is a Json Schema used to validate Json configurations persisted as Airbyte Server configurations that\nalso need to be merged back into the connector configuration at runtime.\nThis is a subset configuration of `complete_oauth_server_input_specification` that filters fields out to retain only the ones that\nare necessary for the connector to function with OAuth. (some fields could be used during oauth flows but not needed afterwards, therefore\nthey would be listed in the `complete_oauth_server_input_specification` but not `complete_oauth_server_output_specification`)\nMust be a valid non-nested JSON describing additional fields configured by the Airbyte Instance or Workspace Admins to be used by the\nconnector when using OAuth flow APIs.\nThese fields are to be merged back to `ConnectorSpecification.connectionSpecification`.\nFor each field, a special annotation `path_in_connector_config` can be specified to determine where to merge it,\nExamples:\n complete_oauth_server_output_specification={\n client_id: {\n type: string,\n path_in_connector_config: ['credentials', 'client_id']\n },\n client_secret: {\n type: string,\n path_in_connector_config: ['credentials', 'client_secret']\n }\n }", 1168 examples=[ 1169 { 1170 "client_id": { 1171 "type": "string,", 1172 "path_in_connector_config": ["credentials", "client_id"], 1173 }, 1174 "client_secret": { 1175 "type": "string,", 1176 "path_in_connector_config": ["credentials", "client_secret"], 1177 }, 1178 } 1179 ], 1180 title="OAuth server output specification", 1181 ) 1182 1183 1184class OffsetIncrement(BaseModel): 1185 type: Literal["OffsetIncrement"] 1186 page_size: Optional[Union[int, str]] = Field( 1187 None, 1188 description="The number of records to include in each pages.", 1189 examples=[100, "{{ config['page_size'] }}"], 1190 title="Limit", 1191 ) 1192 inject_on_first_request: Optional[bool] = Field( 1193 False, 1194 description="Using the `offset` with value `0` during the first request", 1195 title="Inject Offset", 1196 ) 1197 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters") 1198 1199 1200class PageIncrement(BaseModel): 1201 type: Literal["PageIncrement"] 1202 page_size: Optional[Union[int, str]] = Field( 1203 None, 1204 description="The number of records to include in each pages.", 1205 examples=[100, "100", "{{ config['page_size'] }}"], 1206 title="Page Size", 1207 ) 1208 start_from_page: Optional[int] = Field( 1209 0, 1210 description="Index of the first page to request.", 1211 examples=[0, 1], 1212 title="Start From Page", 1213 ) 1214 inject_on_first_request: Optional[bool] = Field( 1215 False, 1216 description="Using the `page number` with value defined by `start_from_page` during the first request", 1217 title="Inject Page Number", 1218 ) 1219 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters") 1220 1221 1222class PrimaryKey(BaseModel): 1223 __root__: Union[str, List[str], List[List[str]]] = Field( 1224 ..., 1225 description="The stream field to be used to distinguish unique records. Can either be a single field, an array of fields representing a composite key, or an array of arrays representing a composite key where the fields are nested fields.", 1226 examples=["id", ["code", "type"]], 1227 title="Primary Key", 1228 ) 1229 1230 1231class PropertyLimitType(Enum): 1232 characters = "characters" 1233 property_count = "property_count" 1234 1235 1236class PropertyChunking(BaseModel): 1237 type: Literal["PropertyChunking"] 1238 property_limit_type: PropertyLimitType = Field( 1239 ..., 1240 description="The type used to determine the maximum number of properties per chunk", 1241 title="Property Limit Type", 1242 ) 1243 property_limit: Optional[int] = Field( 1244 None, 1245 description="The maximum amount of properties that can be retrieved per request according to the limit type.", 1246 title="Property Limit", 1247 ) 1248 record_merge_strategy: Optional[GroupByKeyMergeStrategy] = Field( 1249 None, 1250 description="Dictates how to records that require multiple requests to get all properties should be emitted to the destination", 1251 title="Record Merge Strategy", 1252 ) 1253 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters") 1254 1255 1256class RecordFilter(BaseModel): 1257 type: Literal["RecordFilter"] 1258 condition: Optional[str] = Field( 1259 "", 1260 description="The predicate to filter a record. Records will be removed if evaluated to False.", 1261 examples=[ 1262 "{{ record['created_at'] >= stream_interval['start_time'] }}", 1263 "{{ record.status in ['active', 'expired'] }}", 1264 ], 1265 title="Condition", 1266 ) 1267 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters") 1268 1269 1270class SchemaNormalization(Enum): 1271 Default = "Default" 1272 None_ = "None" 1273 1274 1275class RemoveFields(BaseModel): 1276 type: Literal["RemoveFields"] 1277 condition: Optional[str] = Field( 1278 "", 1279 description="The predicate to filter a property by a property value. Property will be removed if it is empty OR expression is evaluated to True.,", 1280 examples=[ 1281 "{{ property|string == '' }}", 1282 "{{ property is integer }}", 1283 "{{ property|length > 5 }}", 1284 "{{ property == 'some_string_to_match' }}", 1285 ], 1286 ) 1287 field_pointers: List[List[str]] = Field( 1288 ..., 1289 description="Array of paths defining the field to remove. Each item is an array whose field describe the path of a field to remove.", 1290 examples=[["tags"], [["content", "html"], ["content", "plain_text"]]], 1291 title="Field Paths", 1292 ) 1293 1294 1295class RequestPath(BaseModel): 1296 type: Literal["RequestPath"] 1297 1298 1299class InjectInto(Enum): 1300 request_parameter = "request_parameter" 1301 header = "header" 1302 body_data = "body_data" 1303 body_json = "body_json" 1304 1305 1306class RequestOption(BaseModel): 1307 type: Literal["RequestOption"] 1308 field_name: Optional[str] = Field( 1309 None, 1310 description="Configures which key should be used in the location that the descriptor is being injected into. We hope to eventually deprecate this field in favor of `field_path` for all request_options, but must currently maintain it for backwards compatibility in the Builder.", 1311 examples=["segment_id"], 1312 title="Field Name", 1313 ) 1314 field_path: Optional[List[str]] = Field( 1315 None, 1316 description="Configures a path to be used for nested structures in JSON body requests (e.g. GraphQL queries)", 1317 examples=[["data", "viewer", "id"]], 1318 title="Field Path", 1319 ) 1320 inject_into: InjectInto = Field( 1321 ..., 1322 description="Configures where the descriptor should be set on the HTTP requests. Note that request parameters that are already encoded in the URL path will not be duplicated.", 1323 examples=["request_parameter", "header", "body_data", "body_json"], 1324 title="Inject Into", 1325 ) 1326 1327 1328class Schemas(BaseModel): 1329 pass 1330 1331 class Config: 1332 extra = Extra.allow 1333 1334 1335class LegacySessionTokenAuthenticator(BaseModel): 1336 type: Literal["LegacySessionTokenAuthenticator"] 1337 header: str = Field( 1338 ..., 1339 description="The name of the session token header that will be injected in the request", 1340 examples=["X-Session"], 1341 title="Session Request Header", 1342 ) 1343 login_url: str = Field( 1344 ..., 1345 description="Path of the login URL (do not include the base URL)", 1346 examples=["session"], 1347 title="Login Path", 1348 ) 1349 session_token: Optional[str] = Field( 1350 None, 1351 description="Session token to use if using a pre-defined token. Not needed if authenticating with username + password pair", 1352 example=["{{ config['session_token'] }}"], 1353 title="Session Token", 1354 ) 1355 session_token_response_key: str = Field( 1356 ..., 1357 description="Name of the key of the session token to be extracted from the response", 1358 examples=["id"], 1359 title="Response Token Response Key", 1360 ) 1361 username: Optional[str] = Field( 1362 None, 1363 description="Username used to authenticate and obtain a session token", 1364 examples=[" {{ config['username'] }}"], 1365 title="Username", 1366 ) 1367 password: Optional[str] = Field( 1368 "", 1369 description="Password used to authenticate and obtain a session token", 1370 examples=["{{ config['password'] }}", ""], 1371 title="Password", 1372 ) 1373 validate_session_url: str = Field( 1374 ..., 1375 description="Path of the URL to use to validate that the session token is valid (do not include the base URL)", 1376 examples=["user/current"], 1377 title="Validate Session Path", 1378 ) 1379 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters") 1380 1381 1382class CsvDecoder(BaseModel): 1383 type: Literal["CsvDecoder"] 1384 encoding: Optional[str] = "utf-8" 1385 delimiter: Optional[str] = "," 1386 1387 1388class AsyncJobStatusMap(BaseModel): 1389 type: Optional[Literal["AsyncJobStatusMap"]] = None 1390 running: List[str] 1391 completed: List[str] 1392 failed: List[str] 1393 timeout: List[str] 1394 1395 1396class ValueType(Enum): 1397 string = "string" 1398 number = "number" 1399 integer = "integer" 1400 boolean = "boolean" 1401 1402 1403class WaitTimeFromHeader(BaseModel): 1404 type: Literal["WaitTimeFromHeader"] 1405 header: str = Field( 1406 ..., 1407 description="The name of the response header defining how long to wait before retrying.", 1408 examples=["Retry-After"], 1409 title="Response Header Name", 1410 ) 1411 regex: Optional[str] = Field( 1412 None, 1413 description="Optional regex to apply on the header to extract its value. The regex should define a capture group defining the wait time.", 1414 examples=["([-+]?\\d+)"], 1415 title="Extraction Regex", 1416 ) 1417 max_waiting_time_in_seconds: Optional[float] = Field( 1418 None, 1419 description="Given the value extracted from the header is greater than this value, stop the stream.", 1420 examples=[3600], 1421 title="Max Waiting Time in Seconds", 1422 ) 1423 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters") 1424 1425 1426class WaitUntilTimeFromHeader(BaseModel): 1427 type: Literal["WaitUntilTimeFromHeader"] 1428 header: str = Field( 1429 ..., 1430 description="The name of the response header defining how long to wait before retrying.", 1431 examples=["wait_time"], 1432 title="Response Header", 1433 ) 1434 min_wait: Optional[Union[float, str]] = Field( 1435 None, 1436 description="Minimum time to wait before retrying.", 1437 examples=[10, "60"], 1438 title="Minimum Wait Time", 1439 ) 1440 regex: Optional[str] = Field( 1441 None, 1442 description="Optional regex to apply on the header to extract its value. The regex should define a capture group defining the wait time.", 1443 examples=["([-+]?\\d+)"], 1444 title="Extraction Regex", 1445 ) 1446 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters") 1447 1448 1449class ComponentMappingDefinition(BaseModel): 1450 type: Literal["ComponentMappingDefinition"] 1451 field_path: List[str] = Field( 1452 ..., 1453 description="A list of potentially nested fields indicating the full path where value will be added or updated.", 1454 examples=[ 1455 ["data"], 1456 ["data", "records"], 1457 ["data", 1, "name"], 1458 ["data", "{{ components_values.name }}"], 1459 ["data", "*", "record"], 1460 ["*", "**", "name"], 1461 ], 1462 title="Field Path", 1463 ) 1464 value: str = Field( 1465 ..., 1466 description="The dynamic or static value to assign to the key. Interpolated values can be used to dynamically determine the value during runtime.", 1467 examples=[ 1468 "{{ components_values['updates'] }}", 1469 "{{ components_values['MetaData']['LastUpdatedTime'] }}", 1470 "{{ config['segment_id'] }}", 1471 "{{ stream_slice['parent_id'] }}", 1472 "{{ stream_slice['extra_fields']['name'] }}", 1473 ], 1474 title="Value", 1475 ) 1476 value_type: Optional[ValueType] = Field( 1477 None, 1478 description="The expected data type of the value. If omitted, the type will be inferred from the value provided.", 1479 title="Value Type", 1480 ) 1481 create_or_update: Optional[bool] = Field( 1482 False, 1483 description="Determines whether to create a new path if it doesn't exist (true) or only update existing paths (false). When set to true, the resolver will create new paths in the stream template if they don't exist. When false (default), it will only update existing paths.", 1484 title="Create or Update", 1485 ) 1486 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters") 1487 1488 1489class StreamConfig(BaseModel): 1490 type: Literal["StreamConfig"] 1491 configs_pointer: List[str] = Field( 1492 ..., 1493 description="A list of potentially nested fields indicating the full path in source config file where streams configs located.", 1494 examples=[["data"], ["data", "streams"], ["data", "{{ parameters.name }}"]], 1495 title="Configs Pointer", 1496 ) 1497 default_values: Optional[List[Dict[str, Any]]] = Field( 1498 None, 1499 description="A list of default values, each matching the structure expected from the parsed component value.", 1500 title="Default Values", 1501 ) 1502 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters") 1503 1504 1505class ConfigComponentsResolver(BaseModel): 1506 type: Literal["ConfigComponentsResolver"] 1507 stream_config: Union[List[StreamConfig], StreamConfig] 1508 components_mapping: List[ComponentMappingDefinition] 1509 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters") 1510 1511 1512class RequestBodyPlainText(BaseModel): 1513 type: Literal["RequestBodyPlainText"] 1514 value: str 1515 1516 1517class RequestBodyUrlEncodedForm(BaseModel): 1518 type: Literal["RequestBodyUrlEncodedForm"] 1519 value: Dict[str, str] 1520 1521 1522class RequestBodyJsonObject(BaseModel): 1523 type: Literal["RequestBodyJsonObject"] 1524 value: Dict[str, Any] 1525 1526 1527class RequestBodyGraphQlQuery(BaseModel): 1528 class Config: 1529 extra = Extra.allow 1530 1531 query: Dict[str, Any] = Field(..., description="The GraphQL query to be executed") 1532 1533 1534class ValidateAdheresToSchema(BaseModel): 1535 type: Literal["ValidateAdheresToSchema"] 1536 base_schema: Union[str, Dict[str, Any]] = Field( 1537 ..., 1538 description="The base JSON schema against which the user-provided schema will be validated.", 1539 examples=[ 1540 "{{ config['report_validation_schema'] }}", 1541 '\'{\n "$schema": "http://json-schema.org/draft-07/schema#",\n "title": "Person",\n "type": "object",\n "properties": {\n "name": {\n "type": "string",\n "description": "The person\'s name"\n },\n "age": {\n "type": "integer",\n "minimum": 0,\n "description": "The person\'s age"\n }\n },\n "required": ["name", "age"]\n}\'\n', 1542 { 1543 "$schema": "http://json-schema.org/draft-07/schema#", 1544 "title": "Person", 1545 "type": "object", 1546 "properties": { 1547 "name": {"type": "string", "description": "The person's name"}, 1548 "age": { 1549 "type": "integer", 1550 "minimum": 0, 1551 "description": "The person's age", 1552 }, 1553 }, 1554 "required": ["name", "age"], 1555 }, 1556 ], 1557 title="Base JSON Schema", 1558 ) 1559 1560 1561class ConfigRemapField(BaseModel): 1562 type: Literal["ConfigRemapField"] 1563 map: Union[Dict[str, Any], str] = Field( 1564 ..., 1565 description="A mapping of original values to new values. When a field value matches a key in this map, it will be replaced with the corresponding value.", 1566 examples=[ 1567 {"pending": "in_progress", "done": "completed", "cancelled": "terminated"}, 1568 "{{ config['status_mapping'] }}", 1569 ], 1570 title="Value Mapping", 1571 ) 1572 field_path: List[str] = Field( 1573 ..., 1574 description="The path to the field whose value should be remapped. Specified as a list of path components to navigate through nested objects.", 1575 examples=[ 1576 ["status"], 1577 ["data", "status"], 1578 ["data", "{{ config.name }}", "status"], 1579 ["data", "*", "status"], 1580 ], 1581 title="Field Path", 1582 ) 1583 1584 1585class ConfigRemoveFields(BaseModel): 1586 type: Literal["ConfigRemoveFields"] 1587 field_pointers: List[List[str]] = Field( 1588 ..., 1589 description="A list of field pointers to be removed from the config.", 1590 examples=[["tags"], [["content", "html"], ["content", "plain_text"]]], 1591 title="Field Pointers", 1592 ) 1593 condition: Optional[str] = Field( 1594 "", 1595 description="Fields will be removed if expression is evaluated to True.", 1596 examples=[ 1597 "{{ config['environemnt'] == 'sandbox' }}", 1598 "{{ property is integer }}", 1599 "{{ property|length > 5 }}", 1600 "{{ property == 'some_string_to_match' }}", 1601 ], 1602 ) 1603 1604 1605class AddedFieldDefinition(BaseModel): 1606 type: Literal["AddedFieldDefinition"] 1607 path: List[str] = Field( 1608 ..., 1609 description="List of strings defining the path where to add the value on the record.", 1610 examples=[["segment_id"], ["metadata", "segment_id"]], 1611 title="Path", 1612 ) 1613 value: str = Field( 1614 ..., 1615 description="Value of the new field. Use {{ record['existing_field'] }} syntax to refer to other fields in the record.", 1616 examples=[ 1617 "{{ record['updates'] }}", 1618 "{{ record['MetaData']['LastUpdatedTime'] }}", 1619 "{{ stream_partition['segment_id'] }}", 1620 ], 1621 title="Value", 1622 ) 1623 value_type: Optional[ValueType] = Field( 1624 None, 1625 description="Type of the value. If not specified, the type will be inferred from the value.", 1626 title="Value Type", 1627 ) 1628 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters") 1629 1630 1631class AddFields(BaseModel): 1632 type: Literal["AddFields"] 1633 fields: List[AddedFieldDefinition] = Field( 1634 ..., 1635 description="List of transformations (path and corresponding value) that will be added to the record.", 1636 title="Fields", 1637 ) 1638 condition: Optional[str] = Field( 1639 "", 1640 description="Fields will be added if expression is evaluated to True.", 1641 examples=[ 1642 "{{ property|string == '' }}", 1643 "{{ property is integer }}", 1644 "{{ property|length > 5 }}", 1645 "{{ property == 'some_string_to_match' }}", 1646 ], 1647 ) 1648 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters") 1649 1650 1651class ApiKeyAuthenticator(BaseModel): 1652 type: Literal["ApiKeyAuthenticator"] 1653 api_token: Optional[str] = Field( 1654 None, 1655 description="The API key to inject in the request. Fill it in the user inputs.", 1656 examples=["{{ config['api_key'] }}", "Token token={{ config['api_key'] }}"], 1657 title="API Key", 1658 ) 1659 header: Optional[str] = Field( 1660 None, 1661 description="The name of the HTTP header that will be set to the API key. This setting is deprecated, use inject_into instead. Header and inject_into can not be defined at the same time.", 1662 examples=["Authorization", "Api-Token", "X-Auth-Token"], 1663 title="Header Name", 1664 ) 1665 inject_into: Optional[RequestOption] = Field( 1666 None, 1667 description="Configure how the API Key will be sent in requests to the source API. Either inject_into or header has to be defined.", 1668 examples=[ 1669 {"inject_into": "header", "field_name": "Authorization"}, 1670 {"inject_into": "request_parameter", "field_name": "authKey"}, 1671 ], 1672 title="Inject API Key Into Outgoing HTTP Request", 1673 ) 1674 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters") 1675 1676 1677class AuthFlow(BaseModel): 1678 auth_flow_type: Optional[AuthFlowType] = Field( 1679 None, description="The type of auth to use", title="Auth flow type" 1680 ) 1681 predicate_key: Optional[List[str]] = Field( 1682 None, 1683 description="JSON path to a field in the connectorSpecification that should exist for the advanced auth to be applicable.", 1684 examples=[["credentials", "auth_type"]], 1685 title="Predicate key", 1686 ) 1687 predicate_value: Optional[str] = Field( 1688 None, 1689 description="Value of the predicate_key fields for the advanced auth to be applicable.", 1690 examples=["Oauth"], 1691 title="Predicate value", 1692 ) 1693 oauth_config_specification: Optional[OAuthConfigSpecification] = None 1694 1695 1696class CheckStream(BaseModel): 1697 type: Literal["CheckStream"] 1698 stream_names: Optional[List[str]] = Field( 1699 None, 1700 description="Names of the streams to try reading from when running a check operation.", 1701 examples=[["users"], ["users", "contacts"]], 1702 title="Stream Names", 1703 ) 1704 dynamic_streams_check_configs: Optional[List[DynamicStreamCheckConfig]] = None 1705 1706 1707class IncrementingCountCursor(BaseModel): 1708 type: Literal["IncrementingCountCursor"] 1709 cursor_field: str = Field( 1710 ..., 1711 description="The location of the value on a record that will be used as a bookmark during sync. To ensure no data loss, the API must return records in ascending order based on the cursor field. Nested fields are not supported, so the field must be at the top level of the record. You can use a combination of Add Field and Remove Field transformations to move the nested field to the top.", 1712 examples=["created_at", "{{ config['record_cursor'] }}"], 1713 title="Cursor Field", 1714 ) 1715 start_value: Optional[Union[str, int]] = Field( 1716 None, 1717 description="The value that determines the earliest record that should be synced.", 1718 examples=[0, "{{ config['start_value'] }}"], 1719 title="Start Value", 1720 ) 1721 start_value_option: Optional[RequestOption] = Field( 1722 None, 1723 description="Optionally configures how the start value will be sent in requests to the source API.", 1724 title="Inject Start Value Into Outgoing HTTP Request", 1725 ) 1726 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters") 1727 1728 1729class DatetimeBasedCursor(BaseModel): 1730 type: Literal["DatetimeBasedCursor"] 1731 clamping: Optional[Clamping] = Field( 1732 None, 1733 description="This option is used to adjust the upper and lower boundaries of each datetime window to beginning and end of the provided target period (day, week, month)", 1734 title="Date Range Clamping", 1735 ) 1736 cursor_field: str = Field( 1737 ..., 1738 description="The location of the value on a record that will be used as a bookmark during sync. To ensure no data loss, the API must return records in ascending order based on the cursor field. Nested fields are not supported, so the field must be at the top level of the record. You can use a combination of Add Field and Remove Field transformations to move the nested field to the top.", 1739 examples=["created_at", "{{ config['record_cursor'] }}"], 1740 title="Cursor Field", 1741 ) 1742 datetime_format: str = Field( 1743 ..., 1744 description="The datetime format used to format the datetime values that are sent in outgoing requests to the API. Use placeholders starting with \"%\" to describe the format the API is using. The following placeholders are available:\n * **%s**: Epoch unix timestamp - `1686218963`\n * **%s_as_float**: Epoch unix timestamp in seconds as float with microsecond precision - `1686218963.123456`\n * **%ms**: Epoch unix timestamp (milliseconds) - `1686218963123`\n * **%a**: Weekday (abbreviated) - `Sun`\n * **%A**: Weekday (full) - `Sunday`\n * **%w**: Weekday (decimal) - `0` (Sunday), `6` (Saturday)\n * **%d**: Day of the month (zero-padded) - `01`, `02`, ..., `31`\n * **%b**: Month (abbreviated) - `Jan`\n * **%B**: Month (full) - `January`\n * **%m**: Month (zero-padded) - `01`, `02`, ..., `12`\n * **%y**: Year (without century, zero-padded) - `00`, `01`, ..., `99`\n * **%Y**: Year (with century) - `0001`, `0002`, ..., `9999`\n * **%H**: Hour (24-hour, zero-padded) - `00`, `01`, ..., `23`\n * **%I**: Hour (12-hour, zero-padded) - `01`, `02`, ..., `12`\n * **%p**: AM/PM indicator\n * **%M**: Minute (zero-padded) - `00`, `01`, ..., `59`\n * **%S**: Second (zero-padded) - `00`, `01`, ..., `59`\n * **%f**: Microsecond (zero-padded to 6 digits) - `000000`\n * **%_ms**: Millisecond (zero-padded to 3 digits) - `000`\n * **%z**: UTC offset - `(empty)`, `+0000`, `-04:00`\n * **%Z**: Time zone name - `(empty)`, `UTC`, `GMT`\n * **%j**: Day of the year (zero-padded) - `001`, `002`, ..., `366`\n * **%U**: Week number of the year (starting Sunday) - `00`, ..., `53`\n * **%W**: Week number of the year (starting Monday) - `00`, ..., `53`\n * **%c**: Date and time - `Tue Aug 16 21:30:00 1988`\n * **%x**: Date standard format - `08/16/1988`\n * **%X**: Time standard format - `21:30:00`\n * **%%**: Literal '%' character\n\n Some placeholders depend on the locale of the underlying system - in most cases this locale is configured as en/US. For more information see the [Python documentation](https://docs.python.org/3/library/datetime.html#strftime-and-strptime-format-codes).\n", 1745 examples=["%Y-%m-%dT%H:%M:%S.%f%z", "%Y-%m-%d", "%s", "%ms", "%s_as_float"], 1746 title="Outgoing Datetime Format", 1747 ) 1748 start_datetime: Union[str, MinMaxDatetime] = Field( 1749 ..., 1750 description="The datetime that determines the earliest record that should be synced.", 1751 examples=["2020-01-1T00:00:00Z", "{{ config['start_time'] }}"], 1752 title="Start Datetime", 1753 ) 1754 cursor_datetime_formats: Optional[List[str]] = Field( 1755 None, 1756 description="The possible formats for the cursor field, in order of preference. The first format that matches the cursor field value will be used to parse it. If not provided, the `datetime_format` will be used.", 1757 title="Cursor Datetime Formats", 1758 ) 1759 cursor_granularity: Optional[str] = Field( 1760 None, 1761 description="Smallest increment the datetime_format has (ISO 8601 duration) that is used to ensure the start of a slice does not overlap with the end of the previous one, e.g. for %Y-%m-%d the granularity should be P1D, for %Y-%m-%dT%H:%M:%SZ the granularity should be PT1S. Given this field is provided, `step` needs to be provided as well.", 1762 examples=["PT1S"], 1763 title="Cursor Granularity", 1764 ) 1765 end_datetime: Optional[Union[str, MinMaxDatetime]] = Field( 1766 None, 1767 description="The datetime that determines the last record that should be synced. If not provided, `{{ now_utc() }}` will be used.", 1768 examples=["2021-01-1T00:00:00Z", "{{ now_utc() }}", "{{ day_delta(-1) }}"], 1769 title="End Datetime", 1770 ) 1771 end_time_option: Optional[RequestOption] = Field( 1772 None, 1773 description="Optionally configures how the end datetime will be sent in requests to the source API.", 1774 title="Inject End Time Into Outgoing HTTP Request", 1775 ) 1776 is_data_feed: Optional[bool] = Field( 1777 None, 1778 description="A data feed API is an API that does not allow filtering and paginates the content from the most recent to the least recent. Given this, the CDK needs to know when to stop paginating and this field will generate a stop condition for pagination.", 1779 title="Whether the target API is formatted as a data feed", 1780 ) 1781 is_client_side_incremental: Optional[bool] = Field( 1782 None, 1783 description="If the target API endpoint does not take cursor values to filter records and returns all records anyway, the connector with this cursor will filter out records locally, and only emit new records from the last sync, hence incremental. This means that all records would be read from the API, but only new records will be emitted to the destination.", 1784 title="Whether the target API does not support filtering and returns all data (the cursor filters records in the client instead of the API side)", 1785 ) 1786 is_compare_strictly: Optional[bool] = Field( 1787 False, 1788 description="Set to True if the target API does not accept queries where the start time equal the end time.", 1789 title="Whether to skip requests if the start time equals the end time", 1790 ) 1791 global_substream_cursor: Optional[bool] = Field( 1792 False, 1793 description="This setting optimizes performance when the parent stream has thousands of partitions by storing the cursor as a single value rather than per partition. Notably, the substream state is updated only at the end of the sync, which helps prevent data loss in case of a sync failure. See more info in the [docs](https://docs.airbyte.com/connector-development/config-based/understanding-the-yaml-file/incremental-syncs).", 1794 title="Whether to store cursor as one value instead of per partition", 1795 ) 1796 lookback_window: Optional[str] = Field( 1797 None, 1798 description="Time interval before the start_datetime to read data for, e.g. P1M for looking back one month.", 1799 examples=["P1D", "P{{ config['lookback_days'] }}D"], 1800 title="Lookback Window", 1801 ) 1802 partition_field_end: Optional[str] = Field( 1803 None, 1804 description="Name of the partition start time field.", 1805 examples=["ending_time"], 1806 title="Partition Field End", 1807 ) 1808 partition_field_start: Optional[str] = Field( 1809 None, 1810 description="Name of the partition end time field.", 1811 examples=["starting_time"], 1812 title="Partition Field Start", 1813 ) 1814 start_time_option: Optional[RequestOption] = Field( 1815 None, 1816 description="Optionally configures how the start datetime will be sent in requests to the source API.", 1817 title="Inject Start Time Into Outgoing HTTP Request", 1818 ) 1819 step: Optional[str] = Field( 1820 None, 1821 description="The size of the time window (ISO8601 duration). Given this field is provided, `cursor_granularity` needs to be provided as well.", 1822 examples=["P1W", "{{ config['step_increment'] }}"], 1823 title="Step", 1824 ) 1825 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters") 1826 1827 1828class FixedWindowCallRatePolicy(BaseModel): 1829 class Config: 1830 extra = Extra.allow 1831 1832 type: Literal["FixedWindowCallRatePolicy"] 1833 period: str = Field( 1834 ..., description="The time interval for the rate limit window.", title="Period" 1835 ) 1836 call_limit: int = Field( 1837 ..., 1838 description="The maximum number of calls allowed within the period.", 1839 title="Call Limit", 1840 ) 1841 matchers: List[HttpRequestRegexMatcher] = Field( 1842 ..., 1843 description="List of matchers that define which requests this policy applies to.", 1844 title="Matchers", 1845 ) 1846 1847 1848class MovingWindowCallRatePolicy(BaseModel): 1849 class Config: 1850 extra = Extra.allow 1851 1852 type: Literal["MovingWindowCallRatePolicy"] 1853 rates: List[Rate] = Field( 1854 ..., 1855 description="List of rates that define the call limits for different time intervals.", 1856 title="Rates", 1857 ) 1858 matchers: List[HttpRequestRegexMatcher] = Field( 1859 ..., 1860 description="List of matchers that define which requests this policy applies to.", 1861 title="Matchers", 1862 ) 1863 1864 1865class UnlimitedCallRatePolicy(BaseModel): 1866 class Config: 1867 extra = Extra.allow 1868 1869 type: Literal["UnlimitedCallRatePolicy"] 1870 matchers: List[HttpRequestRegexMatcher] = Field( 1871 ..., 1872 description="List of matchers that define which requests this policy applies to.", 1873 title="Matchers", 1874 ) 1875 1876 1877class DefaultErrorHandler(BaseModel): 1878 type: Literal["DefaultErrorHandler"] 1879 backoff_strategies: Optional[ 1880 List[ 1881 Union[ 1882 ConstantBackoffStrategy, 1883 CustomBackoffStrategy, 1884 ExponentialBackoffStrategy, 1885 WaitTimeFromHeader, 1886 WaitUntilTimeFromHeader, 1887 ] 1888 ] 1889 ] = Field( 1890 None, 1891 description="List of backoff strategies to use to determine how long to wait before retrying a retryable request.", 1892 title="Backoff Strategies", 1893 ) 1894 max_retries: Optional[int] = Field( 1895 5, 1896 description="The maximum number of time to retry a retryable request before giving up and failing.", 1897 examples=[5, 0, 10], 1898 title="Max Retry Count", 1899 ) 1900 response_filters: Optional[List[HttpResponseFilter]] = Field( 1901 None, 1902 description="List of response filters to iterate on when deciding how to handle an error. When using an array of multiple filters, the filters will be applied sequentially and the response will be selected if it matches any of the filter's predicate.", 1903 title="Response Filters", 1904 ) 1905 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters") 1906 1907 1908class DefaultPaginator(BaseModel): 1909 type: Literal["DefaultPaginator"] 1910 pagination_strategy: Union[ 1911 PageIncrement, OffsetIncrement, CursorPagination, CustomPaginationStrategy 1912 ] = Field( 1913 ..., 1914 description="Strategy defining how records are paginated.", 1915 title="Pagination Strategy", 1916 ) 1917 page_size_option: Optional[RequestOption] = Field( 1918 None, title="Inject Page Size Into Outgoing HTTP Request" 1919 ) 1920 page_token_option: Optional[Union[RequestOption, RequestPath]] = Field( 1921 None, title="Inject Page Token Into Outgoing HTTP Request" 1922 ) 1923 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters") 1924 1925 1926class SessionTokenRequestApiKeyAuthenticator(BaseModel): 1927 type: Literal["ApiKey"] 1928 inject_into: RequestOption = Field( 1929 ..., 1930 description="Configure how the API Key will be sent in requests to the source API.", 1931 examples=[ 1932 {"inject_into": "header", "field_name": "Authorization"}, 1933 {"inject_into": "request_parameter", "field_name": "authKey"}, 1934 ], 1935 title="Inject API Key Into Outgoing HTTP Request", 1936 ) 1937 1938 1939class ListPartitionRouter(BaseModel): 1940 type: Literal["ListPartitionRouter"] 1941 cursor_field: str = Field( 1942 ..., 1943 description='While iterating over list values, the name of field used to reference a list value. The partition value can be accessed with string interpolation. e.g. "{{ stream_partition[\'my_key\'] }}" where "my_key" is the value of the cursor_field.', 1944 examples=["section", "{{ config['section_key'] }}"], 1945 title="Current Partition Value Identifier", 1946 ) 1947 values: Union[str, List[str]] = Field( 1948 ..., 1949 description="The list of attributes being iterated over and used as input for the requests made to the source API.", 1950 examples=[["section_a", "section_b", "section_c"], "{{ config['sections'] }}"], 1951 title="Partition Values", 1952 ) 1953 request_option: Optional[RequestOption] = Field( 1954 None, 1955 description="A request option describing where the list value should be injected into and under what field name if applicable.", 1956 title="Inject Partition Value Into Outgoing HTTP Request", 1957 ) 1958 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters") 1959 1960 1961class RecordSelector(BaseModel): 1962 type: Literal["RecordSelector"] 1963 extractor: Union[DpathExtractor, CustomRecordExtractor] 1964 record_filter: Optional[Union[RecordFilter, CustomRecordFilter]] = Field( 1965 None, 1966 description="Responsible for filtering records to be emitted by the Source.", 1967 title="Record Filter", 1968 ) 1969 schema_normalization: Optional[Union[SchemaNormalization, CustomSchemaNormalization]] = Field( 1970 None, 1971 description="Responsible for normalization according to the schema.", 1972 title="Schema Normalization", 1973 ) 1974 transform_before_filtering: Optional[bool] = Field( 1975 None, 1976 description="If true, transformation will be applied before record filtering.", 1977 title="Transform Before Filtering", 1978 ) 1979 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters") 1980 1981 1982class GzipDecoder(BaseModel): 1983 type: Literal["GzipDecoder"] 1984 decoder: Union[CsvDecoder, GzipDecoder, JsonDecoder, JsonlDecoder] 1985 1986 1987class RequestBodyGraphQL(BaseModel): 1988 type: Literal["RequestBodyGraphQL"] 1989 value: RequestBodyGraphQlQuery 1990 1991 1992class DpathValidator(BaseModel): 1993 type: Literal["DpathValidator"] 1994 field_path: List[str] = Field( 1995 ..., 1996 description='List of potentially nested fields describing the full path of the field to validate. Use "*" to validate all values from an array.', 1997 examples=[ 1998 ["data"], 1999 ["data", "records"], 2000 ["data", "{{ parameters.name }}"], 2001 ["data", "*", "record"], 2002 ], 2003 title="Field Path", 2004 ) 2005 validation_strategy: ValidateAdheresToSchema = Field( 2006 ..., 2007 description="The condition that the specified config value will be evaluated against", 2008 title="Validation Strategy", 2009 ) 2010 2011 2012class PredicateValidator(BaseModel): 2013 type: Literal["PredicateValidator"] 2014 value: Optional[Union[str, float, Dict[str, Any], List[Any], bool]] = Field( 2015 ..., 2016 description="The value to be validated. Can be a literal value or interpolated from configuration.", 2017 examples=[ 2018 "test-value", 2019 "{{ config['api_version'] }}", 2020 "{{ config['tenant_id'] }}", 2021 123, 2022 ], 2023 title="Value", 2024 ) 2025 validation_strategy: ValidateAdheresToSchema = Field( 2026 ..., 2027 description="The validation strategy to apply to the value.", 2028 title="Validation Strategy", 2029 ) 2030 2031 2032class ConfigAddFields(BaseModel): 2033 type: Literal["ConfigAddFields"] 2034 fields: List[AddedFieldDefinition] = Field( 2035 ..., 2036 description="A list of transformations (path and corresponding value) that will be added to the config.", 2037 title="Fields", 2038 ) 2039 condition: Optional[str] = Field( 2040 "", 2041 description="Fields will be added if expression is evaluated to True.", 2042 examples=[ 2043 "{{ config['environemnt'] == 'sandbox' }}", 2044 "{{ property is integer }}", 2045 "{{ property|length > 5 }}", 2046 "{{ property == 'some_string_to_match' }}", 2047 ], 2048 ) 2049 2050 2051class CompositeErrorHandler(BaseModel): 2052 type: Literal["CompositeErrorHandler"] 2053 error_handlers: List[Union[CompositeErrorHandler, DefaultErrorHandler]] = Field( 2054 ..., 2055 description="List of error handlers to iterate on to determine how to handle a failed response.", 2056 title="Error Handlers", 2057 ) 2058 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters") 2059 2060 2061class HTTPAPIBudget(BaseModel): 2062 class Config: 2063 extra = Extra.allow 2064 2065 type: Literal["HTTPAPIBudget"] 2066 policies: List[ 2067 Union[ 2068 FixedWindowCallRatePolicy, 2069 MovingWindowCallRatePolicy, 2070 UnlimitedCallRatePolicy, 2071 ] 2072 ] = Field( 2073 ..., 2074 description="List of call rate policies that define how many calls are allowed.", 2075 title="Policies", 2076 ) 2077 ratelimit_reset_header: Optional[str] = Field( 2078 "ratelimit-reset", 2079 description="The HTTP response header name that indicates when the rate limit resets.", 2080 title="Rate Limit Reset Header", 2081 ) 2082 ratelimit_remaining_header: Optional[str] = Field( 2083 "ratelimit-remaining", 2084 description="The HTTP response header name that indicates the number of remaining allowed calls.", 2085 title="Rate Limit Remaining Header", 2086 ) 2087 status_codes_for_ratelimit_hit: Optional[List[int]] = Field( 2088 [429], 2089 description="List of HTTP status codes that indicate a rate limit has been hit.", 2090 title="Status Codes for Rate Limit Hit", 2091 ) 2092 2093 2094class ZipfileDecoder(BaseModel): 2095 class Config: 2096 extra = Extra.allow 2097 2098 type: Literal["ZipfileDecoder"] 2099 decoder: Union[CsvDecoder, GzipDecoder, JsonDecoder, JsonlDecoder] = Field( 2100 ..., 2101 description="Parser to parse the decompressed data from the zipfile(s).", 2102 title="Parser", 2103 ) 2104 2105 2106class ConfigMigration(BaseModel): 2107 type: Literal["ConfigMigration"] 2108 description: Optional[str] = Field( 2109 None, description="The description/purpose of the config migration." 2110 ) 2111 transformations: List[Union[ConfigRemapField, ConfigAddFields, ConfigRemoveFields]] = Field( 2112 ..., 2113 description="The list of transformations that will attempt to be applied on an incoming unmigrated config. The transformations will be applied in the order they are defined.", 2114 title="Transformations", 2115 ) 2116 2117 2118class ConfigNormalizationRules(BaseModel): 2119 class Config: 2120 extra = Extra.forbid 2121 2122 config_migrations: Optional[List[ConfigMigration]] = Field( 2123 [], 2124 description="The discrete migrations that will be applied on the incoming config. Each migration will be applied in the order they are defined.", 2125 title="Config Migrations", 2126 ) 2127 transformations: Optional[ 2128 List[Union[ConfigRemapField, ConfigAddFields, ConfigRemoveFields]] 2129 ] = Field( 2130 [], 2131 description="The list of transformations that will be applied on the incoming config at the start of each sync. The transformations will be applied in the order they are defined.", 2132 title="Transformations", 2133 ) 2134 validations: Optional[List[Union[DpathValidator, PredicateValidator]]] = Field( 2135 [], 2136 description="The list of validations that will be performed on the incoming config at the start of each sync.", 2137 title="Validations", 2138 ) 2139 2140 2141class Spec(BaseModel): 2142 type: Literal["Spec"] 2143 connection_specification: Dict[str, Any] = Field( 2144 ..., 2145 description="A connection specification describing how a the connector can be configured.", 2146 title="Connection Specification", 2147 ) 2148 documentation_url: Optional[str] = Field( 2149 None, 2150 description="URL of the connector's documentation page.", 2151 examples=["https://docs.airbyte.com/integrations/sources/dremio"], 2152 title="Documentation URL", 2153 ) 2154 advanced_auth: Optional[AuthFlow] = Field( 2155 None, 2156 description="Advanced specification for configuring the authentication flow.", 2157 title="Advanced Auth", 2158 ) 2159 config_normalization_rules: Optional[ConfigNormalizationRules] = Field( 2160 None, title="Config Normalization Rules" 2161 ) 2162 2163 2164class DeclarativeSource1(BaseModel): 2165 class Config: 2166 extra = Extra.forbid 2167 2168 type: Literal["DeclarativeSource"] 2169 check: Union[CheckStream, CheckDynamicStream] 2170 streams: List[Union[DeclarativeStream, StateDelegatingStream]] 2171 dynamic_streams: Optional[List[DynamicDeclarativeStream]] = None 2172 version: str = Field( 2173 ..., 2174 description="The version of the Airbyte CDK used to build and test the source.", 2175 ) 2176 schemas: Optional[Schemas] = None 2177 definitions: Optional[Dict[str, Any]] = None 2178 spec: Optional[Spec] = None 2179 concurrency_level: Optional[ConcurrencyLevel] = None 2180 api_budget: Optional[HTTPAPIBudget] = None 2181 max_concurrent_async_job_count: Optional[Union[int, str]] = Field( 2182 None, 2183 description="Maximum number of concurrent asynchronous jobs to run. This property is only relevant for sources/streams that support asynchronous job execution through the AsyncRetriever (e.g. a report-based stream that initiates a job, polls the job status, and then fetches the job results). This is often set by the API's maximum number of concurrent jobs on the account level. Refer to the API's documentation for this information.", 2184 examples=[3, "{{ config['max_concurrent_async_job_count'] }}"], 2185 title="Maximum Concurrent Asynchronous Jobs", 2186 ) 2187 metadata: Optional[Dict[str, Any]] = Field( 2188 None, 2189 description="For internal Airbyte use only - DO NOT modify manually. Used by consumers of declarative manifests for storing related metadata.", 2190 ) 2191 description: Optional[str] = Field( 2192 None, 2193 description="A description of the connector. It will be presented on the Source documentation page.", 2194 ) 2195 2196 2197class DeclarativeSource2(BaseModel): 2198 class Config: 2199 extra = Extra.forbid 2200 2201 type: Literal["DeclarativeSource"] 2202 check: Union[CheckStream, CheckDynamicStream] 2203 streams: Optional[List[Union[DeclarativeStream, StateDelegatingStream]]] = None 2204 dynamic_streams: List[DynamicDeclarativeStream] 2205 version: str = Field( 2206 ..., 2207 description="The version of the Airbyte CDK used to build and test the source.", 2208 ) 2209 schemas: Optional[Schemas] = None 2210 definitions: Optional[Dict[str, Any]] = None 2211 spec: Optional[Spec] = None 2212 concurrency_level: Optional[ConcurrencyLevel] = None 2213 api_budget: Optional[HTTPAPIBudget] = None 2214 max_concurrent_async_job_count: Optional[Union[int, str]] = Field( 2215 None, 2216 description="Maximum number of concurrent asynchronous jobs to run. This property is only relevant for sources/streams that support asynchronous job execution through the AsyncRetriever (e.g. a report-based stream that initiates a job, polls the job status, and then fetches the job results). This is often set by the API's maximum number of concurrent jobs on the account level. Refer to the API's documentation for this information.", 2217 examples=[3, "{{ config['max_concurrent_async_job_count'] }}"], 2218 title="Maximum Concurrent Asynchronous Jobs", 2219 ) 2220 metadata: Optional[Dict[str, Any]] = Field( 2221 None, 2222 description="For internal Airbyte use only - DO NOT modify manually. Used by consumers of declarative manifests for storing related metadata.", 2223 ) 2224 description: Optional[str] = Field( 2225 None, 2226 description="A description of the connector. It will be presented on the Source documentation page.", 2227 ) 2228 2229 2230class DeclarativeSource(BaseModel): 2231 class Config: 2232 extra = Extra.forbid 2233 2234 __root__: Union[DeclarativeSource1, DeclarativeSource2] = Field( 2235 ..., 2236 description="An API source that extracts data according to its declarative components.", 2237 title="DeclarativeSource", 2238 ) 2239 2240 2241class SelectiveAuthenticator(BaseModel): 2242 class Config: 2243 extra = Extra.allow 2244 2245 type: Literal["SelectiveAuthenticator"] 2246 authenticator_selection_path: List[str] = Field( 2247 ..., 2248 description="Path of the field in config with selected authenticator name", 2249 examples=[["auth"], ["auth", "type"]], 2250 title="Authenticator Selection Path", 2251 ) 2252 authenticators: Dict[ 2253 str, 2254 Union[ 2255 ApiKeyAuthenticator, 2256 BasicHttpAuthenticator, 2257 BearerAuthenticator, 2258 CustomAuthenticator, 2259 OAuthAuthenticator, 2260 JwtAuthenticator, 2261 SessionTokenAuthenticator, 2262 NoAuth, 2263 LegacySessionTokenAuthenticator, 2264 ], 2265 ] = Field( 2266 ..., 2267 description="Authenticators to select from.", 2268 examples=[ 2269 { 2270 "authenticators": { 2271 "token": "#/definitions/ApiKeyAuthenticator", 2272 "oauth": "#/definitions/OAuthAuthenticator", 2273 "jwt": "#/definitions/JwtAuthenticator", 2274 } 2275 } 2276 ], 2277 title="Authenticators", 2278 ) 2279 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters") 2280 2281 2282class FileUploader(BaseModel): 2283 type: Literal["FileUploader"] 2284 requester: Union[HttpRequester, CustomRequester] = Field( 2285 ..., 2286 description="Requester component that describes how to prepare HTTP requests to send to the source API.", 2287 ) 2288 download_target_extractor: Union[DpathExtractor, CustomRecordExtractor] = Field( 2289 ..., 2290 description="Responsible for fetching the url where the file is located. This is applied on each records and not on the HTTP response", 2291 ) 2292 file_extractor: Optional[Union[DpathExtractor, CustomRecordExtractor]] = Field( 2293 None, 2294 description="Responsible for fetching the content of the file. If not defined, the assumption is that the whole response body is the file content", 2295 ) 2296 filename_extractor: Optional[str] = Field( 2297 None, 2298 description="Defines the name to store the file. Stream name is automatically added to the file path. File unique ID can be used to avoid overwriting files. Random UUID will be used if the extractor is not provided.", 2299 examples=[ 2300 "{{ record.id }}/{{ record.file_name }}/", 2301 "{{ record.id }}_{{ record.file_name }}/", 2302 ], 2303 ) 2304 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters") 2305 2306 2307class DeclarativeStream(BaseModel): 2308 class Config: 2309 extra = Extra.allow 2310 2311 type: Literal["DeclarativeStream"] 2312 name: Optional[str] = Field("", description="The stream name.", example=["Users"], title="Name") 2313 retriever: Union[SimpleRetriever, AsyncRetriever, CustomRetriever] = Field( 2314 ..., 2315 description="Component used to coordinate how records are extracted across stream slices and request pages.", 2316 title="Retriever", 2317 ) 2318 incremental_sync: Optional[ 2319 Union[DatetimeBasedCursor, IncrementingCountCursor, CustomIncrementalSync] 2320 ] = Field( 2321 None, 2322 description="Component used to fetch data incrementally based on a time field in the data.", 2323 title="Incremental Sync", 2324 ) 2325 primary_key: Optional[PrimaryKey] = Field("", title="Primary Key") 2326 schema_loader: Optional[ 2327 Union[ 2328 InlineSchemaLoader, 2329 DynamicSchemaLoader, 2330 JsonFileSchemaLoader, 2331 CustomSchemaLoader, 2332 List[ 2333 Union[ 2334 InlineSchemaLoader, 2335 DynamicSchemaLoader, 2336 JsonFileSchemaLoader, 2337 CustomSchemaLoader, 2338 ] 2339 ], 2340 ] 2341 ] = Field( 2342 None, 2343 description="One or many schema loaders can be used to retrieve the schema for the current stream. When multiple schema loaders are defined, schema properties will be merged together. Schema loaders defined first taking precedence in the event of a conflict.", 2344 title="Schema Loader", 2345 ) 2346 transformations: Optional[ 2347 List[ 2348 Union[ 2349 AddFields, 2350 CustomTransformation, 2351 RemoveFields, 2352 KeysToLower, 2353 KeysToSnakeCase, 2354 FlattenFields, 2355 DpathFlattenFields, 2356 KeysReplace, 2357 ] 2358 ] 2359 ] = Field( 2360 None, 2361 description="A list of transformations to be applied to each output record.", 2362 title="Transformations", 2363 ) 2364 state_migrations: Optional[ 2365 List[Union[LegacyToPerPartitionStateMigration, CustomStateMigration]] 2366 ] = Field( 2367 [], 2368 description="Array of state migrations to be applied on the input state", 2369 title="State Migrations", 2370 ) 2371 file_uploader: Optional[FileUploader] = Field( 2372 None, 2373 description="(experimental) Describes how to fetch a file", 2374 title="File Uploader", 2375 ) 2376 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters") 2377 2378 2379class SessionTokenAuthenticator(BaseModel): 2380 type: Literal["SessionTokenAuthenticator"] 2381 login_requester: HttpRequester = Field( 2382 ..., 2383 description="Description of the request to perform to obtain a session token to perform data requests. The response body is expected to be a JSON object with a session token property.", 2384 examples=[ 2385 { 2386 "type": "HttpRequester", 2387 "url_base": "https://my_api.com", 2388 "path": "/login", 2389 "authenticator": { 2390 "type": "BasicHttpAuthenticator", 2391 "username": "{{ config.username }}", 2392 "password": "{{ config.password }}", 2393 }, 2394 } 2395 ], 2396 title="Login Requester", 2397 ) 2398 session_token_path: List[str] = Field( 2399 ..., 2400 description="The path in the response body returned from the login requester to the session token.", 2401 examples=[["access_token"], ["result", "token"]], 2402 title="Session Token Path", 2403 ) 2404 expiration_duration: Optional[str] = Field( 2405 None, 2406 description="The duration in ISO 8601 duration notation after which the session token expires, starting from the time it was obtained. Omitting it will result in the session token being refreshed for every request.", 2407 examples=["PT1H", "P1D"], 2408 title="Expiration Duration", 2409 ) 2410 request_authentication: Union[ 2411 SessionTokenRequestApiKeyAuthenticator, SessionTokenRequestBearerAuthenticator 2412 ] = Field( 2413 ..., 2414 description="Authentication method to use for requests sent to the API, specifying how to inject the session token.", 2415 title="Data Request Authentication", 2416 ) 2417 decoder: Optional[Union[JsonDecoder, XmlDecoder]] = Field( 2418 None, description="Component used to decode the response.", title="Decoder" 2419 ) 2420 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters") 2421 2422 2423class HttpRequester(BaseModelWithDeprecations): 2424 type: Literal["HttpRequester"] 2425 url_base: Optional[str] = Field( 2426 None, 2427 deprecated=True, 2428 deprecation_message="Use `url` field instead.", 2429 description="Deprecated, use the `url` instead. Base URL of the API source. Do not put sensitive information (e.g. API tokens) into this field - Use the Authenticator component for this.", 2430 examples=[ 2431 "https://connect.squareup.com/v2", 2432 "{{ config['base_url'] or 'https://app.posthog.com'}}/api", 2433 "https://connect.squareup.com/v2/quotes/{{ stream_partition['id'] }}/quote_line_groups", 2434 "https://example.com/api/v1/resource/{{ next_page_token['id'] }}", 2435 ], 2436 title="API Base URL", 2437 ) 2438 url: Optional[str] = Field( 2439 None, 2440 description="The URL of the source API endpoint. Do not put sensitive information (e.g. API tokens) into this field - Use the Authenticator component for this.", 2441 examples=[ 2442 "https://connect.squareup.com/v2", 2443 "{{ config['url'] or 'https://app.posthog.com'}}/api", 2444 "https://connect.squareup.com/v2/quotes/{{ stream_partition['id'] }}/quote_line_groups", 2445 "https://example.com/api/v1/resource/{{ next_page_token['id'] }}", 2446 ], 2447 title="API Endpoint URL", 2448 ) 2449 path: Optional[str] = Field( 2450 None, 2451 deprecated=True, 2452 deprecation_message="Use `url` field instead.", 2453 description="Deprecated, use the `url` instead. Path the specific API endpoint that this stream represents. Do not put sensitive information (e.g. API tokens) into this field - Use the Authenticator component for this.", 2454 examples=[ 2455 "/products", 2456 "/quotes/{{ stream_partition['id'] }}/quote_line_groups", 2457 "/trades/{{ config['symbol_id'] }}/history", 2458 ], 2459 title="URL Path", 2460 ) 2461 http_method: Optional[HttpMethod] = Field( 2462 HttpMethod.GET, 2463 description="The HTTP method used to fetch data from the source (can be GET or POST).", 2464 examples=["GET", "POST"], 2465 title="HTTP Method", 2466 ) 2467 authenticator: Optional[ 2468 Union[ 2469 ApiKeyAuthenticator, 2470 BasicHttpAuthenticator, 2471 BearerAuthenticator, 2472 OAuthAuthenticator, 2473 JwtAuthenticator, 2474 SessionTokenAuthenticator, 2475 SelectiveAuthenticator, 2476 CustomAuthenticator, 2477 NoAuth, 2478 LegacySessionTokenAuthenticator, 2479 ] 2480 ] = Field( 2481 None, 2482 description="Authentication method to use for requests sent to the API.", 2483 title="Authenticator", 2484 ) 2485 fetch_properties_from_endpoint: Optional[PropertiesFromEndpoint] = Field( 2486 None, 2487 description="Allows for retrieving a dynamic set of properties from an API endpoint which can be injected into outbound request using the stream_partition.extra_fields.", 2488 title="Fetch Properties from Endpoint", 2489 ) 2490 request_parameters: Optional[Union[Dict[str, Union[str, QueryProperties]], str]] = Field( 2491 None, 2492 description="Specifies the query parameters that should be set on an outgoing HTTP request given the inputs.", 2493 examples=[ 2494 {"unit": "day"}, 2495 { 2496 "query": 'last_event_time BETWEEN TIMESTAMP "{{ stream_interval.start_time }}" AND TIMESTAMP "{{ stream_interval.end_time }}"' 2497 }, 2498 {"searchIn": "{{ ','.join(config.get('search_in', [])) }}"}, 2499 {"sort_by[asc]": "updated_at"}, 2500 ], 2501 title="Query Parameters", 2502 ) 2503 request_headers: Optional[Union[Dict[str, str], str]] = Field( 2504 None, 2505 description="Return any non-auth headers. Authentication headers will overwrite any overlapping headers returned from this method.", 2506 examples=[{"Output-Format": "JSON"}, {"Version": "{{ config['version'] }}"}], 2507 title="Request Headers", 2508 ) 2509 request_body_data: Optional[Union[Dict[str, str], str]] = Field( 2510 None, 2511 deprecated=True, 2512 deprecation_message="Use `request_body` field instead.", 2513 description="Specifies how to populate the body of the request with a non-JSON payload. Plain text will be sent as is, whereas objects will be converted to a urlencoded form.", 2514 examples=[ 2515 '[{"clause": {"type": "timestamp", "operator": 10, "parameters":\n [{"value": {{ stream_interval[\'start_time\'] | int * 1000 }} }]\n }, "orderBy": 1, "columnName": "Timestamp"}]/\n' 2516 ], 2517 title="Request Body Payload (Non-JSON)", 2518 ) 2519 request_body_json: Optional[Union[Dict[str, Any], str]] = Field( 2520 None, 2521 deprecated=True, 2522 deprecation_message="Use `request_body` field instead.", 2523 description="Specifies how to populate the body of the request with a JSON payload. Can contain nested objects.", 2524 examples=[ 2525 {"sort_order": "ASC", "sort_field": "CREATED_AT"}, 2526 {"key": "{{ config['value'] }}"}, 2527 {"sort": {"field": "updated_at", "order": "ascending"}}, 2528 ], 2529 title="Request Body JSON Payload", 2530 ) 2531 request_body: Optional[ 2532 Union[ 2533 RequestBodyPlainText, 2534 RequestBodyUrlEncodedForm, 2535 RequestBodyJsonObject, 2536 RequestBodyGraphQL, 2537 ] 2538 ] = Field( 2539 None, 2540 description="Specifies how to populate the body of the request with a payload. Can contain nested objects.", 2541 examples=[ 2542 { 2543 "type": "RequestBodyJsonObject", 2544 "value": {"sort_order": "ASC", "sort_field": "CREATED_AT"}, 2545 }, 2546 { 2547 "type": "RequestBodyJsonObject", 2548 "value": {"key": "{{ config['value'] }}"}, 2549 }, 2550 { 2551 "type": "RequestBodyJsonObject", 2552 "value": {"sort": {"field": "updated_at", "order": "ascending"}}, 2553 }, 2554 {"type": "RequestBodyPlainText", "value": "plain_text_body"}, 2555 { 2556 "type": "RequestBodyUrlEncodedForm", 2557 "value": {"param1": "value1", "param2": "{{ config['param2_value'] }}"}, 2558 }, 2559 { 2560 "type": "RequestBodyGraphQL", 2561 "value": { 2562 "query": { 2563 "param1": "value1", 2564 "param2": "{{ config['param2_value'] }}", 2565 } 2566 }, 2567 }, 2568 ], 2569 title="Request Body", 2570 ) 2571 error_handler: Optional[ 2572 Union[DefaultErrorHandler, CompositeErrorHandler, CustomErrorHandler] 2573 ] = Field( 2574 None, 2575 description="Error handler component that defines how to handle errors.", 2576 title="Error Handler", 2577 ) 2578 use_cache: Optional[bool] = Field( 2579 False, 2580 description="Enables stream requests caching. This field is automatically set by the CDK.", 2581 title="Use Cache", 2582 ) 2583 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters") 2584 2585 2586class DynamicSchemaLoader(BaseModel): 2587 type: Literal["DynamicSchemaLoader"] 2588 retriever: Union[AsyncRetriever, CustomRetriever, SimpleRetriever] = Field( 2589 ..., 2590 description="Component used to coordinate how records are extracted across stream slices and request pages.", 2591 title="Retriever", 2592 ) 2593 schema_filter: Optional[Union[RecordFilter, CustomRecordFilter]] = Field( 2594 None, 2595 description="Responsible for filtering fields to be added to json schema.", 2596 title="Schema Filter", 2597 ) 2598 schema_transformations: Optional[ 2599 List[ 2600 Union[ 2601 AddFields, 2602 CustomTransformation, 2603 RemoveFields, 2604 KeysToLower, 2605 KeysToSnakeCase, 2606 FlattenFields, 2607 DpathFlattenFields, 2608 KeysReplace, 2609 ] 2610 ] 2611 ] = Field( 2612 None, 2613 description="A list of transformations to be applied to the schema.", 2614 title="Schema Transformations", 2615 ) 2616 schema_type_identifier: SchemaTypeIdentifier 2617 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters") 2618 2619 2620class ParentStreamConfig(BaseModel): 2621 type: Literal["ParentStreamConfig"] 2622 lazy_read_pointer: Optional[List[str]] = Field( 2623 [], 2624 description="If set, this will enable lazy reading, using the initial read of parent records to extract child records.", 2625 title="Lazy Read Pointer", 2626 ) 2627 parent_key: str = Field( 2628 ..., 2629 description="The primary key of records from the parent stream that will be used during the retrieval of records for the current substream. This parent identifier field is typically a characteristic of the child records being extracted from the source API.", 2630 examples=["id", "{{ config['parent_record_id'] }}"], 2631 title="Parent Key", 2632 ) 2633 stream: Union[DeclarativeStream, StateDelegatingStream] = Field( 2634 ..., description="Reference to the parent stream.", title="Parent Stream" 2635 ) 2636 partition_field: str = Field( 2637 ..., 2638 description="While iterating over parent records during a sync, the parent_key value can be referenced by using this field.", 2639 examples=["parent_id", "{{ config['parent_partition_field'] }}"], 2640 title="Current Parent Key Value Identifier", 2641 ) 2642 request_option: Optional[RequestOption] = Field( 2643 None, 2644 description="A request option describing where the parent key value should be injected into and under what field name if applicable.", 2645 title="Request Option", 2646 ) 2647 incremental_dependency: Optional[bool] = Field( 2648 False, 2649 description="Indicates whether the parent stream should be read incrementally based on updates in the child stream.", 2650 title="Incremental Dependency", 2651 ) 2652 extra_fields: Optional[List[List[str]]] = Field( 2653 None, 2654 description="Array of field paths to include as additional fields in the stream slice. Each path is an array of strings representing keys to access fields in the respective parent record. Accessible via `stream_slice.extra_fields`. Missing fields are set to `None`.", 2655 title="Extra Fields", 2656 ) 2657 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters") 2658 2659 2660class PropertiesFromEndpoint(BaseModel): 2661 type: Literal["PropertiesFromEndpoint"] 2662 property_field_path: List[str] = Field( 2663 ..., 2664 description="Describes the path to the field that should be extracted", 2665 examples=[["name"]], 2666 ) 2667 retriever: Union[SimpleRetriever, CustomRetriever] = Field( 2668 ..., 2669 description="Requester component that describes how to fetch the properties to query from a remote API endpoint.", 2670 ) 2671 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters") 2672 2673 2674class QueryProperties(BaseModel): 2675 type: Literal["QueryProperties"] 2676 property_list: Union[List[str], PropertiesFromEndpoint] = Field( 2677 ..., 2678 description="The set of properties that will be queried for in the outbound request. This can either be statically defined or dynamic based on an API endpoint", 2679 title="Property List", 2680 ) 2681 always_include_properties: Optional[List[str]] = Field( 2682 None, 2683 description="The list of properties that should be included in every set of properties when multiple chunks of properties are being requested.", 2684 title="Always Include Properties", 2685 ) 2686 property_chunking: Optional[PropertyChunking] = Field( 2687 None, 2688 description="Defines how query properties will be grouped into smaller sets for APIs with limitations on the number of properties fetched per API request.", 2689 title="Property Chunking", 2690 ) 2691 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters") 2692 2693 2694class StateDelegatingStream(BaseModel): 2695 type: Literal["StateDelegatingStream"] 2696 name: str = Field(..., description="The stream name.", example=["Users"], title="Name") 2697 full_refresh_stream: DeclarativeStream = Field( 2698 ..., 2699 description="Component used to coordinate how records are extracted across stream slices and request pages when the state is empty or not provided.", 2700 title="Full Refresh Stream", 2701 ) 2702 incremental_stream: DeclarativeStream = Field( 2703 ..., 2704 description="Component used to coordinate how records are extracted across stream slices and request pages when the state provided.", 2705 title="Incremental Stream", 2706 ) 2707 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters") 2708 2709 2710class SimpleRetriever(BaseModel): 2711 type: Literal["SimpleRetriever"] 2712 requester: Union[HttpRequester, CustomRequester] = Field( 2713 ..., 2714 description="Requester component that describes how to prepare HTTP requests to send to the source API.", 2715 ) 2716 decoder: Optional[ 2717 Union[ 2718 JsonDecoder, 2719 XmlDecoder, 2720 CsvDecoder, 2721 JsonlDecoder, 2722 GzipDecoder, 2723 IterableDecoder, 2724 ZipfileDecoder, 2725 CustomDecoder, 2726 ] 2727 ] = Field( 2728 None, 2729 description="Component decoding the response so records can be extracted.", 2730 title="HTTP Response Format", 2731 ) 2732 record_selector: RecordSelector = Field( 2733 ..., 2734 description="Component that describes how to extract records from a HTTP response.", 2735 ) 2736 paginator: Optional[Union[DefaultPaginator, NoPagination]] = Field( 2737 None, 2738 description="Paginator component that describes how to navigate through the API's pages.", 2739 ) 2740 ignore_stream_slicer_parameters_on_paginated_requests: Optional[bool] = Field( 2741 False, 2742 description="If true, the partition router and incremental request options will be ignored when paginating requests. Request options set directly on the requester will not be ignored.", 2743 ) 2744 partition_router: Optional[ 2745 Union[ 2746 ListPartitionRouter, 2747 SubstreamPartitionRouter, 2748 GroupingPartitionRouter, 2749 CustomPartitionRouter, 2750 List[ 2751 Union[ 2752 ListPartitionRouter, 2753 SubstreamPartitionRouter, 2754 GroupingPartitionRouter, 2755 CustomPartitionRouter, 2756 ] 2757 ], 2758 ] 2759 ] = Field( 2760 [], 2761 description="PartitionRouter component that describes how to partition the stream, enabling incremental syncs and checkpointing.", 2762 title="Partition Router", 2763 ) 2764 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters") 2765 2766 2767class AsyncRetriever(BaseModel): 2768 type: Literal["AsyncRetriever"] 2769 record_selector: RecordSelector = Field( 2770 ..., 2771 description="Component that describes how to extract records from a HTTP response.", 2772 ) 2773 status_mapping: AsyncJobStatusMap = Field( 2774 ..., description="Async Job Status to Airbyte CDK Async Job Status mapping." 2775 ) 2776 status_extractor: Union[DpathExtractor, CustomRecordExtractor] = Field( 2777 ..., description="Responsible for fetching the actual status of the async job." 2778 ) 2779 download_target_extractor: Union[DpathExtractor, CustomRecordExtractor] = Field( 2780 ..., 2781 description="Responsible for fetching the final result `urls` provided by the completed / finished / ready async job.", 2782 ) 2783 download_extractor: Optional[ 2784 Union[DpathExtractor, CustomRecordExtractor, ResponseToFileExtractor] 2785 ] = Field(None, description="Responsible for fetching the records from provided urls.") 2786 creation_requester: Union[HttpRequester, CustomRequester] = Field( 2787 ..., 2788 description="Requester component that describes how to prepare HTTP requests to send to the source API to create the async server-side job.", 2789 ) 2790 polling_requester: Union[HttpRequester, CustomRequester] = Field( 2791 ..., 2792 description="Requester component that describes how to prepare HTTP requests to send to the source API to fetch the status of the running async job.", 2793 ) 2794 polling_job_timeout: Optional[Union[int, str]] = Field( 2795 None, 2796 description="The time in minutes after which the single Async Job should be considered as Timed Out.", 2797 ) 2798 download_target_requester: Optional[Union[HttpRequester, CustomRequester]] = Field( 2799 None, 2800 description="Requester component that describes how to prepare HTTP requests to send to the source API to extract the url from polling response by the completed async job.", 2801 ) 2802 download_requester: Union[HttpRequester, CustomRequester] = Field( 2803 ..., 2804 description="Requester component that describes how to prepare HTTP requests to send to the source API to download the data provided by the completed async job.", 2805 ) 2806 download_paginator: Optional[Union[DefaultPaginator, NoPagination]] = Field( 2807 None, 2808 description="Paginator component that describes how to navigate through the API's pages during download.", 2809 ) 2810 abort_requester: Optional[Union[HttpRequester, CustomRequester]] = Field( 2811 None, 2812 description="Requester component that describes how to prepare HTTP requests to send to the source API to abort a job once it is timed out from the source's perspective.", 2813 ) 2814 delete_requester: Optional[Union[HttpRequester, CustomRequester]] = Field( 2815 None, 2816 description="Requester component that describes how to prepare HTTP requests to send to the source API to delete a job once the records are extracted.", 2817 ) 2818 partition_router: Optional[ 2819 Union[ 2820 ListPartitionRouter, 2821 SubstreamPartitionRouter, 2822 GroupingPartitionRouter, 2823 CustomPartitionRouter, 2824 List[ 2825 Union[ 2826 ListPartitionRouter, 2827 SubstreamPartitionRouter, 2828 GroupingPartitionRouter, 2829 CustomPartitionRouter, 2830 ] 2831 ], 2832 ] 2833 ] = Field( 2834 [], 2835 description="PartitionRouter component that describes how to partition the stream, enabling incremental syncs and checkpointing.", 2836 title="Partition Router", 2837 ) 2838 decoder: Optional[ 2839 Union[ 2840 CsvDecoder, 2841 GzipDecoder, 2842 JsonDecoder, 2843 JsonlDecoder, 2844 IterableDecoder, 2845 XmlDecoder, 2846 ZipfileDecoder, 2847 CustomDecoder, 2848 ] 2849 ] = Field( 2850 None, 2851 description="Component decoding the response so records can be extracted.", 2852 title="Decoder", 2853 ) 2854 download_decoder: Optional[ 2855 Union[ 2856 CsvDecoder, 2857 GzipDecoder, 2858 JsonDecoder, 2859 JsonlDecoder, 2860 IterableDecoder, 2861 XmlDecoder, 2862 ZipfileDecoder, 2863 CustomDecoder, 2864 ] 2865 ] = Field( 2866 None, 2867 description="Component decoding the download response so records can be extracted.", 2868 title="Download Decoder", 2869 ) 2870 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters") 2871 2872 2873class SubstreamPartitionRouter(BaseModel): 2874 type: Literal["SubstreamPartitionRouter"] 2875 parent_stream_configs: List[ParentStreamConfig] = Field( 2876 ..., 2877 description="Specifies which parent streams are being iterated over and how parent records should be used to partition the child stream data set.", 2878 title="Parent Stream Configs", 2879 ) 2880 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters") 2881 2882 2883class GroupingPartitionRouter(BaseModel): 2884 type: Literal["GroupingPartitionRouter"] 2885 group_size: int = Field( 2886 ..., 2887 description="The number of partitions to include in each group. This determines how many partition values are batched together in a single slice.", 2888 examples=[10, 50], 2889 title="Group Size", 2890 ) 2891 underlying_partition_router: Union[ 2892 CustomPartitionRouter, ListPartitionRouter, SubstreamPartitionRouter 2893 ] = Field( 2894 ..., 2895 description="The partition router whose output will be grouped. This can be any valid partition router component.", 2896 title="Underlying Partition Router", 2897 ) 2898 deduplicate: Optional[bool] = Field( 2899 True, 2900 description="If true, ensures that partitions are unique within each group by removing duplicates based on the partition key.", 2901 title="Deduplicate Partitions", 2902 ) 2903 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters") 2904 2905 2906class HttpComponentsResolver(BaseModel): 2907 type: Literal["HttpComponentsResolver"] 2908 retriever: Union[AsyncRetriever, CustomRetriever, SimpleRetriever] = Field( 2909 ..., 2910 description="Component used to coordinate how records are extracted across stream slices and request pages.", 2911 title="Retriever", 2912 ) 2913 components_mapping: List[ComponentMappingDefinition] 2914 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters") 2915 2916 2917class DynamicDeclarativeStream(BaseModel): 2918 type: Literal["DynamicDeclarativeStream"] 2919 name: Optional[str] = Field( 2920 "", description="The dynamic stream name.", example=["Tables"], title="Name" 2921 ) 2922 stream_template: Union[DeclarativeStream, StateDelegatingStream] = Field( 2923 ..., description="Reference to the stream template.", title="Stream Template" 2924 ) 2925 components_resolver: Union[HttpComponentsResolver, ConfigComponentsResolver] = Field( 2926 ..., 2927 description="Component resolve and populates stream templates with components values.", 2928 title="Components Resolver", 2929 ) 2930 2931 2932ComplexFieldType.update_forward_refs() 2933GzipDecoder.update_forward_refs() 2934CompositeErrorHandler.update_forward_refs() 2935DeclarativeSource1.update_forward_refs() 2936DeclarativeSource2.update_forward_refs() 2937SelectiveAuthenticator.update_forward_refs() 2938FileUploader.update_forward_refs() 2939DeclarativeStream.update_forward_refs() 2940SessionTokenAuthenticator.update_forward_refs() 2941HttpRequester.update_forward_refs() 2942DynamicSchemaLoader.update_forward_refs() 2943ParentStreamConfig.update_forward_refs() 2944PropertiesFromEndpoint.update_forward_refs() 2945SimpleRetriever.update_forward_refs() 2946AsyncRetriever.update_forward_refs()
class
AuthFlowType(enum.Enum):
An enumeration.
oauth2_0 =
<AuthFlowType.oauth2_0: 'oauth2.0'>
oauth1_0 =
<AuthFlowType.oauth1_0: 'oauth1.0'>
class
BasicHttpAuthenticator(pydantic.v1.main.BaseModel):
22class BasicHttpAuthenticator(BaseModel): 23 type: Literal["BasicHttpAuthenticator"] 24 username: str = Field( 25 ..., 26 description="The username that will be combined with the password, base64 encoded and used to make requests. Fill it in the user inputs.", 27 examples=["{{ config['username'] }}", "{{ config['api_key'] }}"], 28 title="Username", 29 ) 30 password: Optional[str] = Field( 31 "", 32 description="The password that will be combined with the username, base64 encoded and used to make requests. Fill it in the user inputs.", 33 examples=["{{ config['password'] }}", ""], 34 title="Password", 35 ) 36 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters")
class
BearerAuthenticator(pydantic.v1.main.BaseModel):
39class BearerAuthenticator(BaseModel): 40 type: Literal["BearerAuthenticator"] 41 api_token: str = Field( 42 ..., 43 description="Token to inject as request header for authenticating with the API.", 44 examples=["{{ config['api_key'] }}", "{{ config['token'] }}"], 45 title="Bearer Token", 46 ) 47 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters")
class
DynamicStreamCheckConfig(pydantic.v1.main.BaseModel):
50class DynamicStreamCheckConfig(BaseModel): 51 type: Literal["DynamicStreamCheckConfig"] 52 dynamic_stream_name: str = Field( 53 ..., description="The dynamic stream name.", title="Dynamic Stream Name" 54 ) 55 stream_count: Optional[int] = Field( 56 0, 57 description="The number of streams to attempt reading from during a check operation. If `stream_count` exceeds the total number of available streams, the minimum of the two values will be used.", 58 title="Stream Count", 59 )
class
CheckDynamicStream(pydantic.v1.main.BaseModel):
62class CheckDynamicStream(BaseModel): 63 type: Literal["CheckDynamicStream"] 64 stream_count: int = Field( 65 ..., 66 description="Numbers of the streams to try reading from when running a check operation.", 67 title="Stream Count", 68 ) 69 use_check_availability: Optional[bool] = Field( 70 True, 71 description="Enables stream check availability. This field is automatically set by the CDK.", 72 title="Use Check Availability", 73 )
class
ConcurrencyLevel(pydantic.v1.main.BaseModel):
76class ConcurrencyLevel(BaseModel): 77 type: Optional[Literal["ConcurrencyLevel"]] = None 78 default_concurrency: Union[int, str] = Field( 79 ..., 80 description="The amount of concurrency that will applied during a sync. This value can be hardcoded or user-defined in the config if different users have varying volume thresholds in the target API.", 81 examples=[10, "{{ config['num_workers'] or 10 }}"], 82 title="Default Concurrency", 83 ) 84 max_concurrency: Optional[int] = Field( 85 None, 86 description="The maximum level of concurrency that will be used during a sync. This becomes a required field when the default_concurrency derives from the config, because it serves as a safeguard against a user-defined threshold that is too high.", 87 examples=[20, 100], 88 title="Max Concurrency", 89 ) 90 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters")
class
ConstantBackoffStrategy(pydantic.v1.main.BaseModel):
93class ConstantBackoffStrategy(BaseModel): 94 type: Literal["ConstantBackoffStrategy"] 95 backoff_time_in_seconds: Union[float, str] = Field( 96 ..., 97 description="Backoff time in seconds.", 98 examples=[30, 30.5, "{{ config['backoff_time'] }}"], 99 title="Backoff Time", 100 ) 101 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters")
class
CursorPagination(pydantic.v1.main.BaseModel):
104class CursorPagination(BaseModel): 105 type: Literal["CursorPagination"] 106 cursor_value: str = Field( 107 ..., 108 description="Value of the cursor defining the next page to fetch.", 109 examples=[ 110 "{{ headers.link.next.cursor }}", 111 "{{ last_record['key'] }}", 112 "{{ response['nextPage'] }}", 113 ], 114 title="Cursor Value", 115 ) 116 page_size: Optional[int] = Field( 117 None, 118 description="The number of records to include in each pages.", 119 examples=[100], 120 title="Page Size", 121 ) 122 stop_condition: Optional[str] = Field( 123 None, 124 description="Template string evaluating when to stop paginating.", 125 examples=[ 126 "{{ response.data.has_more is false }}", 127 "{{ 'next' not in headers['link'] }}", 128 ], 129 title="Stop Condition", 130 ) 131 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters")
class
CustomAuthenticator(pydantic.v1.main.BaseModel):
134class CustomAuthenticator(BaseModel): 135 class Config: 136 extra = Extra.allow 137 138 type: Literal["CustomAuthenticator"] 139 class_name: str = Field( 140 ..., 141 description="Fully-qualified name of the class that will be implementing the custom authentication strategy. Has to be a sub class of DeclarativeAuthenticator. The format is `source_<name>.<package>.<class_name>`.", 142 examples=["source_railz.components.ShortLivedTokenAuthenticator"], 143 title="Class Name", 144 ) 145 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters")
class
CustomAuthenticator.Config:
class
CustomBackoffStrategy(pydantic.v1.main.BaseModel):
148class CustomBackoffStrategy(BaseModel): 149 class Config: 150 extra = Extra.allow 151 152 type: Literal["CustomBackoffStrategy"] 153 class_name: str = Field( 154 ..., 155 description="Fully-qualified name of the class that will be implementing the custom backoff strategy. The format is `source_<name>.<package>.<class_name>`.", 156 examples=["source_railz.components.MyCustomBackoffStrategy"], 157 title="Class Name", 158 ) 159 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters")
class
CustomBackoffStrategy.Config:
class
CustomErrorHandler(pydantic.v1.main.BaseModel):
162class CustomErrorHandler(BaseModel): 163 class Config: 164 extra = Extra.allow 165 166 type: Literal["CustomErrorHandler"] 167 class_name: str = Field( 168 ..., 169 description="Fully-qualified name of the class that will be implementing the custom error handler. The format is `source_<name>.<package>.<class_name>`.", 170 examples=["source_railz.components.MyCustomErrorHandler"], 171 title="Class Name", 172 ) 173 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters")
class
CustomErrorHandler.Config:
class
CustomIncrementalSync(pydantic.v1.main.BaseModel):
176class CustomIncrementalSync(BaseModel): 177 class Config: 178 extra = Extra.allow 179 180 type: Literal["CustomIncrementalSync"] 181 class_name: str = Field( 182 ..., 183 description="Fully-qualified name of the class that will be implementing the custom incremental sync. The format is `source_<name>.<package>.<class_name>`.", 184 examples=["source_railz.components.MyCustomIncrementalSync"], 185 title="Class Name", 186 ) 187 cursor_field: str = Field( 188 ..., 189 description="The location of the value on a record that will be used as a bookmark during sync.", 190 ) 191 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters")
class
CustomIncrementalSync.Config:
class
CustomPaginationStrategy(pydantic.v1.main.BaseModel):
194class CustomPaginationStrategy(BaseModel): 195 class Config: 196 extra = Extra.allow 197 198 type: Literal["CustomPaginationStrategy"] 199 class_name: str = Field( 200 ..., 201 description="Fully-qualified name of the class that will be implementing the custom pagination strategy. The format is `source_<name>.<package>.<class_name>`.", 202 examples=["source_railz.components.MyCustomPaginationStrategy"], 203 title="Class Name", 204 ) 205 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters")
class
CustomPaginationStrategy.Config:
class
CustomRecordExtractor(pydantic.v1.main.BaseModel):
208class CustomRecordExtractor(BaseModel): 209 class Config: 210 extra = Extra.allow 211 212 type: Literal["CustomRecordExtractor"] 213 class_name: str = Field( 214 ..., 215 description="Fully-qualified name of the class that will be implementing the custom record extraction strategy. The format is `source_<name>.<package>.<class_name>`.", 216 examples=["source_railz.components.MyCustomRecordExtractor"], 217 title="Class Name", 218 ) 219 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters")
class
CustomRecordExtractor.Config:
class
CustomRecordFilter(pydantic.v1.main.BaseModel):
222class CustomRecordFilter(BaseModel): 223 class Config: 224 extra = Extra.allow 225 226 type: Literal["CustomRecordFilter"] 227 class_name: str = Field( 228 ..., 229 description="Fully-qualified name of the class that will be implementing the custom record filter strategy. The format is `source_<name>.<package>.<class_name>`.", 230 examples=["source_railz.components.MyCustomCustomRecordFilter"], 231 title="Class Name", 232 ) 233 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters")
class
CustomRecordFilter.Config:
class
CustomRequester(pydantic.v1.main.BaseModel):
236class CustomRequester(BaseModel): 237 class Config: 238 extra = Extra.allow 239 240 type: Literal["CustomRequester"] 241 class_name: str = Field( 242 ..., 243 description="Fully-qualified name of the class that will be implementing the custom requester strategy. The format is `source_<name>.<package>.<class_name>`.", 244 examples=["source_railz.components.MyCustomRecordExtractor"], 245 title="Class Name", 246 ) 247 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters")
class
CustomRequester.Config:
class
CustomRetriever(pydantic.v1.main.BaseModel):
250class CustomRetriever(BaseModel): 251 class Config: 252 extra = Extra.allow 253 254 type: Literal["CustomRetriever"] 255 class_name: str = Field( 256 ..., 257 description="Fully-qualified name of the class that will be implementing the custom retriever strategy. The format is `source_<name>.<package>.<class_name>`.", 258 examples=["source_railz.components.MyCustomRetriever"], 259 title="Class Name", 260 ) 261 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters")
class
CustomRetriever.Config:
class
CustomPartitionRouter(pydantic.v1.main.BaseModel):
264class CustomPartitionRouter(BaseModel): 265 class Config: 266 extra = Extra.allow 267 268 type: Literal["CustomPartitionRouter"] 269 class_name: str = Field( 270 ..., 271 description="Fully-qualified name of the class that will be implementing the custom partition router. The format is `source_<name>.<package>.<class_name>`.", 272 examples=["source_railz.components.MyCustomPartitionRouter"], 273 title="Class Name", 274 ) 275 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters")
class
CustomPartitionRouter.Config:
class
CustomSchemaLoader(pydantic.v1.main.BaseModel):
278class CustomSchemaLoader(BaseModel): 279 class Config: 280 extra = Extra.allow 281 282 type: Literal["CustomSchemaLoader"] 283 class_name: str = Field( 284 ..., 285 description="Fully-qualified name of the class that will be implementing the custom schema loader. The format is `source_<name>.<package>.<class_name>`.", 286 examples=["source_railz.components.MyCustomSchemaLoader"], 287 title="Class Name", 288 ) 289 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters")
class
CustomSchemaLoader.Config:
class
CustomSchemaNormalization(pydantic.v1.main.BaseModel):
292class CustomSchemaNormalization(BaseModel): 293 class Config: 294 extra = Extra.allow 295 296 type: Literal["CustomSchemaNormalization"] 297 class_name: str = Field( 298 ..., 299 description="Fully-qualified name of the class that will be implementing the custom normalization. The format is `source_<name>.<package>.<class_name>`.", 300 examples=[ 301 "source_amazon_seller_partner.components.LedgerDetailedViewReportsTypeTransformer" 302 ], 303 title="Class Name", 304 ) 305 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters")
class
CustomSchemaNormalization.Config:
class
CustomStateMigration(pydantic.v1.main.BaseModel):
308class CustomStateMigration(BaseModel): 309 class Config: 310 extra = Extra.allow 311 312 type: Literal["CustomStateMigration"] 313 class_name: str = Field( 314 ..., 315 description="Fully-qualified name of the class that will be implementing the custom state migration. The format is `source_<name>.<package>.<class_name>`.", 316 examples=["source_railz.components.MyCustomStateMigration"], 317 title="Class Name", 318 ) 319 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters")
class
CustomStateMigration.Config:
class
CustomTransformation(pydantic.v1.main.BaseModel):
322class CustomTransformation(BaseModel): 323 class Config: 324 extra = Extra.allow 325 326 type: Literal["CustomTransformation"] 327 class_name: str = Field( 328 ..., 329 description="Fully-qualified name of the class that will be implementing the custom transformation. The format is `source_<name>.<package>.<class_name>`.", 330 examples=["source_railz.components.MyCustomTransformation"], 331 title="Class Name", 332 ) 333 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters")
class
CustomTransformation.Config:
class
LegacyToPerPartitionStateMigration(pydantic.v1.main.BaseModel):
class
LegacyToPerPartitionStateMigration.Config:
class
Clamping(pydantic.v1.main.BaseModel):
class
Algorithm(enum.Enum):
353class Algorithm(Enum): 354 HS256 = "HS256" 355 HS384 = "HS384" 356 HS512 = "HS512" 357 ES256 = "ES256" 358 ES256K = "ES256K" 359 ES384 = "ES384" 360 ES512 = "ES512" 361 RS256 = "RS256" 362 RS384 = "RS384" 363 RS512 = "RS512" 364 PS256 = "PS256" 365 PS384 = "PS384" 366 PS512 = "PS512" 367 EdDSA = "EdDSA"
An enumeration.
HS256 =
<Algorithm.HS256: 'HS256'>
HS384 =
<Algorithm.HS384: 'HS384'>
HS512 =
<Algorithm.HS512: 'HS512'>
ES256 =
<Algorithm.ES256: 'ES256'>
ES256K =
<Algorithm.ES256K: 'ES256K'>
ES384 =
<Algorithm.ES384: 'ES384'>
ES512 =
<Algorithm.ES512: 'ES512'>
RS256 =
<Algorithm.RS256: 'RS256'>
RS384 =
<Algorithm.RS384: 'RS384'>
RS512 =
<Algorithm.RS512: 'RS512'>
PS256 =
<Algorithm.PS256: 'PS256'>
PS384 =
<Algorithm.PS384: 'PS384'>
PS512 =
<Algorithm.PS512: 'PS512'>
EdDSA =
<Algorithm.EdDSA: 'EdDSA'>
class
JwtHeaders(pydantic.v1.main.BaseModel):
370class JwtHeaders(BaseModel): 371 class Config: 372 extra = Extra.forbid 373 374 kid: Optional[str] = Field( 375 None, 376 description="Private key ID for user account.", 377 examples=["{{ config['kid'] }}"], 378 title="Key Identifier", 379 ) 380 typ: Optional[str] = Field( 381 "JWT", 382 description="The media type of the complete JWT.", 383 examples=["JWT"], 384 title="Type", 385 ) 386 cty: Optional[str] = Field( 387 None, 388 description="Content type of JWT header.", 389 examples=["JWT"], 390 title="Content Type", 391 )
class
JwtHeaders.Config:
class
JwtPayload(pydantic.v1.main.BaseModel):
394class JwtPayload(BaseModel): 395 class Config: 396 extra = Extra.forbid 397 398 iss: Optional[str] = Field( 399 None, 400 description="The user/principal that issued the JWT. Commonly a value unique to the user.", 401 examples=["{{ config['iss'] }}"], 402 title="Issuer", 403 ) 404 sub: Optional[str] = Field( 405 None, 406 description="The subject of the JWT. Commonly defined by the API.", 407 title="Subject", 408 ) 409 aud: Optional[str] = Field( 410 None, 411 description="The recipient that the JWT is intended for. Commonly defined by the API.", 412 examples=["appstoreconnect-v1"], 413 title="Audience", 414 )
class
JwtPayload.Config:
class
JwtAuthenticator(pydantic.v1.main.BaseModel):
417class JwtAuthenticator(BaseModel): 418 type: Literal["JwtAuthenticator"] 419 secret_key: str = Field( 420 ..., 421 description="Secret used to sign the JSON web token.", 422 examples=["{{ config['secret_key'] }}"], 423 title="Secret Key", 424 ) 425 base64_encode_secret_key: Optional[bool] = Field( 426 False, 427 description='When set to true, the secret key will be base64 encoded prior to being encoded as part of the JWT. Only set to "true" when required by the API.', 428 title="Base64-encode Secret Key", 429 ) 430 algorithm: Algorithm = Field( 431 ..., 432 description="Algorithm used to sign the JSON web token.", 433 examples=["ES256", "HS256", "RS256", "{{ config['algorithm'] }}"], 434 title="Algorithm", 435 ) 436 token_duration: Optional[int] = Field( 437 1200, 438 description="The amount of time in seconds a JWT token can be valid after being issued.", 439 examples=[1200, 3600], 440 title="Token Duration", 441 ) 442 header_prefix: Optional[str] = Field( 443 None, 444 description="The prefix to be used within the Authentication header.", 445 examples=["Bearer", "Basic"], 446 title="Header Prefix", 447 ) 448 jwt_headers: Optional[JwtHeaders] = Field( 449 None, 450 description="JWT headers used when signing JSON web token.", 451 title="JWT Headers", 452 ) 453 additional_jwt_headers: Optional[Dict[str, Any]] = Field( 454 None, 455 description="Additional headers to be included with the JWT headers object.", 456 title="Additional JWT Headers", 457 ) 458 jwt_payload: Optional[JwtPayload] = Field( 459 None, 460 description="JWT Payload used when signing JSON web token.", 461 title="JWT Payload", 462 ) 463 additional_jwt_payload: Optional[Dict[str, Any]] = Field( 464 None, 465 description="Additional properties to be added to the JWT payload.", 466 title="Additional JWT Payload Properties", 467 ) 468 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters")
algorithm: Algorithm
jwt_headers: Optional[JwtHeaders]
jwt_payload: Optional[JwtPayload]
class
RefreshTokenUpdater(pydantic.v1.main.BaseModel):
471class RefreshTokenUpdater(BaseModel): 472 refresh_token_name: Optional[str] = Field( 473 "refresh_token", 474 description="The name of the property which contains the updated refresh token in the response from the token refresh endpoint.", 475 examples=["refresh_token"], 476 title="Refresh Token Property Name", 477 ) 478 access_token_config_path: Optional[List[str]] = Field( 479 ["credentials", "access_token"], 480 description="Config path to the access token. Make sure the field actually exists in the config.", 481 examples=[["credentials", "access_token"], ["access_token"]], 482 title="Config Path To Access Token", 483 ) 484 refresh_token_config_path: Optional[List[str]] = Field( 485 ["credentials", "refresh_token"], 486 description="Config path to the access token. Make sure the field actually exists in the config.", 487 examples=[["credentials", "refresh_token"], ["refresh_token"]], 488 title="Config Path To Refresh Token", 489 ) 490 token_expiry_date_config_path: Optional[List[str]] = Field( 491 ["credentials", "token_expiry_date"], 492 description="Config path to the expiry date. Make sure actually exists in the config.", 493 examples=[["credentials", "token_expiry_date"]], 494 title="Config Path To Expiry Date", 495 ) 496 refresh_token_error_status_codes: Optional[List[int]] = Field( 497 [], 498 description="Status Codes to Identify refresh token error in response (Refresh Token Error Key and Refresh Token Error Values should be also specified). Responses with one of the error status code and containing an error value will be flagged as a config error", 499 examples=[[400, 500]], 500 title="Refresh Token Error Status Codes", 501 ) 502 refresh_token_error_key: Optional[str] = Field( 503 "", 504 description="Key to Identify refresh token error in response (Refresh Token Error Status Codes and Refresh Token Error Values should be also specified).", 505 examples=["error"], 506 title="Refresh Token Error Key", 507 ) 508 refresh_token_error_values: Optional[List[str]] = Field( 509 [], 510 description='List of values to check for exception during token refresh process. Used to check if the error found in the response matches the key from the Refresh Token Error Key field (e.g. response={"error": "invalid_grant"}). Only responses with one of the error status code and containing an error value will be flagged as a config error', 511 examples=[["invalid_grant", "invalid_permissions"]], 512 title="Refresh Token Error Values", 513 )
class
OAuthAuthenticator(pydantic.v1.main.BaseModel):
516class OAuthAuthenticator(BaseModel): 517 type: Literal["OAuthAuthenticator"] 518 client_id_name: Optional[str] = Field( 519 "client_id", 520 description="The name of the property to use to refresh the `access_token`.", 521 examples=["custom_app_id"], 522 title="Client ID Property Name", 523 ) 524 client_id: Optional[str] = Field( 525 None, 526 description="The OAuth client ID. Fill it in the user inputs.", 527 examples=["{{ config['client_id }}", "{{ config['credentials']['client_id }}"], 528 title="Client ID", 529 ) 530 client_secret_name: Optional[str] = Field( 531 "client_secret", 532 description="The name of the property to use to refresh the `access_token`.", 533 examples=["custom_app_secret"], 534 title="Client Secret Property Name", 535 ) 536 client_secret: Optional[str] = Field( 537 None, 538 description="The OAuth client secret. Fill it in the user inputs.", 539 examples=[ 540 "{{ config['client_secret }}", 541 "{{ config['credentials']['client_secret }}", 542 ], 543 title="Client Secret", 544 ) 545 refresh_token_name: Optional[str] = Field( 546 "refresh_token", 547 description="The name of the property to use to refresh the `access_token`.", 548 examples=["custom_app_refresh_value"], 549 title="Refresh Token Property Name", 550 ) 551 refresh_token: Optional[str] = Field( 552 None, 553 description="Credential artifact used to get a new access token.", 554 examples=[ 555 "{{ config['refresh_token'] }}", 556 "{{ config['credentials]['refresh_token'] }}", 557 ], 558 title="Refresh Token", 559 ) 560 token_refresh_endpoint: Optional[str] = Field( 561 None, 562 description="The full URL to call to obtain a new access token.", 563 examples=["https://connect.squareup.com/oauth2/token"], 564 title="Token Refresh Endpoint", 565 ) 566 access_token_name: Optional[str] = Field( 567 "access_token", 568 description="The name of the property which contains the access token in the response from the token refresh endpoint.", 569 examples=["access_token"], 570 title="Access Token Property Name", 571 ) 572 access_token_value: Optional[str] = Field( 573 None, 574 description="The value of the access_token to bypass the token refreshing using `refresh_token`.", 575 examples=["secret_access_token_value"], 576 title="Access Token Value", 577 ) 578 expires_in_name: Optional[str] = Field( 579 "expires_in", 580 description="The name of the property which contains the expiry date in the response from the token refresh endpoint.", 581 examples=["expires_in"], 582 title="Token Expiry Property Name", 583 ) 584 grant_type_name: Optional[str] = Field( 585 "grant_type", 586 description="The name of the property to use to refresh the `access_token`.", 587 examples=["custom_grant_type"], 588 title="Grant Type Property Name", 589 ) 590 grant_type: Optional[str] = Field( 591 "refresh_token", 592 description="Specifies the OAuth2 grant type. If set to refresh_token, the refresh_token needs to be provided as well. For client_credentials, only client id and secret are required. Other grant types are not officially supported.", 593 examples=["refresh_token", "client_credentials"], 594 title="Grant Type", 595 ) 596 refresh_request_body: Optional[Dict[str, Any]] = Field( 597 None, 598 description="Body of the request sent to get a new access token.", 599 examples=[ 600 { 601 "applicationId": "{{ config['application_id'] }}", 602 "applicationSecret": "{{ config['application_secret'] }}", 603 "token": "{{ config['token'] }}", 604 } 605 ], 606 title="Refresh Request Body", 607 ) 608 refresh_request_headers: Optional[Dict[str, Any]] = Field( 609 None, 610 description="Headers of the request sent to get a new access token.", 611 examples=[ 612 { 613 "Authorization": "<AUTH_TOKEN>", 614 "Content-Type": "application/x-www-form-urlencoded", 615 } 616 ], 617 title="Refresh Request Headers", 618 ) 619 scopes: Optional[List[str]] = Field( 620 None, 621 description="List of scopes that should be granted to the access token.", 622 examples=[["crm.list.read", "crm.objects.contacts.read", "crm.schema.contacts.read"]], 623 title="Scopes", 624 ) 625 token_expiry_date: Optional[str] = Field( 626 None, 627 description="The access token expiry date.", 628 examples=["2023-04-06T07:12:10.421833+00:00", 1680842386], 629 title="Token Expiry Date", 630 ) 631 token_expiry_date_format: Optional[str] = Field( 632 None, 633 description="The format of the time to expiration datetime. Provide it if the time is returned as a date-time string instead of seconds.", 634 examples=["%Y-%m-%d %H:%M:%S.%f+00:00"], 635 title="Token Expiry Date Format", 636 ) 637 refresh_token_updater: Optional[RefreshTokenUpdater] = Field( 638 None, 639 description="When the token updater is defined, new refresh tokens, access tokens and the access token expiry date are written back from the authentication response to the config object. This is important if the refresh token can only used once.", 640 title="Token Updater", 641 ) 642 profile_assertion: Optional[JwtAuthenticator] = Field( 643 None, 644 description="The authenticator being used to authenticate the client authenticator.", 645 title="Profile Assertion", 646 ) 647 use_profile_assertion: Optional[bool] = Field( 648 False, 649 description="Enable using profile assertion as a flow for OAuth authorization.", 650 title="Use Profile Assertion", 651 ) 652 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters")
refresh_token_updater: Optional[RefreshTokenUpdater]
profile_assertion: Optional[JwtAuthenticator]
class
Rate(pydantic.v1.main.BaseModel):
655class Rate(BaseModel): 656 class Config: 657 extra = Extra.allow 658 659 limit: Union[int, str] = Field( 660 ..., 661 description="The maximum number of calls allowed within the interval.", 662 title="Limit", 663 ) 664 interval: str = Field( 665 ..., 666 description="The time interval for the rate limit.", 667 examples=["PT1H", "P1D"], 668 title="Interval", 669 )
class
Rate.Config:
class
HttpRequestRegexMatcher(pydantic.v1.main.BaseModel):
672class HttpRequestRegexMatcher(BaseModel): 673 class Config: 674 extra = Extra.allow 675 676 method: Optional[str] = Field( 677 None, description="The HTTP method to match (e.g., GET, POST).", title="Method" 678 ) 679 url_base: Optional[str] = Field( 680 None, 681 description='The base URL (scheme and host, e.g. "https://api.example.com") to match.', 682 title="URL Base", 683 ) 684 url_path_pattern: Optional[str] = Field( 685 None, 686 description="A regular expression pattern to match the URL path.", 687 title="URL Path Pattern", 688 ) 689 params: Optional[Dict[str, Any]] = Field( 690 None, description="The query parameters to match.", title="Parameters" 691 ) 692 headers: Optional[Dict[str, Any]] = Field( 693 None, description="The headers to match.", title="Headers" 694 )
class
HttpRequestRegexMatcher.Config:
class
DpathExtractor(pydantic.v1.main.BaseModel):
697class DpathExtractor(BaseModel): 698 type: Literal["DpathExtractor"] 699 field_path: List[str] = Field( 700 ..., 701 description='List of potentially nested fields describing the full path of the field to extract. Use "*" to extract all values from an array. See more info in the [docs](https://docs.airbyte.com/connector-development/config-based/understanding-the-yaml-file/record-selector).', 702 examples=[ 703 ["data"], 704 ["data", "records"], 705 ["data", "{{ parameters.name }}"], 706 ["data", "*", "record"], 707 ], 708 title="Field Path", 709 ) 710 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters")
class
ResponseToFileExtractor(pydantic.v1.main.BaseModel):
class
ExponentialBackoffStrategy(pydantic.v1.main.BaseModel):
718class ExponentialBackoffStrategy(BaseModel): 719 type: Literal["ExponentialBackoffStrategy"] 720 factor: Optional[Union[float, str]] = Field( 721 5, 722 description="Multiplicative constant applied on each retry.", 723 examples=[5, 5.5, "10"], 724 title="Factor", 725 ) 726 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters")
class
GroupByKeyMergeStrategy(pydantic.v1.main.BaseModel):
729class GroupByKeyMergeStrategy(BaseModel): 730 type: Literal["GroupByKeyMergeStrategy"] 731 key: Union[str, List[str]] = Field( 732 ..., 733 description="The name of the field on the record whose value will be used to group properties that were retrieved through multiple API requests.", 734 examples=["id", ["parent_id", "end_date"]], 735 title="Key", 736 ) 737 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters")
class
SessionTokenRequestBearerAuthenticator(pydantic.v1.main.BaseModel):
class
HttpMethod(enum.Enum):
An enumeration.
GET =
<HttpMethod.GET: 'GET'>
POST =
<HttpMethod.POST: 'POST'>
class
Action(enum.Enum):
749class Action(Enum): 750 SUCCESS = "SUCCESS" 751 FAIL = "FAIL" 752 RETRY = "RETRY" 753 IGNORE = "IGNORE" 754 RATE_LIMITED = "RATE_LIMITED"
An enumeration.
SUCCESS =
<Action.SUCCESS: 'SUCCESS'>
FAIL =
<Action.FAIL: 'FAIL'>
RETRY =
<Action.RETRY: 'RETRY'>
IGNORE =
<Action.IGNORE: 'IGNORE'>
RATE_LIMITED =
<Action.RATE_LIMITED: 'RATE_LIMITED'>
class
FailureType(enum.Enum):
757class FailureType(Enum): 758 system_error = "system_error" 759 config_error = "config_error" 760 transient_error = "transient_error"
An enumeration.
system_error =
<FailureType.system_error: 'system_error'>
config_error =
<FailureType.config_error: 'config_error'>
transient_error =
<FailureType.transient_error: 'transient_error'>
class
HttpResponseFilter(pydantic.v1.main.BaseModel):
763class HttpResponseFilter(BaseModel): 764 type: Literal["HttpResponseFilter"] 765 action: Optional[Action] = Field( 766 None, 767 description="Action to execute if a response matches the filter.", 768 examples=["SUCCESS", "FAIL", "RETRY", "IGNORE", "RATE_LIMITED"], 769 title="Action", 770 ) 771 failure_type: Optional[FailureType] = Field( 772 None, 773 description="Failure type of traced exception if a response matches the filter.", 774 examples=["system_error", "config_error", "transient_error"], 775 title="Failure Type", 776 ) 777 error_message: Optional[str] = Field( 778 None, 779 description="Error Message to display if the response matches the filter.", 780 title="Error Message", 781 ) 782 error_message_contains: Optional[str] = Field( 783 None, 784 description="Match the response if its error message contains the substring.", 785 example=["This API operation is not enabled for this site"], 786 title="Error Message Substring", 787 ) 788 http_codes: Optional[List[int]] = Field( 789 None, 790 description="Match the response if its HTTP code is included in this list.", 791 examples=[[420, 429], [500]], 792 title="HTTP Codes", 793 unique_items=True, 794 ) 795 predicate: Optional[str] = Field( 796 None, 797 description="Match the response if the predicate evaluates to true.", 798 examples=[ 799 "{{ 'Too much requests' in response }}", 800 "{{ 'error_code' in response and response['error_code'] == 'ComplexityException' }}", 801 ], 802 title="Predicate", 803 ) 804 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters")
action: Optional[Action]
failure_type: Optional[FailureType]
class
ComplexFieldType(pydantic.v1.main.BaseModel):
807class ComplexFieldType(BaseModel): 808 field_type: str 809 items: Optional[Union[str, ComplexFieldType]] = None
items: Union[str, ComplexFieldType, NoneType]
class
TypesMap(pydantic.v1.main.BaseModel):
812class TypesMap(BaseModel): 813 target_type: Union[str, List[str], ComplexFieldType] 814 current_type: Union[str, List[str]] 815 condition: Optional[str] = None
target_type: Union[str, List[str], ComplexFieldType]
class
SchemaTypeIdentifier(pydantic.v1.main.BaseModel):
818class SchemaTypeIdentifier(BaseModel): 819 type: Optional[Literal["SchemaTypeIdentifier"]] = None 820 schema_pointer: Optional[List[str]] = Field( 821 [], 822 description="List of nested fields defining the schema field path to extract. Defaults to [].", 823 title="Schema Path", 824 ) 825 key_pointer: List[str] = Field( 826 ..., 827 description="List of potentially nested fields describing the full path of the field key to extract.", 828 title="Key Path", 829 ) 830 type_pointer: Optional[List[str]] = Field( 831 None, 832 description="List of potentially nested fields describing the full path of the field type to extract.", 833 title="Type Path", 834 ) 835 types_mapping: Optional[List[TypesMap]] = None 836 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters")
types_mapping: Optional[List[TypesMap]]
class
InlineSchemaLoader(pydantic.v1.main.BaseModel):
839class InlineSchemaLoader(BaseModel): 840 type: Literal["InlineSchemaLoader"] 841 schema_: Optional[Dict[str, Any]] = Field( 842 None, 843 alias="schema", 844 description='Describes a streams\' schema. Refer to the <a href="https://docs.airbyte.com/understanding-airbyte/supported-data-types/">Data Types documentation</a> for more details on which types are valid.', 845 title="Schema", 846 )
class
JsonFileSchemaLoader(pydantic.v1.main.BaseModel):
849class JsonFileSchemaLoader(BaseModel): 850 type: Literal["JsonFileSchemaLoader"] 851 file_path: Optional[str] = Field( 852 None, 853 description="Path to the JSON file defining the schema. The path is relative to the connector module's root.", 854 example=["./schemas/users.json"], 855 title="File Path", 856 ) 857 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters")
class
JsonDecoder(pydantic.v1.main.BaseModel):
class
JsonlDecoder(pydantic.v1.main.BaseModel):
class
KeysToLower(pydantic.v1.main.BaseModel):
class
KeysToSnakeCase(pydantic.v1.main.BaseModel):
class
FlattenFields(pydantic.v1.main.BaseModel):
878class FlattenFields(BaseModel): 879 type: Literal["FlattenFields"] 880 flatten_lists: Optional[bool] = Field( 881 True, 882 description="Whether to flatten lists or leave it as is. Default is True.", 883 title="Flatten Lists", 884 ) 885 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters")
class
KeyTransformation(pydantic.v1.main.BaseModel):
888class KeyTransformation(BaseModel): 889 type: Literal["KeyTransformation"] 890 prefix: Optional[str] = Field( 891 None, 892 description="Prefix to add for object keys. If not provided original keys remain unchanged.", 893 examples=["flattened_"], 894 title="Key Prefix", 895 ) 896 suffix: Optional[str] = Field( 897 None, 898 description="Suffix to add for object keys. If not provided original keys remain unchanged.", 899 examples=["_flattened"], 900 title="Key Suffix", 901 )
class
DpathFlattenFields(pydantic.v1.main.BaseModel):
904class DpathFlattenFields(BaseModel): 905 type: Literal["DpathFlattenFields"] 906 field_path: List[str] = Field( 907 ..., 908 description="A path to field that needs to be flattened.", 909 examples=[["data"], ["data", "*", "field"]], 910 title="Field Path", 911 ) 912 delete_origin_value: Optional[bool] = Field( 913 None, 914 description="Whether to delete the origin value or keep it. Default is False.", 915 title="Delete Origin Value", 916 ) 917 replace_record: Optional[bool] = Field( 918 None, 919 description="Whether to replace the origin record or not. Default is False.", 920 title="Replace Origin Record", 921 ) 922 key_transformation: Optional[KeyTransformation] = Field( 923 None, 924 description="Transformation for object keys. If not provided, original key will be used.", 925 title="Key transformation", 926 ) 927 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters")
key_transformation: Optional[KeyTransformation]
class
KeysReplace(pydantic.v1.main.BaseModel):
930class KeysReplace(BaseModel): 931 type: Literal["KeysReplace"] 932 old: str = Field( 933 ..., 934 description="Old value to replace.", 935 examples=[ 936 " ", 937 "{{ record.id }}", 938 "{{ config['id'] }}", 939 "{{ stream_slice['id'] }}", 940 ], 941 title="Old value", 942 ) 943 new: str = Field( 944 ..., 945 description="New value to set.", 946 examples=[ 947 "_", 948 "{{ record.id }}", 949 "{{ config['id'] }}", 950 "{{ stream_slice['id'] }}", 951 ], 952 title="New value", 953 ) 954 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters")
class
IterableDecoder(pydantic.v1.main.BaseModel):
class
XmlDecoder(pydantic.v1.main.BaseModel):
class
CustomDecoder(pydantic.v1.main.BaseModel):
965class CustomDecoder(BaseModel): 966 class Config: 967 extra = Extra.allow 968 969 type: Literal["CustomDecoder"] 970 class_name: str = Field( 971 ..., 972 description="Fully-qualified name of the class that will be implementing the custom decoding. Has to be a sub class of Decoder. The format is `source_<name>.<package>.<class_name>`.", 973 examples=["source_amazon_ads.components.GzipJsonlDecoder"], 974 title="Class Name", 975 ) 976 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters")
class
CustomDecoder.Config:
class
MinMaxDatetime(pydantic.v1.main.BaseModel):
979class MinMaxDatetime(BaseModel): 980 type: Literal["MinMaxDatetime"] 981 datetime: str = Field( 982 ..., 983 description="Datetime value.", 984 examples=["2021-01-01", "2021-01-01T00:00:00Z", "{{ config['start_time'] }}"], 985 title="Datetime", 986 ) 987 datetime_format: Optional[str] = Field( 988 "", 989 description='Format of the datetime value. Defaults to "%Y-%m-%dT%H:%M:%S.%f%z" if left empty. Use placeholders starting with "%" to describe the format the API is using. The following placeholders are available:\n * **%s**: Epoch unix timestamp - `1686218963`\n * **%s_as_float**: Epoch unix timestamp in seconds as float with microsecond precision - `1686218963.123456`\n * **%ms**: Epoch unix timestamp - `1686218963123`\n * **%a**: Weekday (abbreviated) - `Sun`\n * **%A**: Weekday (full) - `Sunday`\n * **%w**: Weekday (decimal) - `0` (Sunday), `6` (Saturday)\n * **%d**: Day of the month (zero-padded) - `01`, `02`, ..., `31`\n * **%b**: Month (abbreviated) - `Jan`\n * **%B**: Month (full) - `January`\n * **%m**: Month (zero-padded) - `01`, `02`, ..., `12`\n * **%y**: Year (without century, zero-padded) - `00`, `01`, ..., `99`\n * **%Y**: Year (with century) - `0001`, `0002`, ..., `9999`\n * **%H**: Hour (24-hour, zero-padded) - `00`, `01`, ..., `23`\n * **%I**: Hour (12-hour, zero-padded) - `01`, `02`, ..., `12`\n * **%p**: AM/PM indicator\n * **%M**: Minute (zero-padded) - `00`, `01`, ..., `59`\n * **%S**: Second (zero-padded) - `00`, `01`, ..., `59`\n * **%f**: Microsecond (zero-padded to 6 digits) - `000000`, `000001`, ..., `999999`\n * **%_ms**: Millisecond (zero-padded to 3 digits) - `000`, `001`, ..., `999`\n * **%z**: UTC offset - `(empty)`, `+0000`, `-04:00`\n * **%Z**: Time zone name - `(empty)`, `UTC`, `GMT`\n * **%j**: Day of the year (zero-padded) - `001`, `002`, ..., `366`\n * **%U**: Week number of the year (Sunday as first day) - `00`, `01`, ..., `53`\n * **%W**: Week number of the year (Monday as first day) - `00`, `01`, ..., `53`\n * **%c**: Date and time representation - `Tue Aug 16 21:30:00 1988`\n * **%x**: Date representation - `08/16/1988`\n * **%X**: Time representation - `21:30:00`\n * **%%**: Literal \'%\' character\n\n Some placeholders depend on the locale of the underlying system - in most cases this locale is configured as en/US. For more information see the [Python documentation](https://docs.python.org/3/library/datetime.html#strftime-and-strptime-format-codes).\n', 990 examples=["%Y-%m-%dT%H:%M:%S.%f%z", "%Y-%m-%d", "%s"], 991 title="Datetime Format", 992 ) 993 max_datetime: Optional[str] = Field( 994 None, 995 description="Ceiling applied on the datetime value. Must be formatted with the datetime_format field.", 996 examples=["2021-01-01T00:00:00Z", "2021-01-01"], 997 title="Max Datetime", 998 ) 999 min_datetime: Optional[str] = Field( 1000 None, 1001 description="Floor applied on the datetime value. Must be formatted with the datetime_format field.", 1002 examples=["2010-01-01T00:00:00Z", "2010-01-01"], 1003 title="Min Datetime", 1004 ) 1005 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters")
class
NoAuth(pydantic.v1.main.BaseModel):
class
NoPagination(pydantic.v1.main.BaseModel):
class
State(pydantic.v1.main.BaseModel):
class
State.Config:
class
OauthConnectorInputSpecification(pydantic.v1.main.BaseModel):
1025class OauthConnectorInputSpecification(BaseModel): 1026 class Config: 1027 extra = Extra.allow 1028 1029 consent_url: str = Field( 1030 ..., 1031 description="The DeclarativeOAuth Specific string URL string template to initiate the authentication.\nThe placeholders are replaced during the processing to provide neccessary values.", 1032 examples=[ 1033 "https://domain.host.com/marketing_api/auth?{{client_id_key}}={{client_id_value}}&{{redirect_uri_key}}={{{{redirect_uri_value}} | urlEncoder}}&{{state_key}}={{state_value}}", 1034 "https://endpoint.host.com/oauth2/authorize?{{client_id_key}}={{client_id_value}}&{{redirect_uri_key}}={{{{redirect_uri_value}} | urlEncoder}}&{{scope_key}}={{{{scope_value}} | urlEncoder}}&{{state_key}}={{state_value}}&subdomain={{subdomain}}", 1035 ], 1036 title="Consent URL", 1037 ) 1038 scope: Optional[str] = Field( 1039 None, 1040 description="The DeclarativeOAuth Specific string of the scopes needed to be grant for authenticated user.", 1041 examples=["user:read user:read_orders workspaces:read"], 1042 title="Scopes", 1043 ) 1044 access_token_url: str = Field( 1045 ..., 1046 description="The DeclarativeOAuth Specific URL templated string to obtain the `access_token`, `refresh_token` etc.\nThe placeholders are replaced during the processing to provide neccessary values.", 1047 examples=[ 1048 "https://auth.host.com/oauth2/token?{{client_id_key}}={{client_id_value}}&{{client_secret_key}}={{client_secret_value}}&{{auth_code_key}}={{auth_code_value}}&{{redirect_uri_key}}={{{{redirect_uri_value}} | urlEncoder}}" 1049 ], 1050 title="Access Token URL", 1051 ) 1052 access_token_headers: Optional[Dict[str, Any]] = Field( 1053 None, 1054 description="The DeclarativeOAuth Specific optional headers to inject while exchanging the `auth_code` to `access_token` during `completeOAuthFlow` step.", 1055 examples=[ 1056 { 1057 "Authorization": "Basic {{ {{ client_id_value }}:{{ client_secret_value }} | base64Encoder }}" 1058 } 1059 ], 1060 title="Access Token Headers", 1061 ) 1062 access_token_params: Optional[Dict[str, Any]] = Field( 1063 None, 1064 description="The DeclarativeOAuth Specific optional query parameters to inject while exchanging the `auth_code` to `access_token` during `completeOAuthFlow` step.\nWhen this property is provided, the query params will be encoded as `Json` and included in the outgoing API request.", 1065 examples=[ 1066 { 1067 "{{ auth_code_key }}": "{{ auth_code_value }}", 1068 "{{ client_id_key }}": "{{ client_id_value }}", 1069 "{{ client_secret_key }}": "{{ client_secret_value }}", 1070 } 1071 ], 1072 title="Access Token Query Params (Json Encoded)", 1073 ) 1074 extract_output: Optional[List[str]] = Field( 1075 None, 1076 description="The DeclarativeOAuth Specific list of strings to indicate which keys should be extracted and returned back to the input config.", 1077 examples=[["access_token", "refresh_token", "other_field"]], 1078 title="Extract Output", 1079 ) 1080 state: Optional[State] = Field( 1081 None, 1082 description="The DeclarativeOAuth Specific object to provide the criteria of how the `state` query param should be constructed,\nincluding length and complexity.", 1083 examples=[{"min": 7, "max": 128}], 1084 title="Configurable State Query Param", 1085 ) 1086 client_id_key: Optional[str] = Field( 1087 None, 1088 description="The DeclarativeOAuth Specific optional override to provide the custom `client_id` key name, if required by data-provider.", 1089 examples=["my_custom_client_id_key_name"], 1090 title="Client ID Key Override", 1091 ) 1092 client_secret_key: Optional[str] = Field( 1093 None, 1094 description="The DeclarativeOAuth Specific optional override to provide the custom `client_secret` key name, if required by data-provider.", 1095 examples=["my_custom_client_secret_key_name"], 1096 title="Client Secret Key Override", 1097 ) 1098 scope_key: Optional[str] = Field( 1099 None, 1100 description="The DeclarativeOAuth Specific optional override to provide the custom `scope` key name, if required by data-provider.", 1101 examples=["my_custom_scope_key_key_name"], 1102 title="Scopes Key Override", 1103 ) 1104 state_key: Optional[str] = Field( 1105 None, 1106 description="The DeclarativeOAuth Specific optional override to provide the custom `state` key name, if required by data-provider.", 1107 examples=["my_custom_state_key_key_name"], 1108 title="State Key Override", 1109 ) 1110 auth_code_key: Optional[str] = Field( 1111 None, 1112 description="The DeclarativeOAuth Specific optional override to provide the custom `code` key name to something like `auth_code` or `custom_auth_code`, if required by data-provider.", 1113 examples=["my_custom_auth_code_key_name"], 1114 title="Auth Code Key Override", 1115 ) 1116 redirect_uri_key: Optional[str] = Field( 1117 None, 1118 description="The DeclarativeOAuth Specific optional override to provide the custom `redirect_uri` key name to something like `callback_uri`, if required by data-provider.", 1119 examples=["my_custom_redirect_uri_key_name"], 1120 title="Redirect URI Key Override", 1121 )
state: Optional[State]
class
OauthConnectorInputSpecification.Config:
class
OAuthConfigSpecification(pydantic.v1.main.BaseModel):
1124class OAuthConfigSpecification(BaseModel): 1125 class Config: 1126 extra = Extra.allow 1127 1128 oauth_user_input_from_connector_config_specification: Optional[Dict[str, Any]] = Field( 1129 None, 1130 description="OAuth specific blob. This is a Json Schema used to validate Json configurations used as input to OAuth.\nMust be a valid non-nested JSON that refers to properties from ConnectorSpecification.connectionSpecification\nusing special annotation 'path_in_connector_config'.\nThese are input values the user is entering through the UI to authenticate to the connector, that might also shared\nas inputs for syncing data via the connector.\nExamples:\nif no connector values is shared during oauth flow, oauth_user_input_from_connector_config_specification=[]\nif connector values such as 'app_id' inside the top level are used to generate the API url for the oauth flow,\n oauth_user_input_from_connector_config_specification={\n app_id: {\n type: string\n path_in_connector_config: ['app_id']\n }\n }\nif connector values such as 'info.app_id' nested inside another object are used to generate the API url for the oauth flow,\n oauth_user_input_from_connector_config_specification={\n app_id: {\n type: string\n path_in_connector_config: ['info', 'app_id']\n }\n }", 1131 examples=[ 1132 {"app_id": {"type": "string", "path_in_connector_config": ["app_id"]}}, 1133 { 1134 "app_id": { 1135 "type": "string", 1136 "path_in_connector_config": ["info", "app_id"], 1137 } 1138 }, 1139 ], 1140 title="OAuth user input", 1141 ) 1142 oauth_connector_input_specification: Optional[OauthConnectorInputSpecification] = Field( 1143 None, 1144 description='The DeclarativeOAuth specific blob.\nPertains to the fields defined by the connector relating to the OAuth flow.\n\nInterpolation capabilities:\n- The variables placeholders are declared as `{{my_var}}`.\n- The nested resolution variables like `{{ {{my_nested_var}} }}` is allowed as well.\n\n- The allowed interpolation context is:\n + base64Encoder - encode to `base64`, {{ {{my_var_a}}:{{my_var_b}} | base64Encoder }}\n + base64Decorer - decode from `base64` encoded string, {{ {{my_string_variable_or_string_value}} | base64Decoder }}\n + urlEncoder - encode the input string to URL-like format, {{ https://test.host.com/endpoint | urlEncoder}}\n + urlDecorer - decode the input url-encoded string into text format, {{ urlDecoder:https%3A%2F%2Fairbyte.io | urlDecoder}}\n + codeChallengeS256 - get the `codeChallenge` encoded value to provide additional data-provider specific authorisation values, {{ {{state_value}} | codeChallengeS256 }}\n\nExamples:\n - The TikTok Marketing DeclarativeOAuth spec:\n {\n "oauth_connector_input_specification": {\n "type": "object",\n "additionalProperties": false,\n "properties": {\n "consent_url": "https://ads.tiktok.com/marketing_api/auth?{{client_id_key}}={{client_id_value}}&{{redirect_uri_key}}={{ {{redirect_uri_value}} | urlEncoder}}&{{state_key}}={{state_value}}",\n "access_token_url": "https://business-api.tiktok.com/open_api/v1.3/oauth2/access_token/",\n "access_token_params": {\n "{{ auth_code_key }}": "{{ auth_code_value }}",\n "{{ client_id_key }}": "{{ client_id_value }}",\n "{{ client_secret_key }}": "{{ client_secret_value }}"\n },\n "access_token_headers": {\n "Content-Type": "application/json",\n "Accept": "application/json"\n },\n "extract_output": ["data.access_token"],\n "client_id_key": "app_id",\n "client_secret_key": "secret",\n "auth_code_key": "auth_code"\n }\n }\n }', 1145 title="DeclarativeOAuth Connector Specification", 1146 ) 1147 complete_oauth_output_specification: Optional[Dict[str, Any]] = Field( 1148 None, 1149 description="OAuth specific blob. This is a Json Schema used to validate Json configurations produced by the OAuth flows as they are\nreturned by the distant OAuth APIs.\nMust be a valid JSON describing the fields to merge back to `ConnectorSpecification.connectionSpecification`.\nFor each field, a special annotation `path_in_connector_config` can be specified to determine where to merge it,\nExamples:\n complete_oauth_output_specification={\n refresh_token: {\n type: string,\n path_in_connector_config: ['credentials', 'refresh_token']\n }\n }", 1150 examples=[ 1151 { 1152 "refresh_token": { 1153 "type": "string,", 1154 "path_in_connector_config": ["credentials", "refresh_token"], 1155 } 1156 } 1157 ], 1158 title="OAuth output specification", 1159 ) 1160 complete_oauth_server_input_specification: Optional[Dict[str, Any]] = Field( 1161 None, 1162 description="OAuth specific blob. This is a Json Schema used to validate Json configurations persisted as Airbyte Server configurations.\nMust be a valid non-nested JSON describing additional fields configured by the Airbyte Instance or Workspace Admins to be used by the\nserver when completing an OAuth flow (typically exchanging an auth code for refresh token).\nExamples:\n complete_oauth_server_input_specification={\n client_id: {\n type: string\n },\n client_secret: {\n type: string\n }\n }", 1163 examples=[{"client_id": {"type": "string"}, "client_secret": {"type": "string"}}], 1164 title="OAuth input specification", 1165 ) 1166 complete_oauth_server_output_specification: Optional[Dict[str, Any]] = Field( 1167 None, 1168 description="OAuth specific blob. This is a Json Schema used to validate Json configurations persisted as Airbyte Server configurations that\nalso need to be merged back into the connector configuration at runtime.\nThis is a subset configuration of `complete_oauth_server_input_specification` that filters fields out to retain only the ones that\nare necessary for the connector to function with OAuth. (some fields could be used during oauth flows but not needed afterwards, therefore\nthey would be listed in the `complete_oauth_server_input_specification` but not `complete_oauth_server_output_specification`)\nMust be a valid non-nested JSON describing additional fields configured by the Airbyte Instance or Workspace Admins to be used by the\nconnector when using OAuth flow APIs.\nThese fields are to be merged back to `ConnectorSpecification.connectionSpecification`.\nFor each field, a special annotation `path_in_connector_config` can be specified to determine where to merge it,\nExamples:\n complete_oauth_server_output_specification={\n client_id: {\n type: string,\n path_in_connector_config: ['credentials', 'client_id']\n },\n client_secret: {\n type: string,\n path_in_connector_config: ['credentials', 'client_secret']\n }\n }", 1169 examples=[ 1170 { 1171 "client_id": { 1172 "type": "string,", 1173 "path_in_connector_config": ["credentials", "client_id"], 1174 }, 1175 "client_secret": { 1176 "type": "string,", 1177 "path_in_connector_config": ["credentials", "client_secret"], 1178 }, 1179 } 1180 ], 1181 title="OAuth server output specification", 1182 )
oauth_connector_input_specification: Optional[OauthConnectorInputSpecification]
class
OAuthConfigSpecification.Config:
class
OffsetIncrement(pydantic.v1.main.BaseModel):
1185class OffsetIncrement(BaseModel): 1186 type: Literal["OffsetIncrement"] 1187 page_size: Optional[Union[int, str]] = Field( 1188 None, 1189 description="The number of records to include in each pages.", 1190 examples=[100, "{{ config['page_size'] }}"], 1191 title="Limit", 1192 ) 1193 inject_on_first_request: Optional[bool] = Field( 1194 False, 1195 description="Using the `offset` with value `0` during the first request", 1196 title="Inject Offset", 1197 ) 1198 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters")
class
PageIncrement(pydantic.v1.main.BaseModel):
1201class PageIncrement(BaseModel): 1202 type: Literal["PageIncrement"] 1203 page_size: Optional[Union[int, str]] = Field( 1204 None, 1205 description="The number of records to include in each pages.", 1206 examples=[100, "100", "{{ config['page_size'] }}"], 1207 title="Page Size", 1208 ) 1209 start_from_page: Optional[int] = Field( 1210 0, 1211 description="Index of the first page to request.", 1212 examples=[0, 1], 1213 title="Start From Page", 1214 ) 1215 inject_on_first_request: Optional[bool] = Field( 1216 False, 1217 description="Using the `page number` with value defined by `start_from_page` during the first request", 1218 title="Inject Page Number", 1219 ) 1220 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters")
class
PrimaryKey(pydantic.v1.main.BaseModel):
1223class PrimaryKey(BaseModel): 1224 __root__: Union[str, List[str], List[List[str]]] = Field( 1225 ..., 1226 description="The stream field to be used to distinguish unique records. Can either be a single field, an array of fields representing a composite key, or an array of arrays representing a composite key where the fields are nested fields.", 1227 examples=["id", ["code", "type"]], 1228 title="Primary Key", 1229 )
class
PropertyLimitType(enum.Enum):
1232class PropertyLimitType(Enum): 1233 characters = "characters" 1234 property_count = "property_count"
An enumeration.
characters =
<PropertyLimitType.characters: 'characters'>
property_count =
<PropertyLimitType.property_count: 'property_count'>
class
PropertyChunking(pydantic.v1.main.BaseModel):
1237class PropertyChunking(BaseModel): 1238 type: Literal["PropertyChunking"] 1239 property_limit_type: PropertyLimitType = Field( 1240 ..., 1241 description="The type used to determine the maximum number of properties per chunk", 1242 title="Property Limit Type", 1243 ) 1244 property_limit: Optional[int] = Field( 1245 None, 1246 description="The maximum amount of properties that can be retrieved per request according to the limit type.", 1247 title="Property Limit", 1248 ) 1249 record_merge_strategy: Optional[GroupByKeyMergeStrategy] = Field( 1250 None, 1251 description="Dictates how to records that require multiple requests to get all properties should be emitted to the destination", 1252 title="Record Merge Strategy", 1253 ) 1254 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters")
property_limit_type: PropertyLimitType
record_merge_strategy: Optional[GroupByKeyMergeStrategy]
class
RecordFilter(pydantic.v1.main.BaseModel):
1257class RecordFilter(BaseModel): 1258 type: Literal["RecordFilter"] 1259 condition: Optional[str] = Field( 1260 "", 1261 description="The predicate to filter a record. Records will be removed if evaluated to False.", 1262 examples=[ 1263 "{{ record['created_at'] >= stream_interval['start_time'] }}", 1264 "{{ record.status in ['active', 'expired'] }}", 1265 ], 1266 title="Condition", 1267 ) 1268 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters")
class
SchemaNormalization(enum.Enum):
An enumeration.
Default =
<SchemaNormalization.Default: 'Default'>
None_ =
<SchemaNormalization.None_: 'None'>
class
RemoveFields(pydantic.v1.main.BaseModel):
1276class RemoveFields(BaseModel): 1277 type: Literal["RemoveFields"] 1278 condition: Optional[str] = Field( 1279 "", 1280 description="The predicate to filter a property by a property value. Property will be removed if it is empty OR expression is evaluated to True.,", 1281 examples=[ 1282 "{{ property|string == '' }}", 1283 "{{ property is integer }}", 1284 "{{ property|length > 5 }}", 1285 "{{ property == 'some_string_to_match' }}", 1286 ], 1287 ) 1288 field_pointers: List[List[str]] = Field( 1289 ..., 1290 description="Array of paths defining the field to remove. Each item is an array whose field describe the path of a field to remove.", 1291 examples=[["tags"], [["content", "html"], ["content", "plain_text"]]], 1292 title="Field Paths", 1293 )
class
RequestPath(pydantic.v1.main.BaseModel):
class
InjectInto(enum.Enum):
1300class InjectInto(Enum): 1301 request_parameter = "request_parameter" 1302 header = "header" 1303 body_data = "body_data" 1304 body_json = "body_json"
An enumeration.
request_parameter =
<InjectInto.request_parameter: 'request_parameter'>
header =
<InjectInto.header: 'header'>
body_data =
<InjectInto.body_data: 'body_data'>
body_json =
<InjectInto.body_json: 'body_json'>
class
RequestOption(pydantic.v1.main.BaseModel):
1307class RequestOption(BaseModel): 1308 type: Literal["RequestOption"] 1309 field_name: Optional[str] = Field( 1310 None, 1311 description="Configures which key should be used in the location that the descriptor is being injected into. We hope to eventually deprecate this field in favor of `field_path` for all request_options, but must currently maintain it for backwards compatibility in the Builder.", 1312 examples=["segment_id"], 1313 title="Field Name", 1314 ) 1315 field_path: Optional[List[str]] = Field( 1316 None, 1317 description="Configures a path to be used for nested structures in JSON body requests (e.g. GraphQL queries)", 1318 examples=[["data", "viewer", "id"]], 1319 title="Field Path", 1320 ) 1321 inject_into: InjectInto = Field( 1322 ..., 1323 description="Configures where the descriptor should be set on the HTTP requests. Note that request parameters that are already encoded in the URL path will not be duplicated.", 1324 examples=["request_parameter", "header", "body_data", "body_json"], 1325 title="Inject Into", 1326 )
inject_into: InjectInto
class
Schemas(pydantic.v1.main.BaseModel):
class
Schemas.Config:
class
LegacySessionTokenAuthenticator(pydantic.v1.main.BaseModel):
1336class LegacySessionTokenAuthenticator(BaseModel): 1337 type: Literal["LegacySessionTokenAuthenticator"] 1338 header: str = Field( 1339 ..., 1340 description="The name of the session token header that will be injected in the request", 1341 examples=["X-Session"], 1342 title="Session Request Header", 1343 ) 1344 login_url: str = Field( 1345 ..., 1346 description="Path of the login URL (do not include the base URL)", 1347 examples=["session"], 1348 title="Login Path", 1349 ) 1350 session_token: Optional[str] = Field( 1351 None, 1352 description="Session token to use if using a pre-defined token. Not needed if authenticating with username + password pair", 1353 example=["{{ config['session_token'] }}"], 1354 title="Session Token", 1355 ) 1356 session_token_response_key: str = Field( 1357 ..., 1358 description="Name of the key of the session token to be extracted from the response", 1359 examples=["id"], 1360 title="Response Token Response Key", 1361 ) 1362 username: Optional[str] = Field( 1363 None, 1364 description="Username used to authenticate and obtain a session token", 1365 examples=[" {{ config['username'] }}"], 1366 title="Username", 1367 ) 1368 password: Optional[str] = Field( 1369 "", 1370 description="Password used to authenticate and obtain a session token", 1371 examples=["{{ config['password'] }}", ""], 1372 title="Password", 1373 ) 1374 validate_session_url: str = Field( 1375 ..., 1376 description="Path of the URL to use to validate that the session token is valid (do not include the base URL)", 1377 examples=["user/current"], 1378 title="Validate Session Path", 1379 ) 1380 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters")
class
CsvDecoder(pydantic.v1.main.BaseModel):
class
AsyncJobStatusMap(pydantic.v1.main.BaseModel):
class
ValueType(enum.Enum):
1397class ValueType(Enum): 1398 string = "string" 1399 number = "number" 1400 integer = "integer" 1401 boolean = "boolean"
An enumeration.
string =
<ValueType.string: 'string'>
number =
<ValueType.number: 'number'>
integer =
<ValueType.integer: 'integer'>
boolean =
<ValueType.boolean: 'boolean'>
class
WaitTimeFromHeader(pydantic.v1.main.BaseModel):
1404class WaitTimeFromHeader(BaseModel): 1405 type: Literal["WaitTimeFromHeader"] 1406 header: str = Field( 1407 ..., 1408 description="The name of the response header defining how long to wait before retrying.", 1409 examples=["Retry-After"], 1410 title="Response Header Name", 1411 ) 1412 regex: Optional[str] = Field( 1413 None, 1414 description="Optional regex to apply on the header to extract its value. The regex should define a capture group defining the wait time.", 1415 examples=["([-+]?\\d+)"], 1416 title="Extraction Regex", 1417 ) 1418 max_waiting_time_in_seconds: Optional[float] = Field( 1419 None, 1420 description="Given the value extracted from the header is greater than this value, stop the stream.", 1421 examples=[3600], 1422 title="Max Waiting Time in Seconds", 1423 ) 1424 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters")
class
WaitUntilTimeFromHeader(pydantic.v1.main.BaseModel):
1427class WaitUntilTimeFromHeader(BaseModel): 1428 type: Literal["WaitUntilTimeFromHeader"] 1429 header: str = Field( 1430 ..., 1431 description="The name of the response header defining how long to wait before retrying.", 1432 examples=["wait_time"], 1433 title="Response Header", 1434 ) 1435 min_wait: Optional[Union[float, str]] = Field( 1436 None, 1437 description="Minimum time to wait before retrying.", 1438 examples=[10, "60"], 1439 title="Minimum Wait Time", 1440 ) 1441 regex: Optional[str] = Field( 1442 None, 1443 description="Optional regex to apply on the header to extract its value. The regex should define a capture group defining the wait time.", 1444 examples=["([-+]?\\d+)"], 1445 title="Extraction Regex", 1446 ) 1447 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters")
class
ComponentMappingDefinition(pydantic.v1.main.BaseModel):
1450class ComponentMappingDefinition(BaseModel): 1451 type: Literal["ComponentMappingDefinition"] 1452 field_path: List[str] = Field( 1453 ..., 1454 description="A list of potentially nested fields indicating the full path where value will be added or updated.", 1455 examples=[ 1456 ["data"], 1457 ["data", "records"], 1458 ["data", 1, "name"], 1459 ["data", "{{ components_values.name }}"], 1460 ["data", "*", "record"], 1461 ["*", "**", "name"], 1462 ], 1463 title="Field Path", 1464 ) 1465 value: str = Field( 1466 ..., 1467 description="The dynamic or static value to assign to the key. Interpolated values can be used to dynamically determine the value during runtime.", 1468 examples=[ 1469 "{{ components_values['updates'] }}", 1470 "{{ components_values['MetaData']['LastUpdatedTime'] }}", 1471 "{{ config['segment_id'] }}", 1472 "{{ stream_slice['parent_id'] }}", 1473 "{{ stream_slice['extra_fields']['name'] }}", 1474 ], 1475 title="Value", 1476 ) 1477 value_type: Optional[ValueType] = Field( 1478 None, 1479 description="The expected data type of the value. If omitted, the type will be inferred from the value provided.", 1480 title="Value Type", 1481 ) 1482 create_or_update: Optional[bool] = Field( 1483 False, 1484 description="Determines whether to create a new path if it doesn't exist (true) or only update existing paths (false). When set to true, the resolver will create new paths in the stream template if they don't exist. When false (default), it will only update existing paths.", 1485 title="Create or Update", 1486 ) 1487 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters")
value_type: Optional[ValueType]
class
StreamConfig(pydantic.v1.main.BaseModel):
1490class StreamConfig(BaseModel): 1491 type: Literal["StreamConfig"] 1492 configs_pointer: List[str] = Field( 1493 ..., 1494 description="A list of potentially nested fields indicating the full path in source config file where streams configs located.", 1495 examples=[["data"], ["data", "streams"], ["data", "{{ parameters.name }}"]], 1496 title="Configs Pointer", 1497 ) 1498 default_values: Optional[List[Dict[str, Any]]] = Field( 1499 None, 1500 description="A list of default values, each matching the structure expected from the parsed component value.", 1501 title="Default Values", 1502 ) 1503 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters")
class
ConfigComponentsResolver(pydantic.v1.main.BaseModel):
1506class ConfigComponentsResolver(BaseModel): 1507 type: Literal["ConfigComponentsResolver"] 1508 stream_config: Union[List[StreamConfig], StreamConfig] 1509 components_mapping: List[ComponentMappingDefinition] 1510 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters")
stream_config: Union[List[StreamConfig], StreamConfig]
components_mapping: List[ComponentMappingDefinition]
class
RequestBodyPlainText(pydantic.v1.main.BaseModel):
class
RequestBodyUrlEncodedForm(pydantic.v1.main.BaseModel):
class
RequestBodyJsonObject(pydantic.v1.main.BaseModel):
class
RequestBodyGraphQlQuery(pydantic.v1.main.BaseModel):
class
RequestBodyGraphQlQuery.Config:
class
ValidateAdheresToSchema(pydantic.v1.main.BaseModel):
1535class ValidateAdheresToSchema(BaseModel): 1536 type: Literal["ValidateAdheresToSchema"] 1537 base_schema: Union[str, Dict[str, Any]] = Field( 1538 ..., 1539 description="The base JSON schema against which the user-provided schema will be validated.", 1540 examples=[ 1541 "{{ config['report_validation_schema'] }}", 1542 '\'{\n "$schema": "http://json-schema.org/draft-07/schema#",\n "title": "Person",\n "type": "object",\n "properties": {\n "name": {\n "type": "string",\n "description": "The person\'s name"\n },\n "age": {\n "type": "integer",\n "minimum": 0,\n "description": "The person\'s age"\n }\n },\n "required": ["name", "age"]\n}\'\n', 1543 { 1544 "$schema": "http://json-schema.org/draft-07/schema#", 1545 "title": "Person", 1546 "type": "object", 1547 "properties": { 1548 "name": {"type": "string", "description": "The person's name"}, 1549 "age": { 1550 "type": "integer", 1551 "minimum": 0, 1552 "description": "The person's age", 1553 }, 1554 }, 1555 "required": ["name", "age"], 1556 }, 1557 ], 1558 title="Base JSON Schema", 1559 )
class
ConfigRemapField(pydantic.v1.main.BaseModel):
1562class ConfigRemapField(BaseModel): 1563 type: Literal["ConfigRemapField"] 1564 map: Union[Dict[str, Any], str] = Field( 1565 ..., 1566 description="A mapping of original values to new values. When a field value matches a key in this map, it will be replaced with the corresponding value.", 1567 examples=[ 1568 {"pending": "in_progress", "done": "completed", "cancelled": "terminated"}, 1569 "{{ config['status_mapping'] }}", 1570 ], 1571 title="Value Mapping", 1572 ) 1573 field_path: List[str] = Field( 1574 ..., 1575 description="The path to the field whose value should be remapped. Specified as a list of path components to navigate through nested objects.", 1576 examples=[ 1577 ["status"], 1578 ["data", "status"], 1579 ["data", "{{ config.name }}", "status"], 1580 ["data", "*", "status"], 1581 ], 1582 title="Field Path", 1583 )
class
ConfigRemoveFields(pydantic.v1.main.BaseModel):
1586class ConfigRemoveFields(BaseModel): 1587 type: Literal["ConfigRemoveFields"] 1588 field_pointers: List[List[str]] = Field( 1589 ..., 1590 description="A list of field pointers to be removed from the config.", 1591 examples=[["tags"], [["content", "html"], ["content", "plain_text"]]], 1592 title="Field Pointers", 1593 ) 1594 condition: Optional[str] = Field( 1595 "", 1596 description="Fields will be removed if expression is evaluated to True.", 1597 examples=[ 1598 "{{ config['environemnt'] == 'sandbox' }}", 1599 "{{ property is integer }}", 1600 "{{ property|length > 5 }}", 1601 "{{ property == 'some_string_to_match' }}", 1602 ], 1603 )
class
AddedFieldDefinition(pydantic.v1.main.BaseModel):
1606class AddedFieldDefinition(BaseModel): 1607 type: Literal["AddedFieldDefinition"] 1608 path: List[str] = Field( 1609 ..., 1610 description="List of strings defining the path where to add the value on the record.", 1611 examples=[["segment_id"], ["metadata", "segment_id"]], 1612 title="Path", 1613 ) 1614 value: str = Field( 1615 ..., 1616 description="Value of the new field. Use {{ record['existing_field'] }} syntax to refer to other fields in the record.", 1617 examples=[ 1618 "{{ record['updates'] }}", 1619 "{{ record['MetaData']['LastUpdatedTime'] }}", 1620 "{{ stream_partition['segment_id'] }}", 1621 ], 1622 title="Value", 1623 ) 1624 value_type: Optional[ValueType] = Field( 1625 None, 1626 description="Type of the value. If not specified, the type will be inferred from the value.", 1627 title="Value Type", 1628 ) 1629 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters")
value_type: Optional[ValueType]
class
AddFields(pydantic.v1.main.BaseModel):
1632class AddFields(BaseModel): 1633 type: Literal["AddFields"] 1634 fields: List[AddedFieldDefinition] = Field( 1635 ..., 1636 description="List of transformations (path and corresponding value) that will be added to the record.", 1637 title="Fields", 1638 ) 1639 condition: Optional[str] = Field( 1640 "", 1641 description="Fields will be added if expression is evaluated to True.", 1642 examples=[ 1643 "{{ property|string == '' }}", 1644 "{{ property is integer }}", 1645 "{{ property|length > 5 }}", 1646 "{{ property == 'some_string_to_match' }}", 1647 ], 1648 ) 1649 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters")
fields: List[AddedFieldDefinition]
class
ApiKeyAuthenticator(pydantic.v1.main.BaseModel):
1652class ApiKeyAuthenticator(BaseModel): 1653 type: Literal["ApiKeyAuthenticator"] 1654 api_token: Optional[str] = Field( 1655 None, 1656 description="The API key to inject in the request. Fill it in the user inputs.", 1657 examples=["{{ config['api_key'] }}", "Token token={{ config['api_key'] }}"], 1658 title="API Key", 1659 ) 1660 header: Optional[str] = Field( 1661 None, 1662 description="The name of the HTTP header that will be set to the API key. This setting is deprecated, use inject_into instead. Header and inject_into can not be defined at the same time.", 1663 examples=["Authorization", "Api-Token", "X-Auth-Token"], 1664 title="Header Name", 1665 ) 1666 inject_into: Optional[RequestOption] = Field( 1667 None, 1668 description="Configure how the API Key will be sent in requests to the source API. Either inject_into or header has to be defined.", 1669 examples=[ 1670 {"inject_into": "header", "field_name": "Authorization"}, 1671 {"inject_into": "request_parameter", "field_name": "authKey"}, 1672 ], 1673 title="Inject API Key Into Outgoing HTTP Request", 1674 ) 1675 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters")
inject_into: Optional[RequestOption]
class
AuthFlow(pydantic.v1.main.BaseModel):
1678class AuthFlow(BaseModel): 1679 auth_flow_type: Optional[AuthFlowType] = Field( 1680 None, description="The type of auth to use", title="Auth flow type" 1681 ) 1682 predicate_key: Optional[List[str]] = Field( 1683 None, 1684 description="JSON path to a field in the connectorSpecification that should exist for the advanced auth to be applicable.", 1685 examples=[["credentials", "auth_type"]], 1686 title="Predicate key", 1687 ) 1688 predicate_value: Optional[str] = Field( 1689 None, 1690 description="Value of the predicate_key fields for the advanced auth to be applicable.", 1691 examples=["Oauth"], 1692 title="Predicate value", 1693 ) 1694 oauth_config_specification: Optional[OAuthConfigSpecification] = None
auth_flow_type: Optional[AuthFlowType]
oauth_config_specification: Optional[OAuthConfigSpecification]
class
CheckStream(pydantic.v1.main.BaseModel):
1697class CheckStream(BaseModel): 1698 type: Literal["CheckStream"] 1699 stream_names: Optional[List[str]] = Field( 1700 None, 1701 description="Names of the streams to try reading from when running a check operation.", 1702 examples=[["users"], ["users", "contacts"]], 1703 title="Stream Names", 1704 ) 1705 dynamic_streams_check_configs: Optional[List[DynamicStreamCheckConfig]] = None
dynamic_streams_check_configs: Optional[List[DynamicStreamCheckConfig]]
class
IncrementingCountCursor(pydantic.v1.main.BaseModel):
1708class IncrementingCountCursor(BaseModel): 1709 type: Literal["IncrementingCountCursor"] 1710 cursor_field: str = Field( 1711 ..., 1712 description="The location of the value on a record that will be used as a bookmark during sync. To ensure no data loss, the API must return records in ascending order based on the cursor field. Nested fields are not supported, so the field must be at the top level of the record. You can use a combination of Add Field and Remove Field transformations to move the nested field to the top.", 1713 examples=["created_at", "{{ config['record_cursor'] }}"], 1714 title="Cursor Field", 1715 ) 1716 start_value: Optional[Union[str, int]] = Field( 1717 None, 1718 description="The value that determines the earliest record that should be synced.", 1719 examples=[0, "{{ config['start_value'] }}"], 1720 title="Start Value", 1721 ) 1722 start_value_option: Optional[RequestOption] = Field( 1723 None, 1724 description="Optionally configures how the start value will be sent in requests to the source API.", 1725 title="Inject Start Value Into Outgoing HTTP Request", 1726 ) 1727 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters")
start_value_option: Optional[RequestOption]
class
DatetimeBasedCursor(pydantic.v1.main.BaseModel):
1730class DatetimeBasedCursor(BaseModel): 1731 type: Literal["DatetimeBasedCursor"] 1732 clamping: Optional[Clamping] = Field( 1733 None, 1734 description="This option is used to adjust the upper and lower boundaries of each datetime window to beginning and end of the provided target period (day, week, month)", 1735 title="Date Range Clamping", 1736 ) 1737 cursor_field: str = Field( 1738 ..., 1739 description="The location of the value on a record that will be used as a bookmark during sync. To ensure no data loss, the API must return records in ascending order based on the cursor field. Nested fields are not supported, so the field must be at the top level of the record. You can use a combination of Add Field and Remove Field transformations to move the nested field to the top.", 1740 examples=["created_at", "{{ config['record_cursor'] }}"], 1741 title="Cursor Field", 1742 ) 1743 datetime_format: str = Field( 1744 ..., 1745 description="The datetime format used to format the datetime values that are sent in outgoing requests to the API. Use placeholders starting with \"%\" to describe the format the API is using. The following placeholders are available:\n * **%s**: Epoch unix timestamp - `1686218963`\n * **%s_as_float**: Epoch unix timestamp in seconds as float with microsecond precision - `1686218963.123456`\n * **%ms**: Epoch unix timestamp (milliseconds) - `1686218963123`\n * **%a**: Weekday (abbreviated) - `Sun`\n * **%A**: Weekday (full) - `Sunday`\n * **%w**: Weekday (decimal) - `0` (Sunday), `6` (Saturday)\n * **%d**: Day of the month (zero-padded) - `01`, `02`, ..., `31`\n * **%b**: Month (abbreviated) - `Jan`\n * **%B**: Month (full) - `January`\n * **%m**: Month (zero-padded) - `01`, `02`, ..., `12`\n * **%y**: Year (without century, zero-padded) - `00`, `01`, ..., `99`\n * **%Y**: Year (with century) - `0001`, `0002`, ..., `9999`\n * **%H**: Hour (24-hour, zero-padded) - `00`, `01`, ..., `23`\n * **%I**: Hour (12-hour, zero-padded) - `01`, `02`, ..., `12`\n * **%p**: AM/PM indicator\n * **%M**: Minute (zero-padded) - `00`, `01`, ..., `59`\n * **%S**: Second (zero-padded) - `00`, `01`, ..., `59`\n * **%f**: Microsecond (zero-padded to 6 digits) - `000000`\n * **%_ms**: Millisecond (zero-padded to 3 digits) - `000`\n * **%z**: UTC offset - `(empty)`, `+0000`, `-04:00`\n * **%Z**: Time zone name - `(empty)`, `UTC`, `GMT`\n * **%j**: Day of the year (zero-padded) - `001`, `002`, ..., `366`\n * **%U**: Week number of the year (starting Sunday) - `00`, ..., `53`\n * **%W**: Week number of the year (starting Monday) - `00`, ..., `53`\n * **%c**: Date and time - `Tue Aug 16 21:30:00 1988`\n * **%x**: Date standard format - `08/16/1988`\n * **%X**: Time standard format - `21:30:00`\n * **%%**: Literal '%' character\n\n Some placeholders depend on the locale of the underlying system - in most cases this locale is configured as en/US. For more information see the [Python documentation](https://docs.python.org/3/library/datetime.html#strftime-and-strptime-format-codes).\n", 1746 examples=["%Y-%m-%dT%H:%M:%S.%f%z", "%Y-%m-%d", "%s", "%ms", "%s_as_float"], 1747 title="Outgoing Datetime Format", 1748 ) 1749 start_datetime: Union[str, MinMaxDatetime] = Field( 1750 ..., 1751 description="The datetime that determines the earliest record that should be synced.", 1752 examples=["2020-01-1T00:00:00Z", "{{ config['start_time'] }}"], 1753 title="Start Datetime", 1754 ) 1755 cursor_datetime_formats: Optional[List[str]] = Field( 1756 None, 1757 description="The possible formats for the cursor field, in order of preference. The first format that matches the cursor field value will be used to parse it. If not provided, the `datetime_format` will be used.", 1758 title="Cursor Datetime Formats", 1759 ) 1760 cursor_granularity: Optional[str] = Field( 1761 None, 1762 description="Smallest increment the datetime_format has (ISO 8601 duration) that is used to ensure the start of a slice does not overlap with the end of the previous one, e.g. for %Y-%m-%d the granularity should be P1D, for %Y-%m-%dT%H:%M:%SZ the granularity should be PT1S. Given this field is provided, `step` needs to be provided as well.", 1763 examples=["PT1S"], 1764 title="Cursor Granularity", 1765 ) 1766 end_datetime: Optional[Union[str, MinMaxDatetime]] = Field( 1767 None, 1768 description="The datetime that determines the last record that should be synced. If not provided, `{{ now_utc() }}` will be used.", 1769 examples=["2021-01-1T00:00:00Z", "{{ now_utc() }}", "{{ day_delta(-1) }}"], 1770 title="End Datetime", 1771 ) 1772 end_time_option: Optional[RequestOption] = Field( 1773 None, 1774 description="Optionally configures how the end datetime will be sent in requests to the source API.", 1775 title="Inject End Time Into Outgoing HTTP Request", 1776 ) 1777 is_data_feed: Optional[bool] = Field( 1778 None, 1779 description="A data feed API is an API that does not allow filtering and paginates the content from the most recent to the least recent. Given this, the CDK needs to know when to stop paginating and this field will generate a stop condition for pagination.", 1780 title="Whether the target API is formatted as a data feed", 1781 ) 1782 is_client_side_incremental: Optional[bool] = Field( 1783 None, 1784 description="If the target API endpoint does not take cursor values to filter records and returns all records anyway, the connector with this cursor will filter out records locally, and only emit new records from the last sync, hence incremental. This means that all records would be read from the API, but only new records will be emitted to the destination.", 1785 title="Whether the target API does not support filtering and returns all data (the cursor filters records in the client instead of the API side)", 1786 ) 1787 is_compare_strictly: Optional[bool] = Field( 1788 False, 1789 description="Set to True if the target API does not accept queries where the start time equal the end time.", 1790 title="Whether to skip requests if the start time equals the end time", 1791 ) 1792 global_substream_cursor: Optional[bool] = Field( 1793 False, 1794 description="This setting optimizes performance when the parent stream has thousands of partitions by storing the cursor as a single value rather than per partition. Notably, the substream state is updated only at the end of the sync, which helps prevent data loss in case of a sync failure. See more info in the [docs](https://docs.airbyte.com/connector-development/config-based/understanding-the-yaml-file/incremental-syncs).", 1795 title="Whether to store cursor as one value instead of per partition", 1796 ) 1797 lookback_window: Optional[str] = Field( 1798 None, 1799 description="Time interval before the start_datetime to read data for, e.g. P1M for looking back one month.", 1800 examples=["P1D", "P{{ config['lookback_days'] }}D"], 1801 title="Lookback Window", 1802 ) 1803 partition_field_end: Optional[str] = Field( 1804 None, 1805 description="Name of the partition start time field.", 1806 examples=["ending_time"], 1807 title="Partition Field End", 1808 ) 1809 partition_field_start: Optional[str] = Field( 1810 None, 1811 description="Name of the partition end time field.", 1812 examples=["starting_time"], 1813 title="Partition Field Start", 1814 ) 1815 start_time_option: Optional[RequestOption] = Field( 1816 None, 1817 description="Optionally configures how the start datetime will be sent in requests to the source API.", 1818 title="Inject Start Time Into Outgoing HTTP Request", 1819 ) 1820 step: Optional[str] = Field( 1821 None, 1822 description="The size of the time window (ISO8601 duration). Given this field is provided, `cursor_granularity` needs to be provided as well.", 1823 examples=["P1W", "{{ config['step_increment'] }}"], 1824 title="Step", 1825 ) 1826 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters")
clamping: Optional[Clamping]
start_datetime: Union[str, MinMaxDatetime]
end_datetime: Union[str, MinMaxDatetime, NoneType]
end_time_option: Optional[RequestOption]
start_time_option: Optional[RequestOption]
class
FixedWindowCallRatePolicy(pydantic.v1.main.BaseModel):
1829class FixedWindowCallRatePolicy(BaseModel): 1830 class Config: 1831 extra = Extra.allow 1832 1833 type: Literal["FixedWindowCallRatePolicy"] 1834 period: str = Field( 1835 ..., description="The time interval for the rate limit window.", title="Period" 1836 ) 1837 call_limit: int = Field( 1838 ..., 1839 description="The maximum number of calls allowed within the period.", 1840 title="Call Limit", 1841 ) 1842 matchers: List[HttpRequestRegexMatcher] = Field( 1843 ..., 1844 description="List of matchers that define which requests this policy applies to.", 1845 title="Matchers", 1846 )
matchers: List[HttpRequestRegexMatcher]
class
FixedWindowCallRatePolicy.Config:
class
MovingWindowCallRatePolicy(pydantic.v1.main.BaseModel):
1849class MovingWindowCallRatePolicy(BaseModel): 1850 class Config: 1851 extra = Extra.allow 1852 1853 type: Literal["MovingWindowCallRatePolicy"] 1854 rates: List[Rate] = Field( 1855 ..., 1856 description="List of rates that define the call limits for different time intervals.", 1857 title="Rates", 1858 ) 1859 matchers: List[HttpRequestRegexMatcher] = Field( 1860 ..., 1861 description="List of matchers that define which requests this policy applies to.", 1862 title="Matchers", 1863 )
rates: List[Rate]
matchers: List[HttpRequestRegexMatcher]
class
MovingWindowCallRatePolicy.Config:
class
UnlimitedCallRatePolicy(pydantic.v1.main.BaseModel):
1866class UnlimitedCallRatePolicy(BaseModel): 1867 class Config: 1868 extra = Extra.allow 1869 1870 type: Literal["UnlimitedCallRatePolicy"] 1871 matchers: List[HttpRequestRegexMatcher] = Field( 1872 ..., 1873 description="List of matchers that define which requests this policy applies to.", 1874 title="Matchers", 1875 )
matchers: List[HttpRequestRegexMatcher]
class
UnlimitedCallRatePolicy.Config:
class
DefaultErrorHandler(pydantic.v1.main.BaseModel):
1878class DefaultErrorHandler(BaseModel): 1879 type: Literal["DefaultErrorHandler"] 1880 backoff_strategies: Optional[ 1881 List[ 1882 Union[ 1883 ConstantBackoffStrategy, 1884 CustomBackoffStrategy, 1885 ExponentialBackoffStrategy, 1886 WaitTimeFromHeader, 1887 WaitUntilTimeFromHeader, 1888 ] 1889 ] 1890 ] = Field( 1891 None, 1892 description="List of backoff strategies to use to determine how long to wait before retrying a retryable request.", 1893 title="Backoff Strategies", 1894 ) 1895 max_retries: Optional[int] = Field( 1896 5, 1897 description="The maximum number of time to retry a retryable request before giving up and failing.", 1898 examples=[5, 0, 10], 1899 title="Max Retry Count", 1900 ) 1901 response_filters: Optional[List[HttpResponseFilter]] = Field( 1902 None, 1903 description="List of response filters to iterate on when deciding how to handle an error. When using an array of multiple filters, the filters will be applied sequentially and the response will be selected if it matches any of the filter's predicate.", 1904 title="Response Filters", 1905 ) 1906 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters")
backoff_strategies: Optional[List[Union[ConstantBackoffStrategy, CustomBackoffStrategy, ExponentialBackoffStrategy, WaitTimeFromHeader, WaitUntilTimeFromHeader]]]
response_filters: Optional[List[HttpResponseFilter]]
class
DefaultPaginator(pydantic.v1.main.BaseModel):
1909class DefaultPaginator(BaseModel): 1910 type: Literal["DefaultPaginator"] 1911 pagination_strategy: Union[ 1912 PageIncrement, OffsetIncrement, CursorPagination, CustomPaginationStrategy 1913 ] = Field( 1914 ..., 1915 description="Strategy defining how records are paginated.", 1916 title="Pagination Strategy", 1917 ) 1918 page_size_option: Optional[RequestOption] = Field( 1919 None, title="Inject Page Size Into Outgoing HTTP Request" 1920 ) 1921 page_token_option: Optional[Union[RequestOption, RequestPath]] = Field( 1922 None, title="Inject Page Token Into Outgoing HTTP Request" 1923 ) 1924 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters")
pagination_strategy: Union[PageIncrement, OffsetIncrement, CursorPagination, CustomPaginationStrategy]
page_size_option: Optional[RequestOption]
page_token_option: Union[RequestOption, RequestPath, NoneType]
class
SessionTokenRequestApiKeyAuthenticator(pydantic.v1.main.BaseModel):
1927class SessionTokenRequestApiKeyAuthenticator(BaseModel): 1928 type: Literal["ApiKey"] 1929 inject_into: RequestOption = Field( 1930 ..., 1931 description="Configure how the API Key will be sent in requests to the source API.", 1932 examples=[ 1933 {"inject_into": "header", "field_name": "Authorization"}, 1934 {"inject_into": "request_parameter", "field_name": "authKey"}, 1935 ], 1936 title="Inject API Key Into Outgoing HTTP Request", 1937 )
inject_into: RequestOption
class
ListPartitionRouter(pydantic.v1.main.BaseModel):
1940class ListPartitionRouter(BaseModel): 1941 type: Literal["ListPartitionRouter"] 1942 cursor_field: str = Field( 1943 ..., 1944 description='While iterating over list values, the name of field used to reference a list value. The partition value can be accessed with string interpolation. e.g. "{{ stream_partition[\'my_key\'] }}" where "my_key" is the value of the cursor_field.', 1945 examples=["section", "{{ config['section_key'] }}"], 1946 title="Current Partition Value Identifier", 1947 ) 1948 values: Union[str, List[str]] = Field( 1949 ..., 1950 description="The list of attributes being iterated over and used as input for the requests made to the source API.", 1951 examples=[["section_a", "section_b", "section_c"], "{{ config['sections'] }}"], 1952 title="Partition Values", 1953 ) 1954 request_option: Optional[RequestOption] = Field( 1955 None, 1956 description="A request option describing where the list value should be injected into and under what field name if applicable.", 1957 title="Inject Partition Value Into Outgoing HTTP Request", 1958 ) 1959 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters")
request_option: Optional[RequestOption]
class
RecordSelector(pydantic.v1.main.BaseModel):
1962class RecordSelector(BaseModel): 1963 type: Literal["RecordSelector"] 1964 extractor: Union[DpathExtractor, CustomRecordExtractor] 1965 record_filter: Optional[Union[RecordFilter, CustomRecordFilter]] = Field( 1966 None, 1967 description="Responsible for filtering records to be emitted by the Source.", 1968 title="Record Filter", 1969 ) 1970 schema_normalization: Optional[Union[SchemaNormalization, CustomSchemaNormalization]] = Field( 1971 None, 1972 description="Responsible for normalization according to the schema.", 1973 title="Schema Normalization", 1974 ) 1975 transform_before_filtering: Optional[bool] = Field( 1976 None, 1977 description="If true, transformation will be applied before record filtering.", 1978 title="Transform Before Filtering", 1979 ) 1980 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters")
extractor: Union[DpathExtractor, CustomRecordExtractor]
record_filter: Union[RecordFilter, CustomRecordFilter, NoneType]
schema_normalization: Union[SchemaNormalization, CustomSchemaNormalization, NoneType]
class
GzipDecoder(pydantic.v1.main.BaseModel):
1983class GzipDecoder(BaseModel): 1984 type: Literal["GzipDecoder"] 1985 decoder: Union[CsvDecoder, GzipDecoder, JsonDecoder, JsonlDecoder]
decoder: Union[CsvDecoder, GzipDecoder, JsonDecoder, JsonlDecoder]
class
RequestBodyGraphQL(pydantic.v1.main.BaseModel):
1988class RequestBodyGraphQL(BaseModel): 1989 type: Literal["RequestBodyGraphQL"] 1990 value: RequestBodyGraphQlQuery
value: RequestBodyGraphQlQuery
class
DpathValidator(pydantic.v1.main.BaseModel):
1993class DpathValidator(BaseModel): 1994 type: Literal["DpathValidator"] 1995 field_path: List[str] = Field( 1996 ..., 1997 description='List of potentially nested fields describing the full path of the field to validate. Use "*" to validate all values from an array.', 1998 examples=[ 1999 ["data"], 2000 ["data", "records"], 2001 ["data", "{{ parameters.name }}"], 2002 ["data", "*", "record"], 2003 ], 2004 title="Field Path", 2005 ) 2006 validation_strategy: ValidateAdheresToSchema = Field( 2007 ..., 2008 description="The condition that the specified config value will be evaluated against", 2009 title="Validation Strategy", 2010 )
validation_strategy: ValidateAdheresToSchema
class
PredicateValidator(pydantic.v1.main.BaseModel):
2013class PredicateValidator(BaseModel): 2014 type: Literal["PredicateValidator"] 2015 value: Optional[Union[str, float, Dict[str, Any], List[Any], bool]] = Field( 2016 ..., 2017 description="The value to be validated. Can be a literal value or interpolated from configuration.", 2018 examples=[ 2019 "test-value", 2020 "{{ config['api_version'] }}", 2021 "{{ config['tenant_id'] }}", 2022 123, 2023 ], 2024 title="Value", 2025 ) 2026 validation_strategy: ValidateAdheresToSchema = Field( 2027 ..., 2028 description="The validation strategy to apply to the value.", 2029 title="Validation Strategy", 2030 )
validation_strategy: ValidateAdheresToSchema
class
ConfigAddFields(pydantic.v1.main.BaseModel):
2033class ConfigAddFields(BaseModel): 2034 type: Literal["ConfigAddFields"] 2035 fields: List[AddedFieldDefinition] = Field( 2036 ..., 2037 description="A list of transformations (path and corresponding value) that will be added to the config.", 2038 title="Fields", 2039 ) 2040 condition: Optional[str] = Field( 2041 "", 2042 description="Fields will be added if expression is evaluated to True.", 2043 examples=[ 2044 "{{ config['environemnt'] == 'sandbox' }}", 2045 "{{ property is integer }}", 2046 "{{ property|length > 5 }}", 2047 "{{ property == 'some_string_to_match' }}", 2048 ], 2049 )
fields: List[AddedFieldDefinition]
class
CompositeErrorHandler(pydantic.v1.main.BaseModel):
2052class CompositeErrorHandler(BaseModel): 2053 type: Literal["CompositeErrorHandler"] 2054 error_handlers: List[Union[CompositeErrorHandler, DefaultErrorHandler]] = Field( 2055 ..., 2056 description="List of error handlers to iterate on to determine how to handle a failed response.", 2057 title="Error Handlers", 2058 ) 2059 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters")
error_handlers: List[Union[CompositeErrorHandler, DefaultErrorHandler]]
class
HTTPAPIBudget(pydantic.v1.main.BaseModel):
2062class HTTPAPIBudget(BaseModel): 2063 class Config: 2064 extra = Extra.allow 2065 2066 type: Literal["HTTPAPIBudget"] 2067 policies: List[ 2068 Union[ 2069 FixedWindowCallRatePolicy, 2070 MovingWindowCallRatePolicy, 2071 UnlimitedCallRatePolicy, 2072 ] 2073 ] = Field( 2074 ..., 2075 description="List of call rate policies that define how many calls are allowed.", 2076 title="Policies", 2077 ) 2078 ratelimit_reset_header: Optional[str] = Field( 2079 "ratelimit-reset", 2080 description="The HTTP response header name that indicates when the rate limit resets.", 2081 title="Rate Limit Reset Header", 2082 ) 2083 ratelimit_remaining_header: Optional[str] = Field( 2084 "ratelimit-remaining", 2085 description="The HTTP response header name that indicates the number of remaining allowed calls.", 2086 title="Rate Limit Remaining Header", 2087 ) 2088 status_codes_for_ratelimit_hit: Optional[List[int]] = Field( 2089 [429], 2090 description="List of HTTP status codes that indicate a rate limit has been hit.", 2091 title="Status Codes for Rate Limit Hit", 2092 )
policies: List[Union[FixedWindowCallRatePolicy, MovingWindowCallRatePolicy, UnlimitedCallRatePolicy]]
class
HTTPAPIBudget.Config:
class
ZipfileDecoder(pydantic.v1.main.BaseModel):
2095class ZipfileDecoder(BaseModel): 2096 class Config: 2097 extra = Extra.allow 2098 2099 type: Literal["ZipfileDecoder"] 2100 decoder: Union[CsvDecoder, GzipDecoder, JsonDecoder, JsonlDecoder] = Field( 2101 ..., 2102 description="Parser to parse the decompressed data from the zipfile(s).", 2103 title="Parser", 2104 )
decoder: Union[CsvDecoder, GzipDecoder, JsonDecoder, JsonlDecoder]
class
ZipfileDecoder.Config:
class
ConfigMigration(pydantic.v1.main.BaseModel):
2107class ConfigMigration(BaseModel): 2108 type: Literal["ConfigMigration"] 2109 description: Optional[str] = Field( 2110 None, description="The description/purpose of the config migration." 2111 ) 2112 transformations: List[Union[ConfigRemapField, ConfigAddFields, ConfigRemoveFields]] = Field( 2113 ..., 2114 description="The list of transformations that will attempt to be applied on an incoming unmigrated config. The transformations will be applied in the order they are defined.", 2115 title="Transformations", 2116 )
transformations: List[Union[ConfigRemapField, ConfigAddFields, ConfigRemoveFields]]
class
ConfigNormalizationRules(pydantic.v1.main.BaseModel):
2119class ConfigNormalizationRules(BaseModel): 2120 class Config: 2121 extra = Extra.forbid 2122 2123 config_migrations: Optional[List[ConfigMigration]] = Field( 2124 [], 2125 description="The discrete migrations that will be applied on the incoming config. Each migration will be applied in the order they are defined.", 2126 title="Config Migrations", 2127 ) 2128 transformations: Optional[ 2129 List[Union[ConfigRemapField, ConfigAddFields, ConfigRemoveFields]] 2130 ] = Field( 2131 [], 2132 description="The list of transformations that will be applied on the incoming config at the start of each sync. The transformations will be applied in the order they are defined.", 2133 title="Transformations", 2134 ) 2135 validations: Optional[List[Union[DpathValidator, PredicateValidator]]] = Field( 2136 [], 2137 description="The list of validations that will be performed on the incoming config at the start of each sync.", 2138 title="Validations", 2139 )
config_migrations: Optional[List[ConfigMigration]]
transformations: Optional[List[Union[ConfigRemapField, ConfigAddFields, ConfigRemoveFields]]]
validations: Optional[List[Union[DpathValidator, PredicateValidator]]]
class
ConfigNormalizationRules.Config:
class
Spec(pydantic.v1.main.BaseModel):
2142class Spec(BaseModel): 2143 type: Literal["Spec"] 2144 connection_specification: Dict[str, Any] = Field( 2145 ..., 2146 description="A connection specification describing how a the connector can be configured.", 2147 title="Connection Specification", 2148 ) 2149 documentation_url: Optional[str] = Field( 2150 None, 2151 description="URL of the connector's documentation page.", 2152 examples=["https://docs.airbyte.com/integrations/sources/dremio"], 2153 title="Documentation URL", 2154 ) 2155 advanced_auth: Optional[AuthFlow] = Field( 2156 None, 2157 description="Advanced specification for configuring the authentication flow.", 2158 title="Advanced Auth", 2159 ) 2160 config_normalization_rules: Optional[ConfigNormalizationRules] = Field( 2161 None, title="Config Normalization Rules" 2162 )
advanced_auth: Optional[AuthFlow]
config_normalization_rules: Optional[ConfigNormalizationRules]
class
DeclarativeSource1(pydantic.v1.main.BaseModel):
2165class DeclarativeSource1(BaseModel): 2166 class Config: 2167 extra = Extra.forbid 2168 2169 type: Literal["DeclarativeSource"] 2170 check: Union[CheckStream, CheckDynamicStream] 2171 streams: List[Union[DeclarativeStream, StateDelegatingStream]] 2172 dynamic_streams: Optional[List[DynamicDeclarativeStream]] = None 2173 version: str = Field( 2174 ..., 2175 description="The version of the Airbyte CDK used to build and test the source.", 2176 ) 2177 schemas: Optional[Schemas] = None 2178 definitions: Optional[Dict[str, Any]] = None 2179 spec: Optional[Spec] = None 2180 concurrency_level: Optional[ConcurrencyLevel] = None 2181 api_budget: Optional[HTTPAPIBudget] = None 2182 max_concurrent_async_job_count: Optional[Union[int, str]] = Field( 2183 None, 2184 description="Maximum number of concurrent asynchronous jobs to run. This property is only relevant for sources/streams that support asynchronous job execution through the AsyncRetriever (e.g. a report-based stream that initiates a job, polls the job status, and then fetches the job results). This is often set by the API's maximum number of concurrent jobs on the account level. Refer to the API's documentation for this information.", 2185 examples=[3, "{{ config['max_concurrent_async_job_count'] }}"], 2186 title="Maximum Concurrent Asynchronous Jobs", 2187 ) 2188 metadata: Optional[Dict[str, Any]] = Field( 2189 None, 2190 description="For internal Airbyte use only - DO NOT modify manually. Used by consumers of declarative manifests for storing related metadata.", 2191 ) 2192 description: Optional[str] = Field( 2193 None, 2194 description="A description of the connector. It will be presented on the Source documentation page.", 2195 )
check: Union[CheckStream, CheckDynamicStream]
streams: List[Union[DeclarativeStream, StateDelegatingStream]]
dynamic_streams: Optional[List[DynamicDeclarativeStream]]
schemas: Optional[Schemas]
spec: Optional[Spec]
concurrency_level: Optional[ConcurrencyLevel]
api_budget: Optional[HTTPAPIBudget]
class
DeclarativeSource1.Config:
class
DeclarativeSource2(pydantic.v1.main.BaseModel):
2198class DeclarativeSource2(BaseModel): 2199 class Config: 2200 extra = Extra.forbid 2201 2202 type: Literal["DeclarativeSource"] 2203 check: Union[CheckStream, CheckDynamicStream] 2204 streams: Optional[List[Union[DeclarativeStream, StateDelegatingStream]]] = None 2205 dynamic_streams: List[DynamicDeclarativeStream] 2206 version: str = Field( 2207 ..., 2208 description="The version of the Airbyte CDK used to build and test the source.", 2209 ) 2210 schemas: Optional[Schemas] = None 2211 definitions: Optional[Dict[str, Any]] = None 2212 spec: Optional[Spec] = None 2213 concurrency_level: Optional[ConcurrencyLevel] = None 2214 api_budget: Optional[HTTPAPIBudget] = None 2215 max_concurrent_async_job_count: Optional[Union[int, str]] = Field( 2216 None, 2217 description="Maximum number of concurrent asynchronous jobs to run. This property is only relevant for sources/streams that support asynchronous job execution through the AsyncRetriever (e.g. a report-based stream that initiates a job, polls the job status, and then fetches the job results). This is often set by the API's maximum number of concurrent jobs on the account level. Refer to the API's documentation for this information.", 2218 examples=[3, "{{ config['max_concurrent_async_job_count'] }}"], 2219 title="Maximum Concurrent Asynchronous Jobs", 2220 ) 2221 metadata: Optional[Dict[str, Any]] = Field( 2222 None, 2223 description="For internal Airbyte use only - DO NOT modify manually. Used by consumers of declarative manifests for storing related metadata.", 2224 ) 2225 description: Optional[str] = Field( 2226 None, 2227 description="A description of the connector. It will be presented on the Source documentation page.", 2228 )
check: Union[CheckStream, CheckDynamicStream]
streams: Optional[List[Union[DeclarativeStream, StateDelegatingStream]]]
dynamic_streams: List[DynamicDeclarativeStream]
schemas: Optional[Schemas]
spec: Optional[Spec]
concurrency_level: Optional[ConcurrencyLevel]
api_budget: Optional[HTTPAPIBudget]
class
DeclarativeSource2.Config:
class
DeclarativeSource(pydantic.v1.main.BaseModel):
2231class DeclarativeSource(BaseModel): 2232 class Config: 2233 extra = Extra.forbid 2234 2235 __root__: Union[DeclarativeSource1, DeclarativeSource2] = Field( 2236 ..., 2237 description="An API source that extracts data according to its declarative components.", 2238 title="DeclarativeSource", 2239 )
class
DeclarativeSource.Config:
class
SelectiveAuthenticator(pydantic.v1.main.BaseModel):
2242class SelectiveAuthenticator(BaseModel): 2243 class Config: 2244 extra = Extra.allow 2245 2246 type: Literal["SelectiveAuthenticator"] 2247 authenticator_selection_path: List[str] = Field( 2248 ..., 2249 description="Path of the field in config with selected authenticator name", 2250 examples=[["auth"], ["auth", "type"]], 2251 title="Authenticator Selection Path", 2252 ) 2253 authenticators: Dict[ 2254 str, 2255 Union[ 2256 ApiKeyAuthenticator, 2257 BasicHttpAuthenticator, 2258 BearerAuthenticator, 2259 CustomAuthenticator, 2260 OAuthAuthenticator, 2261 JwtAuthenticator, 2262 SessionTokenAuthenticator, 2263 NoAuth, 2264 LegacySessionTokenAuthenticator, 2265 ], 2266 ] = Field( 2267 ..., 2268 description="Authenticators to select from.", 2269 examples=[ 2270 { 2271 "authenticators": { 2272 "token": "#/definitions/ApiKeyAuthenticator", 2273 "oauth": "#/definitions/OAuthAuthenticator", 2274 "jwt": "#/definitions/JwtAuthenticator", 2275 } 2276 } 2277 ], 2278 title="Authenticators", 2279 ) 2280 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters")
authenticators: Dict[str, Union[ApiKeyAuthenticator, BasicHttpAuthenticator, BearerAuthenticator, CustomAuthenticator, OAuthAuthenticator, JwtAuthenticator, SessionTokenAuthenticator, NoAuth, LegacySessionTokenAuthenticator]]
class
SelectiveAuthenticator.Config:
class
FileUploader(pydantic.v1.main.BaseModel):
2283class FileUploader(BaseModel): 2284 type: Literal["FileUploader"] 2285 requester: Union[HttpRequester, CustomRequester] = Field( 2286 ..., 2287 description="Requester component that describes how to prepare HTTP requests to send to the source API.", 2288 ) 2289 download_target_extractor: Union[DpathExtractor, CustomRecordExtractor] = Field( 2290 ..., 2291 description="Responsible for fetching the url where the file is located. This is applied on each records and not on the HTTP response", 2292 ) 2293 file_extractor: Optional[Union[DpathExtractor, CustomRecordExtractor]] = Field( 2294 None, 2295 description="Responsible for fetching the content of the file. If not defined, the assumption is that the whole response body is the file content", 2296 ) 2297 filename_extractor: Optional[str] = Field( 2298 None, 2299 description="Defines the name to store the file. Stream name is automatically added to the file path. File unique ID can be used to avoid overwriting files. Random UUID will be used if the extractor is not provided.", 2300 examples=[ 2301 "{{ record.id }}/{{ record.file_name }}/", 2302 "{{ record.id }}_{{ record.file_name }}/", 2303 ], 2304 ) 2305 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters")
requester: Union[HttpRequester, CustomRequester]
download_target_extractor: Union[DpathExtractor, CustomRecordExtractor]
file_extractor: Union[DpathExtractor, CustomRecordExtractor, NoneType]
class
DeclarativeStream(pydantic.v1.main.BaseModel):
2308class DeclarativeStream(BaseModel): 2309 class Config: 2310 extra = Extra.allow 2311 2312 type: Literal["DeclarativeStream"] 2313 name: Optional[str] = Field("", description="The stream name.", example=["Users"], title="Name") 2314 retriever: Union[SimpleRetriever, AsyncRetriever, CustomRetriever] = Field( 2315 ..., 2316 description="Component used to coordinate how records are extracted across stream slices and request pages.", 2317 title="Retriever", 2318 ) 2319 incremental_sync: Optional[ 2320 Union[DatetimeBasedCursor, IncrementingCountCursor, CustomIncrementalSync] 2321 ] = Field( 2322 None, 2323 description="Component used to fetch data incrementally based on a time field in the data.", 2324 title="Incremental Sync", 2325 ) 2326 primary_key: Optional[PrimaryKey] = Field("", title="Primary Key") 2327 schema_loader: Optional[ 2328 Union[ 2329 InlineSchemaLoader, 2330 DynamicSchemaLoader, 2331 JsonFileSchemaLoader, 2332 CustomSchemaLoader, 2333 List[ 2334 Union[ 2335 InlineSchemaLoader, 2336 DynamicSchemaLoader, 2337 JsonFileSchemaLoader, 2338 CustomSchemaLoader, 2339 ] 2340 ], 2341 ] 2342 ] = Field( 2343 None, 2344 description="One or many schema loaders can be used to retrieve the schema for the current stream. When multiple schema loaders are defined, schema properties will be merged together. Schema loaders defined first taking precedence in the event of a conflict.", 2345 title="Schema Loader", 2346 ) 2347 transformations: Optional[ 2348 List[ 2349 Union[ 2350 AddFields, 2351 CustomTransformation, 2352 RemoveFields, 2353 KeysToLower, 2354 KeysToSnakeCase, 2355 FlattenFields, 2356 DpathFlattenFields, 2357 KeysReplace, 2358 ] 2359 ] 2360 ] = Field( 2361 None, 2362 description="A list of transformations to be applied to each output record.", 2363 title="Transformations", 2364 ) 2365 state_migrations: Optional[ 2366 List[Union[LegacyToPerPartitionStateMigration, CustomStateMigration]] 2367 ] = Field( 2368 [], 2369 description="Array of state migrations to be applied on the input state", 2370 title="State Migrations", 2371 ) 2372 file_uploader: Optional[FileUploader] = Field( 2373 None, 2374 description="(experimental) Describes how to fetch a file", 2375 title="File Uploader", 2376 ) 2377 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters")
retriever: Union[SimpleRetriever, AsyncRetriever, CustomRetriever]
incremental_sync: Union[DatetimeBasedCursor, IncrementingCountCursor, CustomIncrementalSync, NoneType]
primary_key: Optional[PrimaryKey]
schema_loader: Union[InlineSchemaLoader, DynamicSchemaLoader, JsonFileSchemaLoader, CustomSchemaLoader, List[Union[InlineSchemaLoader, DynamicSchemaLoader, JsonFileSchemaLoader, CustomSchemaLoader]], NoneType]
transformations: Optional[List[Union[AddFields, CustomTransformation, RemoveFields, KeysToLower, KeysToSnakeCase, FlattenFields, DpathFlattenFields, KeysReplace]]]
state_migrations: Optional[List[Union[LegacyToPerPartitionStateMigration, CustomStateMigration]]]
file_uploader: Optional[FileUploader]
class
DeclarativeStream.Config:
class
SessionTokenAuthenticator(pydantic.v1.main.BaseModel):
2380class SessionTokenAuthenticator(BaseModel): 2381 type: Literal["SessionTokenAuthenticator"] 2382 login_requester: HttpRequester = Field( 2383 ..., 2384 description="Description of the request to perform to obtain a session token to perform data requests. The response body is expected to be a JSON object with a session token property.", 2385 examples=[ 2386 { 2387 "type": "HttpRequester", 2388 "url_base": "https://my_api.com", 2389 "path": "/login", 2390 "authenticator": { 2391 "type": "BasicHttpAuthenticator", 2392 "username": "{{ config.username }}", 2393 "password": "{{ config.password }}", 2394 }, 2395 } 2396 ], 2397 title="Login Requester", 2398 ) 2399 session_token_path: List[str] = Field( 2400 ..., 2401 description="The path in the response body returned from the login requester to the session token.", 2402 examples=[["access_token"], ["result", "token"]], 2403 title="Session Token Path", 2404 ) 2405 expiration_duration: Optional[str] = Field( 2406 None, 2407 description="The duration in ISO 8601 duration notation after which the session token expires, starting from the time it was obtained. Omitting it will result in the session token being refreshed for every request.", 2408 examples=["PT1H", "P1D"], 2409 title="Expiration Duration", 2410 ) 2411 request_authentication: Union[ 2412 SessionTokenRequestApiKeyAuthenticator, SessionTokenRequestBearerAuthenticator 2413 ] = Field( 2414 ..., 2415 description="Authentication method to use for requests sent to the API, specifying how to inject the session token.", 2416 title="Data Request Authentication", 2417 ) 2418 decoder: Optional[Union[JsonDecoder, XmlDecoder]] = Field( 2419 None, description="Component used to decode the response.", title="Decoder" 2420 ) 2421 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters")
login_requester: HttpRequester
request_authentication: Union[SessionTokenRequestApiKeyAuthenticator, SessionTokenRequestBearerAuthenticator]
decoder: Union[JsonDecoder, XmlDecoder, NoneType]
class
HttpRequester(airbyte_cdk.sources.declarative.models.base_model_with_deprecations.BaseModelWithDeprecations):
2424class HttpRequester(BaseModelWithDeprecations): 2425 type: Literal["HttpRequester"] 2426 url_base: Optional[str] = Field( 2427 None, 2428 deprecated=True, 2429 deprecation_message="Use `url` field instead.", 2430 description="Deprecated, use the `url` instead. Base URL of the API source. Do not put sensitive information (e.g. API tokens) into this field - Use the Authenticator component for this.", 2431 examples=[ 2432 "https://connect.squareup.com/v2", 2433 "{{ config['base_url'] or 'https://app.posthog.com'}}/api", 2434 "https://connect.squareup.com/v2/quotes/{{ stream_partition['id'] }}/quote_line_groups", 2435 "https://example.com/api/v1/resource/{{ next_page_token['id'] }}", 2436 ], 2437 title="API Base URL", 2438 ) 2439 url: Optional[str] = Field( 2440 None, 2441 description="The URL of the source API endpoint. Do not put sensitive information (e.g. API tokens) into this field - Use the Authenticator component for this.", 2442 examples=[ 2443 "https://connect.squareup.com/v2", 2444 "{{ config['url'] or 'https://app.posthog.com'}}/api", 2445 "https://connect.squareup.com/v2/quotes/{{ stream_partition['id'] }}/quote_line_groups", 2446 "https://example.com/api/v1/resource/{{ next_page_token['id'] }}", 2447 ], 2448 title="API Endpoint URL", 2449 ) 2450 path: Optional[str] = Field( 2451 None, 2452 deprecated=True, 2453 deprecation_message="Use `url` field instead.", 2454 description="Deprecated, use the `url` instead. Path the specific API endpoint that this stream represents. Do not put sensitive information (e.g. API tokens) into this field - Use the Authenticator component for this.", 2455 examples=[ 2456 "/products", 2457 "/quotes/{{ stream_partition['id'] }}/quote_line_groups", 2458 "/trades/{{ config['symbol_id'] }}/history", 2459 ], 2460 title="URL Path", 2461 ) 2462 http_method: Optional[HttpMethod] = Field( 2463 HttpMethod.GET, 2464 description="The HTTP method used to fetch data from the source (can be GET or POST).", 2465 examples=["GET", "POST"], 2466 title="HTTP Method", 2467 ) 2468 authenticator: Optional[ 2469 Union[ 2470 ApiKeyAuthenticator, 2471 BasicHttpAuthenticator, 2472 BearerAuthenticator, 2473 OAuthAuthenticator, 2474 JwtAuthenticator, 2475 SessionTokenAuthenticator, 2476 SelectiveAuthenticator, 2477 CustomAuthenticator, 2478 NoAuth, 2479 LegacySessionTokenAuthenticator, 2480 ] 2481 ] = Field( 2482 None, 2483 description="Authentication method to use for requests sent to the API.", 2484 title="Authenticator", 2485 ) 2486 fetch_properties_from_endpoint: Optional[PropertiesFromEndpoint] = Field( 2487 None, 2488 description="Allows for retrieving a dynamic set of properties from an API endpoint which can be injected into outbound request using the stream_partition.extra_fields.", 2489 title="Fetch Properties from Endpoint", 2490 ) 2491 request_parameters: Optional[Union[Dict[str, Union[str, QueryProperties]], str]] = Field( 2492 None, 2493 description="Specifies the query parameters that should be set on an outgoing HTTP request given the inputs.", 2494 examples=[ 2495 {"unit": "day"}, 2496 { 2497 "query": 'last_event_time BETWEEN TIMESTAMP "{{ stream_interval.start_time }}" AND TIMESTAMP "{{ stream_interval.end_time }}"' 2498 }, 2499 {"searchIn": "{{ ','.join(config.get('search_in', [])) }}"}, 2500 {"sort_by[asc]": "updated_at"}, 2501 ], 2502 title="Query Parameters", 2503 ) 2504 request_headers: Optional[Union[Dict[str, str], str]] = Field( 2505 None, 2506 description="Return any non-auth headers. Authentication headers will overwrite any overlapping headers returned from this method.", 2507 examples=[{"Output-Format": "JSON"}, {"Version": "{{ config['version'] }}"}], 2508 title="Request Headers", 2509 ) 2510 request_body_data: Optional[Union[Dict[str, str], str]] = Field( 2511 None, 2512 deprecated=True, 2513 deprecation_message="Use `request_body` field instead.", 2514 description="Specifies how to populate the body of the request with a non-JSON payload. Plain text will be sent as is, whereas objects will be converted to a urlencoded form.", 2515 examples=[ 2516 '[{"clause": {"type": "timestamp", "operator": 10, "parameters":\n [{"value": {{ stream_interval[\'start_time\'] | int * 1000 }} }]\n }, "orderBy": 1, "columnName": "Timestamp"}]/\n' 2517 ], 2518 title="Request Body Payload (Non-JSON)", 2519 ) 2520 request_body_json: Optional[Union[Dict[str, Any], str]] = Field( 2521 None, 2522 deprecated=True, 2523 deprecation_message="Use `request_body` field instead.", 2524 description="Specifies how to populate the body of the request with a JSON payload. Can contain nested objects.", 2525 examples=[ 2526 {"sort_order": "ASC", "sort_field": "CREATED_AT"}, 2527 {"key": "{{ config['value'] }}"}, 2528 {"sort": {"field": "updated_at", "order": "ascending"}}, 2529 ], 2530 title="Request Body JSON Payload", 2531 ) 2532 request_body: Optional[ 2533 Union[ 2534 RequestBodyPlainText, 2535 RequestBodyUrlEncodedForm, 2536 RequestBodyJsonObject, 2537 RequestBodyGraphQL, 2538 ] 2539 ] = Field( 2540 None, 2541 description="Specifies how to populate the body of the request with a payload. Can contain nested objects.", 2542 examples=[ 2543 { 2544 "type": "RequestBodyJsonObject", 2545 "value": {"sort_order": "ASC", "sort_field": "CREATED_AT"}, 2546 }, 2547 { 2548 "type": "RequestBodyJsonObject", 2549 "value": {"key": "{{ config['value'] }}"}, 2550 }, 2551 { 2552 "type": "RequestBodyJsonObject", 2553 "value": {"sort": {"field": "updated_at", "order": "ascending"}}, 2554 }, 2555 {"type": "RequestBodyPlainText", "value": "plain_text_body"}, 2556 { 2557 "type": "RequestBodyUrlEncodedForm", 2558 "value": {"param1": "value1", "param2": "{{ config['param2_value'] }}"}, 2559 }, 2560 { 2561 "type": "RequestBodyGraphQL", 2562 "value": { 2563 "query": { 2564 "param1": "value1", 2565 "param2": "{{ config['param2_value'] }}", 2566 } 2567 }, 2568 }, 2569 ], 2570 title="Request Body", 2571 ) 2572 error_handler: Optional[ 2573 Union[DefaultErrorHandler, CompositeErrorHandler, CustomErrorHandler] 2574 ] = Field( 2575 None, 2576 description="Error handler component that defines how to handle errors.", 2577 title="Error Handler", 2578 ) 2579 use_cache: Optional[bool] = Field( 2580 False, 2581 description="Enables stream requests caching. This field is automatically set by the CDK.", 2582 title="Use Cache", 2583 ) 2584 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters")
Pydantic BaseModel that warns when deprecated fields are accessed. The deprecation message is stored in the field's extra attributes. This class is used to create models that can have deprecated fields and show warnings when those fields are accessed or initialized.
The _deprecation_logs attribute is stored in the model itself.
The collected deprecation warnings are further propagated to the Airbyte log messages,
during the component creation process, in model_to_component._collect_model_deprecations().
The component implementation is not responsible for handling the deprecation warnings, since the deprecation warnings are already handled in the model itself.
http_method: Optional[HttpMethod]
authenticator: Union[ApiKeyAuthenticator, BasicHttpAuthenticator, BearerAuthenticator, OAuthAuthenticator, JwtAuthenticator, SessionTokenAuthenticator, SelectiveAuthenticator, CustomAuthenticator, NoAuth, LegacySessionTokenAuthenticator, NoneType]
fetch_properties_from_endpoint: Optional[PropertiesFromEndpoint]
request_parameters: Union[Dict[str, Union[str, QueryProperties]], str, NoneType]
request_body: Union[RequestBodyPlainText, RequestBodyUrlEncodedForm, RequestBodyJsonObject, RequestBodyGraphQL, NoneType]
error_handler: Union[DefaultErrorHandler, CompositeErrorHandler, CustomErrorHandler, NoneType]
class
DynamicSchemaLoader(pydantic.v1.main.BaseModel):
2587class DynamicSchemaLoader(BaseModel): 2588 type: Literal["DynamicSchemaLoader"] 2589 retriever: Union[AsyncRetriever, CustomRetriever, SimpleRetriever] = Field( 2590 ..., 2591 description="Component used to coordinate how records are extracted across stream slices and request pages.", 2592 title="Retriever", 2593 ) 2594 schema_filter: Optional[Union[RecordFilter, CustomRecordFilter]] = Field( 2595 None, 2596 description="Responsible for filtering fields to be added to json schema.", 2597 title="Schema Filter", 2598 ) 2599 schema_transformations: Optional[ 2600 List[ 2601 Union[ 2602 AddFields, 2603 CustomTransformation, 2604 RemoveFields, 2605 KeysToLower, 2606 KeysToSnakeCase, 2607 FlattenFields, 2608 DpathFlattenFields, 2609 KeysReplace, 2610 ] 2611 ] 2612 ] = Field( 2613 None, 2614 description="A list of transformations to be applied to the schema.", 2615 title="Schema Transformations", 2616 ) 2617 schema_type_identifier: SchemaTypeIdentifier 2618 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters")
retriever: Union[AsyncRetriever, CustomRetriever, SimpleRetriever]
schema_filter: Union[RecordFilter, CustomRecordFilter, NoneType]
schema_transformations: Optional[List[Union[AddFields, CustomTransformation, RemoveFields, KeysToLower, KeysToSnakeCase, FlattenFields, DpathFlattenFields, KeysReplace]]]
schema_type_identifier: SchemaTypeIdentifier
class
ParentStreamConfig(pydantic.v1.main.BaseModel):
2621class ParentStreamConfig(BaseModel): 2622 type: Literal["ParentStreamConfig"] 2623 lazy_read_pointer: Optional[List[str]] = Field( 2624 [], 2625 description="If set, this will enable lazy reading, using the initial read of parent records to extract child records.", 2626 title="Lazy Read Pointer", 2627 ) 2628 parent_key: str = Field( 2629 ..., 2630 description="The primary key of records from the parent stream that will be used during the retrieval of records for the current substream. This parent identifier field is typically a characteristic of the child records being extracted from the source API.", 2631 examples=["id", "{{ config['parent_record_id'] }}"], 2632 title="Parent Key", 2633 ) 2634 stream: Union[DeclarativeStream, StateDelegatingStream] = Field( 2635 ..., description="Reference to the parent stream.", title="Parent Stream" 2636 ) 2637 partition_field: str = Field( 2638 ..., 2639 description="While iterating over parent records during a sync, the parent_key value can be referenced by using this field.", 2640 examples=["parent_id", "{{ config['parent_partition_field'] }}"], 2641 title="Current Parent Key Value Identifier", 2642 ) 2643 request_option: Optional[RequestOption] = Field( 2644 None, 2645 description="A request option describing where the parent key value should be injected into and under what field name if applicable.", 2646 title="Request Option", 2647 ) 2648 incremental_dependency: Optional[bool] = Field( 2649 False, 2650 description="Indicates whether the parent stream should be read incrementally based on updates in the child stream.", 2651 title="Incremental Dependency", 2652 ) 2653 extra_fields: Optional[List[List[str]]] = Field( 2654 None, 2655 description="Array of field paths to include as additional fields in the stream slice. Each path is an array of strings representing keys to access fields in the respective parent record. Accessible via `stream_slice.extra_fields`. Missing fields are set to `None`.", 2656 title="Extra Fields", 2657 ) 2658 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters")
stream: Union[DeclarativeStream, StateDelegatingStream]
request_option: Optional[RequestOption]
class
PropertiesFromEndpoint(pydantic.v1.main.BaseModel):
2661class PropertiesFromEndpoint(BaseModel): 2662 type: Literal["PropertiesFromEndpoint"] 2663 property_field_path: List[str] = Field( 2664 ..., 2665 description="Describes the path to the field that should be extracted", 2666 examples=[["name"]], 2667 ) 2668 retriever: Union[SimpleRetriever, CustomRetriever] = Field( 2669 ..., 2670 description="Requester component that describes how to fetch the properties to query from a remote API endpoint.", 2671 ) 2672 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters")
retriever: Union[SimpleRetriever, CustomRetriever]
class
QueryProperties(pydantic.v1.main.BaseModel):
2675class QueryProperties(BaseModel): 2676 type: Literal["QueryProperties"] 2677 property_list: Union[List[str], PropertiesFromEndpoint] = Field( 2678 ..., 2679 description="The set of properties that will be queried for in the outbound request. This can either be statically defined or dynamic based on an API endpoint", 2680 title="Property List", 2681 ) 2682 always_include_properties: Optional[List[str]] = Field( 2683 None, 2684 description="The list of properties that should be included in every set of properties when multiple chunks of properties are being requested.", 2685 title="Always Include Properties", 2686 ) 2687 property_chunking: Optional[PropertyChunking] = Field( 2688 None, 2689 description="Defines how query properties will be grouped into smaller sets for APIs with limitations on the number of properties fetched per API request.", 2690 title="Property Chunking", 2691 ) 2692 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters")
property_list: Union[List[str], PropertiesFromEndpoint]
property_chunking: Optional[PropertyChunking]
class
StateDelegatingStream(pydantic.v1.main.BaseModel):
2695class StateDelegatingStream(BaseModel): 2696 type: Literal["StateDelegatingStream"] 2697 name: str = Field(..., description="The stream name.", example=["Users"], title="Name") 2698 full_refresh_stream: DeclarativeStream = Field( 2699 ..., 2700 description="Component used to coordinate how records are extracted across stream slices and request pages when the state is empty or not provided.", 2701 title="Full Refresh Stream", 2702 ) 2703 incremental_stream: DeclarativeStream = Field( 2704 ..., 2705 description="Component used to coordinate how records are extracted across stream slices and request pages when the state provided.", 2706 title="Incremental Stream", 2707 ) 2708 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters")
full_refresh_stream: DeclarativeStream
incremental_stream: DeclarativeStream
class
SimpleRetriever(pydantic.v1.main.BaseModel):
2711class SimpleRetriever(BaseModel): 2712 type: Literal["SimpleRetriever"] 2713 requester: Union[HttpRequester, CustomRequester] = Field( 2714 ..., 2715 description="Requester component that describes how to prepare HTTP requests to send to the source API.", 2716 ) 2717 decoder: Optional[ 2718 Union[ 2719 JsonDecoder, 2720 XmlDecoder, 2721 CsvDecoder, 2722 JsonlDecoder, 2723 GzipDecoder, 2724 IterableDecoder, 2725 ZipfileDecoder, 2726 CustomDecoder, 2727 ] 2728 ] = Field( 2729 None, 2730 description="Component decoding the response so records can be extracted.", 2731 title="HTTP Response Format", 2732 ) 2733 record_selector: RecordSelector = Field( 2734 ..., 2735 description="Component that describes how to extract records from a HTTP response.", 2736 ) 2737 paginator: Optional[Union[DefaultPaginator, NoPagination]] = Field( 2738 None, 2739 description="Paginator component that describes how to navigate through the API's pages.", 2740 ) 2741 ignore_stream_slicer_parameters_on_paginated_requests: Optional[bool] = Field( 2742 False, 2743 description="If true, the partition router and incremental request options will be ignored when paginating requests. Request options set directly on the requester will not be ignored.", 2744 ) 2745 partition_router: Optional[ 2746 Union[ 2747 ListPartitionRouter, 2748 SubstreamPartitionRouter, 2749 GroupingPartitionRouter, 2750 CustomPartitionRouter, 2751 List[ 2752 Union[ 2753 ListPartitionRouter, 2754 SubstreamPartitionRouter, 2755 GroupingPartitionRouter, 2756 CustomPartitionRouter, 2757 ] 2758 ], 2759 ] 2760 ] = Field( 2761 [], 2762 description="PartitionRouter component that describes how to partition the stream, enabling incremental syncs and checkpointing.", 2763 title="Partition Router", 2764 ) 2765 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters")
requester: Union[HttpRequester, CustomRequester]
decoder: Union[JsonDecoder, XmlDecoder, CsvDecoder, JsonlDecoder, GzipDecoder, IterableDecoder, ZipfileDecoder, CustomDecoder, NoneType]
record_selector: RecordSelector
paginator: Union[DefaultPaginator, NoPagination, NoneType]
partition_router: Union[ListPartitionRouter, SubstreamPartitionRouter, GroupingPartitionRouter, CustomPartitionRouter, List[Union[ListPartitionRouter, SubstreamPartitionRouter, GroupingPartitionRouter, CustomPartitionRouter]], NoneType]
class
AsyncRetriever(pydantic.v1.main.BaseModel):
2768class AsyncRetriever(BaseModel): 2769 type: Literal["AsyncRetriever"] 2770 record_selector: RecordSelector = Field( 2771 ..., 2772 description="Component that describes how to extract records from a HTTP response.", 2773 ) 2774 status_mapping: AsyncJobStatusMap = Field( 2775 ..., description="Async Job Status to Airbyte CDK Async Job Status mapping." 2776 ) 2777 status_extractor: Union[DpathExtractor, CustomRecordExtractor] = Field( 2778 ..., description="Responsible for fetching the actual status of the async job." 2779 ) 2780 download_target_extractor: Union[DpathExtractor, CustomRecordExtractor] = Field( 2781 ..., 2782 description="Responsible for fetching the final result `urls` provided by the completed / finished / ready async job.", 2783 ) 2784 download_extractor: Optional[ 2785 Union[DpathExtractor, CustomRecordExtractor, ResponseToFileExtractor] 2786 ] = Field(None, description="Responsible for fetching the records from provided urls.") 2787 creation_requester: Union[HttpRequester, CustomRequester] = Field( 2788 ..., 2789 description="Requester component that describes how to prepare HTTP requests to send to the source API to create the async server-side job.", 2790 ) 2791 polling_requester: Union[HttpRequester, CustomRequester] = Field( 2792 ..., 2793 description="Requester component that describes how to prepare HTTP requests to send to the source API to fetch the status of the running async job.", 2794 ) 2795 polling_job_timeout: Optional[Union[int, str]] = Field( 2796 None, 2797 description="The time in minutes after which the single Async Job should be considered as Timed Out.", 2798 ) 2799 download_target_requester: Optional[Union[HttpRequester, CustomRequester]] = Field( 2800 None, 2801 description="Requester component that describes how to prepare HTTP requests to send to the source API to extract the url from polling response by the completed async job.", 2802 ) 2803 download_requester: Union[HttpRequester, CustomRequester] = Field( 2804 ..., 2805 description="Requester component that describes how to prepare HTTP requests to send to the source API to download the data provided by the completed async job.", 2806 ) 2807 download_paginator: Optional[Union[DefaultPaginator, NoPagination]] = Field( 2808 None, 2809 description="Paginator component that describes how to navigate through the API's pages during download.", 2810 ) 2811 abort_requester: Optional[Union[HttpRequester, CustomRequester]] = Field( 2812 None, 2813 description="Requester component that describes how to prepare HTTP requests to send to the source API to abort a job once it is timed out from the source's perspective.", 2814 ) 2815 delete_requester: Optional[Union[HttpRequester, CustomRequester]] = Field( 2816 None, 2817 description="Requester component that describes how to prepare HTTP requests to send to the source API to delete a job once the records are extracted.", 2818 ) 2819 partition_router: Optional[ 2820 Union[ 2821 ListPartitionRouter, 2822 SubstreamPartitionRouter, 2823 GroupingPartitionRouter, 2824 CustomPartitionRouter, 2825 List[ 2826 Union[ 2827 ListPartitionRouter, 2828 SubstreamPartitionRouter, 2829 GroupingPartitionRouter, 2830 CustomPartitionRouter, 2831 ] 2832 ], 2833 ] 2834 ] = Field( 2835 [], 2836 description="PartitionRouter component that describes how to partition the stream, enabling incremental syncs and checkpointing.", 2837 title="Partition Router", 2838 ) 2839 decoder: Optional[ 2840 Union[ 2841 CsvDecoder, 2842 GzipDecoder, 2843 JsonDecoder, 2844 JsonlDecoder, 2845 IterableDecoder, 2846 XmlDecoder, 2847 ZipfileDecoder, 2848 CustomDecoder, 2849 ] 2850 ] = Field( 2851 None, 2852 description="Component decoding the response so records can be extracted.", 2853 title="Decoder", 2854 ) 2855 download_decoder: Optional[ 2856 Union[ 2857 CsvDecoder, 2858 GzipDecoder, 2859 JsonDecoder, 2860 JsonlDecoder, 2861 IterableDecoder, 2862 XmlDecoder, 2863 ZipfileDecoder, 2864 CustomDecoder, 2865 ] 2866 ] = Field( 2867 None, 2868 description="Component decoding the download response so records can be extracted.", 2869 title="Download Decoder", 2870 ) 2871 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters")
record_selector: RecordSelector
status_mapping: AsyncJobStatusMap
status_extractor: Union[DpathExtractor, CustomRecordExtractor]
download_target_extractor: Union[DpathExtractor, CustomRecordExtractor]
download_extractor: Union[DpathExtractor, CustomRecordExtractor, ResponseToFileExtractor, NoneType]
creation_requester: Union[HttpRequester, CustomRequester]
polling_requester: Union[HttpRequester, CustomRequester]
download_target_requester: Union[HttpRequester, CustomRequester, NoneType]
download_requester: Union[HttpRequester, CustomRequester]
download_paginator: Union[DefaultPaginator, NoPagination, NoneType]
abort_requester: Union[HttpRequester, CustomRequester, NoneType]
delete_requester: Union[HttpRequester, CustomRequester, NoneType]
partition_router: Union[ListPartitionRouter, SubstreamPartitionRouter, GroupingPartitionRouter, CustomPartitionRouter, List[Union[ListPartitionRouter, SubstreamPartitionRouter, GroupingPartitionRouter, CustomPartitionRouter]], NoneType]
decoder: Union[JsonDecoder, XmlDecoder, CsvDecoder, JsonlDecoder, GzipDecoder, IterableDecoder, ZipfileDecoder, CustomDecoder, NoneType]
download_decoder: Union[JsonDecoder, XmlDecoder, CsvDecoder, JsonlDecoder, GzipDecoder, IterableDecoder, ZipfileDecoder, CustomDecoder, NoneType]
class
SubstreamPartitionRouter(pydantic.v1.main.BaseModel):
2874class SubstreamPartitionRouter(BaseModel): 2875 type: Literal["SubstreamPartitionRouter"] 2876 parent_stream_configs: List[ParentStreamConfig] = Field( 2877 ..., 2878 description="Specifies which parent streams are being iterated over and how parent records should be used to partition the child stream data set.", 2879 title="Parent Stream Configs", 2880 ) 2881 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters")
parent_stream_configs: List[ParentStreamConfig]
class
GroupingPartitionRouter(pydantic.v1.main.BaseModel):
2884class GroupingPartitionRouter(BaseModel): 2885 type: Literal["GroupingPartitionRouter"] 2886 group_size: int = Field( 2887 ..., 2888 description="The number of partitions to include in each group. This determines how many partition values are batched together in a single slice.", 2889 examples=[10, 50], 2890 title="Group Size", 2891 ) 2892 underlying_partition_router: Union[ 2893 CustomPartitionRouter, ListPartitionRouter, SubstreamPartitionRouter 2894 ] = Field( 2895 ..., 2896 description="The partition router whose output will be grouped. This can be any valid partition router component.", 2897 title="Underlying Partition Router", 2898 ) 2899 deduplicate: Optional[bool] = Field( 2900 True, 2901 description="If true, ensures that partitions are unique within each group by removing duplicates based on the partition key.", 2902 title="Deduplicate Partitions", 2903 ) 2904 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters")
underlying_partition_router: Union[CustomPartitionRouter, ListPartitionRouter, SubstreamPartitionRouter]
class
HttpComponentsResolver(pydantic.v1.main.BaseModel):
2907class HttpComponentsResolver(BaseModel): 2908 type: Literal["HttpComponentsResolver"] 2909 retriever: Union[AsyncRetriever, CustomRetriever, SimpleRetriever] = Field( 2910 ..., 2911 description="Component used to coordinate how records are extracted across stream slices and request pages.", 2912 title="Retriever", 2913 ) 2914 components_mapping: List[ComponentMappingDefinition] 2915 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters")
retriever: Union[AsyncRetriever, CustomRetriever, SimpleRetriever]
components_mapping: List[ComponentMappingDefinition]
class
DynamicDeclarativeStream(pydantic.v1.main.BaseModel):
2918class DynamicDeclarativeStream(BaseModel): 2919 type: Literal["DynamicDeclarativeStream"] 2920 name: Optional[str] = Field( 2921 "", description="The dynamic stream name.", example=["Tables"], title="Name" 2922 ) 2923 stream_template: Union[DeclarativeStream, StateDelegatingStream] = Field( 2924 ..., description="Reference to the stream template.", title="Stream Template" 2925 ) 2926 components_resolver: Union[HttpComponentsResolver, ConfigComponentsResolver] = Field( 2927 ..., 2928 description="Component resolve and populates stream templates with components values.", 2929 title="Components Resolver", 2930 )
stream_template: Union[DeclarativeStream, StateDelegatingStream]
components_resolver: Union[HttpComponentsResolver, ConfigComponentsResolver]