airbyte_cdk.sources.declarative.models.declarative_component_schema
1# generated by datamodel-codegen: 2# filename: declarative_component_schema.yaml 3 4from __future__ import annotations 5 6from enum import Enum 7from typing import Any, Dict, List, Literal, Optional, Union 8 9from pydantic.v1 import BaseModel, Extra, Field 10 11from airbyte_cdk.sources.declarative.models.base_model_with_deprecations import ( 12 BaseModelWithDeprecations, 13) 14 15 16class AuthFlowType(Enum): 17 oauth2_0 = "oauth2.0" 18 oauth1_0 = "oauth1.0" 19 20 21class BasicHttpAuthenticator(BaseModel): 22 type: Literal["BasicHttpAuthenticator"] 23 username: str = Field( 24 ..., 25 description="The username that will be combined with the password, base64 encoded and used to make requests. Fill it in the user inputs.", 26 examples=["{{ config['username'] }}", "{{ config['api_key'] }}"], 27 title="Username", 28 ) 29 password: Optional[str] = Field( 30 "", 31 description="The password that will be combined with the username, base64 encoded and used to make requests. Fill it in the user inputs.", 32 examples=["{{ config['password'] }}", ""], 33 title="Password", 34 ) 35 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters") 36 37 38class BearerAuthenticator(BaseModel): 39 type: Literal["BearerAuthenticator"] 40 api_token: str = Field( 41 ..., 42 description="Token to inject as request header for authenticating with the API.", 43 examples=["{{ config['api_key'] }}", "{{ config['token'] }}"], 44 title="Bearer Token", 45 ) 46 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters") 47 48 49class DynamicStreamCheckConfig(BaseModel): 50 type: Literal["DynamicStreamCheckConfig"] 51 dynamic_stream_name: str = Field( 52 ..., description="The dynamic stream name.", title="Dynamic Stream Name" 53 ) 54 stream_count: Optional[int] = Field( 55 0, 56 description="The number of streams to attempt reading from during a check operation. If `stream_count` exceeds the total number of available streams, the minimum of the two values will be used.", 57 title="Stream Count", 58 ) 59 60 61class CheckDynamicStream(BaseModel): 62 type: Literal["CheckDynamicStream"] 63 stream_count: int = Field( 64 ..., 65 description="Numbers of the streams to try reading from when running a check operation.", 66 title="Stream Count", 67 ) 68 use_check_availability: Optional[bool] = Field( 69 True, 70 description="Enables stream check availability. This field is automatically set by the CDK.", 71 title="Use Check Availability", 72 ) 73 74 75class ConcurrencyLevel(BaseModel): 76 type: Optional[Literal["ConcurrencyLevel"]] = None 77 default_concurrency: Union[int, str] = Field( 78 ..., 79 description="The amount of concurrency that will applied during a sync. This value can be hardcoded or user-defined in the config if different users have varying volume thresholds in the target API.", 80 examples=[10, "{{ config['num_workers'] or 10 }}"], 81 title="Default Concurrency", 82 ) 83 max_concurrency: Optional[int] = Field( 84 None, 85 description="The maximum level of concurrency that will be used during a sync. This becomes a required field when the default_concurrency derives from the config, because it serves as a safeguard against a user-defined threshold that is too high.", 86 examples=[20, 100], 87 title="Max Concurrency", 88 ) 89 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters") 90 91 92class ConstantBackoffStrategy(BaseModel): 93 type: Literal["ConstantBackoffStrategy"] 94 backoff_time_in_seconds: Union[float, str] = Field( 95 ..., 96 description="Backoff time in seconds.", 97 examples=[30, 30.5, "{{ config['backoff_time'] }}"], 98 title="Backoff Time", 99 ) 100 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters") 101 102 103class CursorPagination(BaseModel): 104 type: Literal["CursorPagination"] 105 cursor_value: str = Field( 106 ..., 107 description="Value of the cursor defining the next page to fetch.", 108 examples=[ 109 "{{ headers.link.next.cursor }}", 110 "{{ last_record['key'] }}", 111 "{{ response['nextPage'] }}", 112 ], 113 title="Cursor Value", 114 ) 115 page_size: Optional[int] = Field( 116 None, 117 description="The number of records to include in each pages.", 118 examples=[100], 119 title="Page Size", 120 ) 121 stop_condition: Optional[str] = Field( 122 None, 123 description="Template string evaluating when to stop paginating.", 124 examples=[ 125 "{{ response.data.has_more is false }}", 126 "{{ 'next' not in headers['link'] }}", 127 ], 128 title="Stop Condition", 129 ) 130 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters") 131 132 133class CustomAuthenticator(BaseModel): 134 class Config: 135 extra = Extra.allow 136 137 type: Literal["CustomAuthenticator"] 138 class_name: str = Field( 139 ..., 140 description="Fully-qualified name of the class that will be implementing the custom authentication strategy. Has to be a sub class of DeclarativeAuthenticator. The format is `source_<name>.<package>.<class_name>`.", 141 examples=["source_railz.components.ShortLivedTokenAuthenticator"], 142 title="Class Name", 143 ) 144 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters") 145 146 147class CustomBackoffStrategy(BaseModel): 148 class Config: 149 extra = Extra.allow 150 151 type: Literal["CustomBackoffStrategy"] 152 class_name: str = Field( 153 ..., 154 description="Fully-qualified name of the class that will be implementing the custom backoff strategy. The format is `source_<name>.<package>.<class_name>`.", 155 examples=["source_railz.components.MyCustomBackoffStrategy"], 156 title="Class Name", 157 ) 158 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters") 159 160 161class CustomErrorHandler(BaseModel): 162 class Config: 163 extra = Extra.allow 164 165 type: Literal["CustomErrorHandler"] 166 class_name: str = Field( 167 ..., 168 description="Fully-qualified name of the class that will be implementing the custom error handler. The format is `source_<name>.<package>.<class_name>`.", 169 examples=["source_railz.components.MyCustomErrorHandler"], 170 title="Class Name", 171 ) 172 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters") 173 174 175class CustomIncrementalSync(BaseModel): 176 class Config: 177 extra = Extra.allow 178 179 type: Literal["CustomIncrementalSync"] 180 class_name: str = Field( 181 ..., 182 description="Fully-qualified name of the class that will be implementing the custom incremental sync. The format is `source_<name>.<package>.<class_name>`.", 183 examples=["source_railz.components.MyCustomIncrementalSync"], 184 title="Class Name", 185 ) 186 cursor_field: str = Field( 187 ..., 188 description="The location of the value on a record that will be used as a bookmark during sync.", 189 ) 190 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters") 191 192 193class CustomPaginationStrategy(BaseModel): 194 class Config: 195 extra = Extra.allow 196 197 type: Literal["CustomPaginationStrategy"] 198 class_name: str = Field( 199 ..., 200 description="Fully-qualified name of the class that will be implementing the custom pagination strategy. The format is `source_<name>.<package>.<class_name>`.", 201 examples=["source_railz.components.MyCustomPaginationStrategy"], 202 title="Class Name", 203 ) 204 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters") 205 206 207class CustomRecordExtractor(BaseModel): 208 class Config: 209 extra = Extra.allow 210 211 type: Literal["CustomRecordExtractor"] 212 class_name: str = Field( 213 ..., 214 description="Fully-qualified name of the class that will be implementing the custom record extraction strategy. The format is `source_<name>.<package>.<class_name>`.", 215 examples=["source_railz.components.MyCustomRecordExtractor"], 216 title="Class Name", 217 ) 218 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters") 219 220 221class CustomRecordFilter(BaseModel): 222 class Config: 223 extra = Extra.allow 224 225 type: Literal["CustomRecordFilter"] 226 class_name: str = Field( 227 ..., 228 description="Fully-qualified name of the class that will be implementing the custom record filter strategy. The format is `source_<name>.<package>.<class_name>`.", 229 examples=["source_railz.components.MyCustomCustomRecordFilter"], 230 title="Class Name", 231 ) 232 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters") 233 234 235class CustomRequester(BaseModel): 236 class Config: 237 extra = Extra.allow 238 239 type: Literal["CustomRequester"] 240 class_name: str = Field( 241 ..., 242 description="Fully-qualified name of the class that will be implementing the custom requester strategy. The format is `source_<name>.<package>.<class_name>`.", 243 examples=["source_railz.components.MyCustomRecordExtractor"], 244 title="Class Name", 245 ) 246 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters") 247 248 249class CustomRetriever(BaseModel): 250 class Config: 251 extra = Extra.allow 252 253 type: Literal["CustomRetriever"] 254 class_name: str = Field( 255 ..., 256 description="Fully-qualified name of the class that will be implementing the custom retriever strategy. The format is `source_<name>.<package>.<class_name>`.", 257 examples=["source_railz.components.MyCustomRetriever"], 258 title="Class Name", 259 ) 260 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters") 261 262 263class CustomPartitionRouter(BaseModel): 264 class Config: 265 extra = Extra.allow 266 267 type: Literal["CustomPartitionRouter"] 268 class_name: str = Field( 269 ..., 270 description="Fully-qualified name of the class that will be implementing the custom partition router. The format is `source_<name>.<package>.<class_name>`.", 271 examples=["source_railz.components.MyCustomPartitionRouter"], 272 title="Class Name", 273 ) 274 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters") 275 276 277class CustomSchemaLoader(BaseModel): 278 class Config: 279 extra = Extra.allow 280 281 type: Literal["CustomSchemaLoader"] 282 class_name: str = Field( 283 ..., 284 description="Fully-qualified name of the class that will be implementing the custom schema loader. The format is `source_<name>.<package>.<class_name>`.", 285 examples=["source_railz.components.MyCustomSchemaLoader"], 286 title="Class Name", 287 ) 288 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters") 289 290 291class CustomSchemaNormalization(BaseModel): 292 class Config: 293 extra = Extra.allow 294 295 type: Literal["CustomSchemaNormalization"] 296 class_name: str = Field( 297 ..., 298 description="Fully-qualified name of the class that will be implementing the custom normalization. The format is `source_<name>.<package>.<class_name>`.", 299 examples=[ 300 "source_amazon_seller_partner.components.LedgerDetailedViewReportsTypeTransformer" 301 ], 302 title="Class Name", 303 ) 304 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters") 305 306 307class CustomStateMigration(BaseModel): 308 class Config: 309 extra = Extra.allow 310 311 type: Literal["CustomStateMigration"] 312 class_name: str = Field( 313 ..., 314 description="Fully-qualified name of the class that will be implementing the custom state migration. The format is `source_<name>.<package>.<class_name>`.", 315 examples=["source_railz.components.MyCustomStateMigration"], 316 title="Class Name", 317 ) 318 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters") 319 320 321class CustomTransformation(BaseModel): 322 class Config: 323 extra = Extra.allow 324 325 type: Literal["CustomTransformation"] 326 class_name: str = Field( 327 ..., 328 description="Fully-qualified name of the class that will be implementing the custom transformation. The format is `source_<name>.<package>.<class_name>`.", 329 examples=["source_railz.components.MyCustomTransformation"], 330 title="Class Name", 331 ) 332 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters") 333 334 335class LegacyToPerPartitionStateMigration(BaseModel): 336 class Config: 337 extra = Extra.allow 338 339 type: Optional[Literal["LegacyToPerPartitionStateMigration"]] = None 340 341 342class Clamping(BaseModel): 343 target: str = Field( 344 ..., 345 description="The period of time that datetime windows will be clamped by", 346 examples=["DAY", "WEEK", "MONTH", "{{ config['target'] }}"], 347 title="Target", 348 ) 349 target_details: Optional[Dict[str, Any]] = None 350 351 352class Algorithm(Enum): 353 HS256 = "HS256" 354 HS384 = "HS384" 355 HS512 = "HS512" 356 ES256 = "ES256" 357 ES256K = "ES256K" 358 ES384 = "ES384" 359 ES512 = "ES512" 360 RS256 = "RS256" 361 RS384 = "RS384" 362 RS512 = "RS512" 363 PS256 = "PS256" 364 PS384 = "PS384" 365 PS512 = "PS512" 366 EdDSA = "EdDSA" 367 368 369class JwtHeaders(BaseModel): 370 class Config: 371 extra = Extra.forbid 372 373 kid: Optional[str] = Field( 374 None, 375 description="Private key ID for user account.", 376 examples=["{{ config['kid'] }}"], 377 title="Key Identifier", 378 ) 379 typ: Optional[str] = Field( 380 "JWT", 381 description="The media type of the complete JWT.", 382 examples=["JWT"], 383 title="Type", 384 ) 385 cty: Optional[str] = Field( 386 None, 387 description="Content type of JWT header.", 388 examples=["JWT"], 389 title="Content Type", 390 ) 391 392 393class JwtPayload(BaseModel): 394 class Config: 395 extra = Extra.forbid 396 397 iss: Optional[str] = Field( 398 None, 399 description="The user/principal that issued the JWT. Commonly a value unique to the user.", 400 examples=["{{ config['iss'] }}"], 401 title="Issuer", 402 ) 403 sub: Optional[str] = Field( 404 None, 405 description="The subject of the JWT. Commonly defined by the API.", 406 title="Subject", 407 ) 408 aud: Optional[str] = Field( 409 None, 410 description="The recipient that the JWT is intended for. Commonly defined by the API.", 411 examples=["appstoreconnect-v1"], 412 title="Audience", 413 ) 414 415 416class JwtAuthenticator(BaseModel): 417 type: Literal["JwtAuthenticator"] 418 secret_key: str = Field( 419 ..., 420 description="Secret used to sign the JSON web token.", 421 examples=["{{ config['secret_key'] }}"], 422 title="Secret Key", 423 ) 424 base64_encode_secret_key: Optional[bool] = Field( 425 False, 426 description='When set to true, the secret key will be base64 encoded prior to being encoded as part of the JWT. Only set to "true" when required by the API.', 427 title="Base64-encode Secret Key", 428 ) 429 algorithm: Algorithm = Field( 430 ..., 431 description="Algorithm used to sign the JSON web token.", 432 examples=["ES256", "HS256", "RS256", "{{ config['algorithm'] }}"], 433 title="Algorithm", 434 ) 435 token_duration: Optional[int] = Field( 436 1200, 437 description="The amount of time in seconds a JWT token can be valid after being issued.", 438 examples=[1200, 3600], 439 title="Token Duration", 440 ) 441 header_prefix: Optional[str] = Field( 442 None, 443 description="The prefix to be used within the Authentication header.", 444 examples=["Bearer", "Basic"], 445 title="Header Prefix", 446 ) 447 jwt_headers: Optional[JwtHeaders] = Field( 448 None, 449 description="JWT headers used when signing JSON web token.", 450 title="JWT Headers", 451 ) 452 additional_jwt_headers: Optional[Dict[str, Any]] = Field( 453 None, 454 description="Additional headers to be included with the JWT headers object.", 455 title="Additional JWT Headers", 456 ) 457 jwt_payload: Optional[JwtPayload] = Field( 458 None, 459 description="JWT Payload used when signing JSON web token.", 460 title="JWT Payload", 461 ) 462 additional_jwt_payload: Optional[Dict[str, Any]] = Field( 463 None, 464 description="Additional properties to be added to the JWT payload.", 465 title="Additional JWT Payload Properties", 466 ) 467 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters") 468 469 470class RefreshTokenUpdater(BaseModel): 471 refresh_token_name: Optional[str] = Field( 472 "refresh_token", 473 description="The name of the property which contains the updated refresh token in the response from the token refresh endpoint.", 474 examples=["refresh_token"], 475 title="Refresh Token Property Name", 476 ) 477 access_token_config_path: Optional[List[str]] = Field( 478 ["credentials", "access_token"], 479 description="Config path to the access token. Make sure the field actually exists in the config.", 480 examples=[["credentials", "access_token"], ["access_token"]], 481 title="Config Path To Access Token", 482 ) 483 refresh_token_config_path: Optional[List[str]] = Field( 484 ["credentials", "refresh_token"], 485 description="Config path to the access token. Make sure the field actually exists in the config.", 486 examples=[["credentials", "refresh_token"], ["refresh_token"]], 487 title="Config Path To Refresh Token", 488 ) 489 token_expiry_date_config_path: Optional[List[str]] = Field( 490 ["credentials", "token_expiry_date"], 491 description="Config path to the expiry date. Make sure actually exists in the config.", 492 examples=[["credentials", "token_expiry_date"]], 493 title="Config Path To Expiry Date", 494 ) 495 refresh_token_error_status_codes: Optional[List[int]] = Field( 496 [], 497 description="Status Codes to Identify refresh token error in response (Refresh Token Error Key and Refresh Token Error Values should be also specified). Responses with one of the error status code and containing an error value will be flagged as a config error", 498 examples=[[400, 500]], 499 title="Refresh Token Error Status Codes", 500 ) 501 refresh_token_error_key: Optional[str] = Field( 502 "", 503 description="Key to Identify refresh token error in response (Refresh Token Error Status Codes and Refresh Token Error Values should be also specified).", 504 examples=["error"], 505 title="Refresh Token Error Key", 506 ) 507 refresh_token_error_values: Optional[List[str]] = Field( 508 [], 509 description='List of values to check for exception during token refresh process. Used to check if the error found in the response matches the key from the Refresh Token Error Key field (e.g. response={"error": "invalid_grant"}). Only responses with one of the error status code and containing an error value will be flagged as a config error', 510 examples=[["invalid_grant", "invalid_permissions"]], 511 title="Refresh Token Error Values", 512 ) 513 514 515class OAuthAuthenticator(BaseModel): 516 type: Literal["OAuthAuthenticator"] 517 client_id_name: Optional[str] = Field( 518 "client_id", 519 description="The name of the property to use to refresh the `access_token`.", 520 examples=["custom_app_id"], 521 title="Client ID Property Name", 522 ) 523 client_id: Optional[str] = Field( 524 None, 525 description="The OAuth client ID. Fill it in the user inputs.", 526 examples=["{{ config['client_id }}", "{{ config['credentials']['client_id }}"], 527 title="Client ID", 528 ) 529 client_secret_name: Optional[str] = Field( 530 "client_secret", 531 description="The name of the property to use to refresh the `access_token`.", 532 examples=["custom_app_secret"], 533 title="Client Secret Property Name", 534 ) 535 client_secret: Optional[str] = Field( 536 None, 537 description="The OAuth client secret. Fill it in the user inputs.", 538 examples=[ 539 "{{ config['client_secret }}", 540 "{{ config['credentials']['client_secret }}", 541 ], 542 title="Client Secret", 543 ) 544 refresh_token_name: Optional[str] = Field( 545 "refresh_token", 546 description="The name of the property to use to refresh the `access_token`.", 547 examples=["custom_app_refresh_value"], 548 title="Refresh Token Property Name", 549 ) 550 refresh_token: Optional[str] = Field( 551 None, 552 description="Credential artifact used to get a new access token.", 553 examples=[ 554 "{{ config['refresh_token'] }}", 555 "{{ config['credentials]['refresh_token'] }}", 556 ], 557 title="Refresh Token", 558 ) 559 token_refresh_endpoint: Optional[str] = Field( 560 None, 561 description="The full URL to call to obtain a new access token.", 562 examples=["https://connect.squareup.com/oauth2/token"], 563 title="Token Refresh Endpoint", 564 ) 565 access_token_name: Optional[str] = Field( 566 "access_token", 567 description="The name of the property which contains the access token in the response from the token refresh endpoint.", 568 examples=["access_token"], 569 title="Access Token Property Name", 570 ) 571 access_token_value: Optional[str] = Field( 572 None, 573 description="The value of the access_token to bypass the token refreshing using `refresh_token`.", 574 examples=["secret_access_token_value"], 575 title="Access Token Value", 576 ) 577 expires_in_name: Optional[str] = Field( 578 "expires_in", 579 description="The name of the property which contains the expiry date in the response from the token refresh endpoint.", 580 examples=["expires_in"], 581 title="Token Expiry Property Name", 582 ) 583 grant_type_name: Optional[str] = Field( 584 "grant_type", 585 description="The name of the property to use to refresh the `access_token`.", 586 examples=["custom_grant_type"], 587 title="Grant Type Property Name", 588 ) 589 grant_type: Optional[str] = Field( 590 "refresh_token", 591 description="Specifies the OAuth2 grant type. If set to refresh_token, the refresh_token needs to be provided as well. For client_credentials, only client id and secret are required. Other grant types are not officially supported.", 592 examples=["refresh_token", "client_credentials"], 593 title="Grant Type", 594 ) 595 refresh_request_body: Optional[Dict[str, Any]] = Field( 596 None, 597 description="Body of the request sent to get a new access token.", 598 examples=[ 599 { 600 "applicationId": "{{ config['application_id'] }}", 601 "applicationSecret": "{{ config['application_secret'] }}", 602 "token": "{{ config['token'] }}", 603 } 604 ], 605 title="Refresh Request Body", 606 ) 607 refresh_request_headers: Optional[Dict[str, Any]] = Field( 608 None, 609 description="Headers of the request sent to get a new access token.", 610 examples=[ 611 { 612 "Authorization": "<AUTH_TOKEN>", 613 "Content-Type": "application/x-www-form-urlencoded", 614 } 615 ], 616 title="Refresh Request Headers", 617 ) 618 scopes: Optional[List[str]] = Field( 619 None, 620 description="List of scopes that should be granted to the access token.", 621 examples=[["crm.list.read", "crm.objects.contacts.read", "crm.schema.contacts.read"]], 622 title="Scopes", 623 ) 624 token_expiry_date: Optional[str] = Field( 625 None, 626 description="The access token expiry date.", 627 examples=["2023-04-06T07:12:10.421833+00:00", 1680842386], 628 title="Token Expiry Date", 629 ) 630 token_expiry_date_format: Optional[str] = Field( 631 None, 632 description="The format of the time to expiration datetime. Provide it if the time is returned as a date-time string instead of seconds.", 633 examples=["%Y-%m-%d %H:%M:%S.%f+00:00"], 634 title="Token Expiry Date Format", 635 ) 636 refresh_token_updater: Optional[RefreshTokenUpdater] = Field( 637 None, 638 description="When the token updater is defined, new refresh tokens, access tokens and the access token expiry date are written back from the authentication response to the config object. This is important if the refresh token can only used once.", 639 title="Token Updater", 640 ) 641 profile_assertion: Optional[JwtAuthenticator] = Field( 642 None, 643 description="The authenticator being used to authenticate the client authenticator.", 644 title="Profile Assertion", 645 ) 646 use_profile_assertion: Optional[bool] = Field( 647 False, 648 description="Enable using profile assertion as a flow for OAuth authorization.", 649 title="Use Profile Assertion", 650 ) 651 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters") 652 653 654class Rate(BaseModel): 655 class Config: 656 extra = Extra.allow 657 658 limit: Union[int, str] = Field( 659 ..., 660 description="The maximum number of calls allowed within the interval.", 661 title="Limit", 662 ) 663 interval: str = Field( 664 ..., 665 description="The time interval for the rate limit.", 666 examples=["PT1H", "P1D"], 667 title="Interval", 668 ) 669 670 671class HttpRequestRegexMatcher(BaseModel): 672 class Config: 673 extra = Extra.allow 674 675 method: Optional[str] = Field( 676 None, description="The HTTP method to match (e.g., GET, POST).", title="Method" 677 ) 678 url_base: Optional[str] = Field( 679 None, 680 description='The base URL (scheme and host, e.g. "https://api.example.com") to match.', 681 title="URL Base", 682 ) 683 url_path_pattern: Optional[str] = Field( 684 None, 685 description="A regular expression pattern to match the URL path.", 686 title="URL Path Pattern", 687 ) 688 params: Optional[Dict[str, Any]] = Field( 689 None, description="The query parameters to match.", title="Parameters" 690 ) 691 headers: Optional[Dict[str, Any]] = Field( 692 None, description="The headers to match.", title="Headers" 693 ) 694 695 696class DpathExtractor(BaseModel): 697 type: Literal["DpathExtractor"] 698 field_path: List[str] = Field( 699 ..., 700 description='List of potentially nested fields describing the full path of the field to extract. Use "*" to extract all values from an array. See more info in the [docs](https://docs.airbyte.com/connector-development/config-based/understanding-the-yaml-file/record-selector).', 701 examples=[ 702 ["data"], 703 ["data", "records"], 704 ["data", "{{ parameters.name }}"], 705 ["data", "*", "record"], 706 ], 707 title="Field Path", 708 ) 709 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters") 710 711 712class ResponseToFileExtractor(BaseModel): 713 type: Literal["ResponseToFileExtractor"] 714 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters") 715 716 717class ExponentialBackoffStrategy(BaseModel): 718 type: Literal["ExponentialBackoffStrategy"] 719 factor: Optional[Union[float, str]] = Field( 720 5, 721 description="Multiplicative constant applied on each retry.", 722 examples=[5, 5.5, "10"], 723 title="Factor", 724 ) 725 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters") 726 727 728class GroupByKeyMergeStrategy(BaseModel): 729 type: Literal["GroupByKeyMergeStrategy"] 730 key: Union[str, List[str]] = Field( 731 ..., 732 description="The name of the field on the record whose value will be used to group properties that were retrieved through multiple API requests.", 733 examples=["id", ["parent_id", "end_date"]], 734 title="Key", 735 ) 736 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters") 737 738 739class SessionTokenRequestBearerAuthenticator(BaseModel): 740 type: Literal["Bearer"] 741 742 743class HttpMethod(Enum): 744 GET = "GET" 745 POST = "POST" 746 747 748class Action(Enum): 749 SUCCESS = "SUCCESS" 750 FAIL = "FAIL" 751 RETRY = "RETRY" 752 IGNORE = "IGNORE" 753 RATE_LIMITED = "RATE_LIMITED" 754 755 756class FailureType(Enum): 757 system_error = "system_error" 758 config_error = "config_error" 759 transient_error = "transient_error" 760 761 762class HttpResponseFilter(BaseModel): 763 type: Literal["HttpResponseFilter"] 764 action: Optional[Action] = Field( 765 None, 766 description="Action to execute if a response matches the filter.", 767 examples=["SUCCESS", "FAIL", "RETRY", "IGNORE", "RATE_LIMITED"], 768 title="Action", 769 ) 770 failure_type: Optional[FailureType] = Field( 771 None, 772 description="Failure type of traced exception if a response matches the filter.", 773 examples=["system_error", "config_error", "transient_error"], 774 title="Failure Type", 775 ) 776 error_message: Optional[str] = Field( 777 None, 778 description="Error Message to display if the response matches the filter.", 779 title="Error Message", 780 ) 781 error_message_contains: Optional[str] = Field( 782 None, 783 description="Match the response if its error message contains the substring.", 784 example=["This API operation is not enabled for this site"], 785 title="Error Message Substring", 786 ) 787 http_codes: Optional[List[int]] = Field( 788 None, 789 description="Match the response if its HTTP code is included in this list.", 790 examples=[[420, 429], [500]], 791 title="HTTP Codes", 792 unique_items=True, 793 ) 794 predicate: Optional[str] = Field( 795 None, 796 description="Match the response if the predicate evaluates to true.", 797 examples=[ 798 "{{ 'Too much requests' in response }}", 799 "{{ 'error_code' in response and response['error_code'] == 'ComplexityException' }}", 800 ], 801 title="Predicate", 802 ) 803 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters") 804 805 806class ComplexFieldType(BaseModel): 807 field_type: str 808 items: Optional[Union[str, ComplexFieldType]] = None 809 810 811class TypesMap(BaseModel): 812 target_type: Union[str, List[str], ComplexFieldType] 813 current_type: Union[str, List[str]] 814 condition: Optional[str] = None 815 816 817class SchemaTypeIdentifier(BaseModel): 818 type: Optional[Literal["SchemaTypeIdentifier"]] = None 819 schema_pointer: Optional[List[str]] = Field( 820 [], 821 description="List of nested fields defining the schema field path to extract. Defaults to [].", 822 title="Schema Path", 823 ) 824 key_pointer: List[str] = Field( 825 ..., 826 description="List of potentially nested fields describing the full path of the field key to extract.", 827 title="Key Path", 828 ) 829 type_pointer: Optional[List[str]] = Field( 830 None, 831 description="List of potentially nested fields describing the full path of the field type to extract.", 832 title="Type Path", 833 ) 834 types_mapping: Optional[List[TypesMap]] = None 835 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters") 836 837 838class InlineSchemaLoader(BaseModel): 839 type: Literal["InlineSchemaLoader"] 840 schema_: Optional[Dict[str, Any]] = Field( 841 None, 842 alias="schema", 843 description='Describes a streams\' schema. Refer to the <a href="https://docs.airbyte.com/understanding-airbyte/supported-data-types/">Data Types documentation</a> for more details on which types are valid.', 844 title="Schema", 845 ) 846 847 848class JsonFileSchemaLoader(BaseModel): 849 type: Literal["JsonFileSchemaLoader"] 850 file_path: Optional[str] = Field( 851 None, 852 description="Path to the JSON file defining the schema. The path is relative to the connector module's root.", 853 example=["./schemas/users.json"], 854 title="File Path", 855 ) 856 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters") 857 858 859class JsonDecoder(BaseModel): 860 type: Literal["JsonDecoder"] 861 862 863class JsonlDecoder(BaseModel): 864 type: Literal["JsonlDecoder"] 865 866 867class KeysToLower(BaseModel): 868 type: Literal["KeysToLower"] 869 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters") 870 871 872class KeysToSnakeCase(BaseModel): 873 type: Literal["KeysToSnakeCase"] 874 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters") 875 876 877class FlattenFields(BaseModel): 878 type: Literal["FlattenFields"] 879 flatten_lists: Optional[bool] = Field( 880 True, 881 description="Whether to flatten lists or leave it as is. Default is True.", 882 title="Flatten Lists", 883 ) 884 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters") 885 886 887class KeyTransformation(BaseModel): 888 type: Literal["KeyTransformation"] 889 prefix: Optional[str] = Field( 890 None, 891 description="Prefix to add for object keys. If not provided original keys remain unchanged.", 892 examples=["flattened_"], 893 title="Key Prefix", 894 ) 895 suffix: Optional[str] = Field( 896 None, 897 description="Suffix to add for object keys. If not provided original keys remain unchanged.", 898 examples=["_flattened"], 899 title="Key Suffix", 900 ) 901 902 903class DpathFlattenFields(BaseModel): 904 type: Literal["DpathFlattenFields"] 905 field_path: List[str] = Field( 906 ..., 907 description="A path to field that needs to be flattened.", 908 examples=[["data"], ["data", "*", "field"]], 909 title="Field Path", 910 ) 911 delete_origin_value: Optional[bool] = Field( 912 None, 913 description="Whether to delete the origin value or keep it. Default is False.", 914 title="Delete Origin Value", 915 ) 916 replace_record: Optional[bool] = Field( 917 None, 918 description="Whether to replace the origin record or not. Default is False.", 919 title="Replace Origin Record", 920 ) 921 key_transformation: Optional[KeyTransformation] = Field( 922 None, 923 description="Transformation for object keys. If not provided, original key will be used.", 924 title="Key transformation", 925 ) 926 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters") 927 928 929class KeysReplace(BaseModel): 930 type: Literal["KeysReplace"] 931 old: str = Field( 932 ..., 933 description="Old value to replace.", 934 examples=[ 935 " ", 936 "{{ record.id }}", 937 "{{ config['id'] }}", 938 "{{ stream_slice['id'] }}", 939 ], 940 title="Old value", 941 ) 942 new: str = Field( 943 ..., 944 description="New value to set.", 945 examples=[ 946 "_", 947 "{{ record.id }}", 948 "{{ config['id'] }}", 949 "{{ stream_slice['id'] }}", 950 ], 951 title="New value", 952 ) 953 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters") 954 955 956class IterableDecoder(BaseModel): 957 type: Literal["IterableDecoder"] 958 959 960class XmlDecoder(BaseModel): 961 type: Literal["XmlDecoder"] 962 963 964class CustomDecoder(BaseModel): 965 class Config: 966 extra = Extra.allow 967 968 type: Literal["CustomDecoder"] 969 class_name: str = Field( 970 ..., 971 description="Fully-qualified name of the class that will be implementing the custom decoding. Has to be a sub class of Decoder. The format is `source_<name>.<package>.<class_name>`.", 972 examples=["source_amazon_ads.components.GzipJsonlDecoder"], 973 title="Class Name", 974 ) 975 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters") 976 977 978class MinMaxDatetime(BaseModel): 979 type: Literal["MinMaxDatetime"] 980 datetime: str = Field( 981 ..., 982 description="Datetime value.", 983 examples=["2021-01-01", "2021-01-01T00:00:00Z", "{{ config['start_time'] }}"], 984 title="Datetime", 985 ) 986 datetime_format: Optional[str] = Field( 987 "", 988 description='Format of the datetime value. Defaults to "%Y-%m-%dT%H:%M:%S.%f%z" if left empty. Use placeholders starting with "%" to describe the format the API is using. The following placeholders are available:\n * **%s**: Epoch unix timestamp - `1686218963`\n * **%s_as_float**: Epoch unix timestamp in seconds as float with microsecond precision - `1686218963.123456`\n * **%ms**: Epoch unix timestamp - `1686218963123`\n * **%a**: Weekday (abbreviated) - `Sun`\n * **%A**: Weekday (full) - `Sunday`\n * **%w**: Weekday (decimal) - `0` (Sunday), `6` (Saturday)\n * **%d**: Day of the month (zero-padded) - `01`, `02`, ..., `31`\n * **%b**: Month (abbreviated) - `Jan`\n * **%B**: Month (full) - `January`\n * **%m**: Month (zero-padded) - `01`, `02`, ..., `12`\n * **%y**: Year (without century, zero-padded) - `00`, `01`, ..., `99`\n * **%Y**: Year (with century) - `0001`, `0002`, ..., `9999`\n * **%H**: Hour (24-hour, zero-padded) - `00`, `01`, ..., `23`\n * **%I**: Hour (12-hour, zero-padded) - `01`, `02`, ..., `12`\n * **%p**: AM/PM indicator\n * **%M**: Minute (zero-padded) - `00`, `01`, ..., `59`\n * **%S**: Second (zero-padded) - `00`, `01`, ..., `59`\n * **%f**: Microsecond (zero-padded to 6 digits) - `000000`, `000001`, ..., `999999`\n * **%_ms**: Millisecond (zero-padded to 3 digits) - `000`, `001`, ..., `999`\n * **%z**: UTC offset - `(empty)`, `+0000`, `-04:00`\n * **%Z**: Time zone name - `(empty)`, `UTC`, `GMT`\n * **%j**: Day of the year (zero-padded) - `001`, `002`, ..., `366`\n * **%U**: Week number of the year (Sunday as first day) - `00`, `01`, ..., `53`\n * **%W**: Week number of the year (Monday as first day) - `00`, `01`, ..., `53`\n * **%c**: Date and time representation - `Tue Aug 16 21:30:00 1988`\n * **%x**: Date representation - `08/16/1988`\n * **%X**: Time representation - `21:30:00`\n * **%%**: Literal \'%\' character\n\n Some placeholders depend on the locale of the underlying system - in most cases this locale is configured as en/US. For more information see the [Python documentation](https://docs.python.org/3/library/datetime.html#strftime-and-strptime-format-codes).\n', 989 examples=["%Y-%m-%dT%H:%M:%S.%f%z", "%Y-%m-%d", "%s"], 990 title="Datetime Format", 991 ) 992 max_datetime: Optional[str] = Field( 993 None, 994 description="Ceiling applied on the datetime value. Must be formatted with the datetime_format field.", 995 examples=["2021-01-01T00:00:00Z", "2021-01-01"], 996 title="Max Datetime", 997 ) 998 min_datetime: Optional[str] = Field( 999 None, 1000 description="Floor applied on the datetime value. Must be formatted with the datetime_format field.", 1001 examples=["2010-01-01T00:00:00Z", "2010-01-01"], 1002 title="Min Datetime", 1003 ) 1004 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters") 1005 1006 1007class NoAuth(BaseModel): 1008 type: Literal["NoAuth"] 1009 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters") 1010 1011 1012class NoPagination(BaseModel): 1013 type: Literal["NoPagination"] 1014 1015 1016class State(BaseModel): 1017 class Config: 1018 extra = Extra.allow 1019 1020 min: int 1021 max: int 1022 1023 1024class OauthConnectorInputSpecification(BaseModel): 1025 class Config: 1026 extra = Extra.allow 1027 1028 consent_url: str = Field( 1029 ..., 1030 description="The DeclarativeOAuth Specific string URL string template to initiate the authentication.\nThe placeholders are replaced during the processing to provide neccessary values.", 1031 examples=[ 1032 "https://domain.host.com/marketing_api/auth?{{client_id_key}}={{client_id_value}}&{{redirect_uri_key}}={{{{redirect_uri_value}} | urlEncoder}}&{{state_key}}={{state_value}}", 1033 "https://endpoint.host.com/oauth2/authorize?{{client_id_key}}={{client_id_value}}&{{redirect_uri_key}}={{{{redirect_uri_value}} | urlEncoder}}&{{scope_key}}={{{{scope_value}} | urlEncoder}}&{{state_key}}={{state_value}}&subdomain={{subdomain}}", 1034 ], 1035 title="Consent URL", 1036 ) 1037 scope: Optional[str] = Field( 1038 None, 1039 description="The DeclarativeOAuth Specific string of the scopes needed to be grant for authenticated user.", 1040 examples=["user:read user:read_orders workspaces:read"], 1041 title="Scopes", 1042 ) 1043 access_token_url: str = Field( 1044 ..., 1045 description="The DeclarativeOAuth Specific URL templated string to obtain the `access_token`, `refresh_token` etc.\nThe placeholders are replaced during the processing to provide neccessary values.", 1046 examples=[ 1047 "https://auth.host.com/oauth2/token?{{client_id_key}}={{client_id_value}}&{{client_secret_key}}={{client_secret_value}}&{{auth_code_key}}={{auth_code_value}}&{{redirect_uri_key}}={{{{redirect_uri_value}} | urlEncoder}}" 1048 ], 1049 title="Access Token URL", 1050 ) 1051 access_token_headers: Optional[Dict[str, Any]] = Field( 1052 None, 1053 description="The DeclarativeOAuth Specific optional headers to inject while exchanging the `auth_code` to `access_token` during `completeOAuthFlow` step.", 1054 examples=[ 1055 { 1056 "Authorization": "Basic {{ {{ client_id_value }}:{{ client_secret_value }} | base64Encoder }}" 1057 } 1058 ], 1059 title="Access Token Headers", 1060 ) 1061 access_token_params: Optional[Dict[str, Any]] = Field( 1062 None, 1063 description="The DeclarativeOAuth Specific optional query parameters to inject while exchanging the `auth_code` to `access_token` during `completeOAuthFlow` step.\nWhen this property is provided, the query params will be encoded as `Json` and included in the outgoing API request.", 1064 examples=[ 1065 { 1066 "{{ auth_code_key }}": "{{ auth_code_value }}", 1067 "{{ client_id_key }}": "{{ client_id_value }}", 1068 "{{ client_secret_key }}": "{{ client_secret_value }}", 1069 } 1070 ], 1071 title="Access Token Query Params (Json Encoded)", 1072 ) 1073 extract_output: Optional[List[str]] = Field( 1074 None, 1075 description="The DeclarativeOAuth Specific list of strings to indicate which keys should be extracted and returned back to the input config.", 1076 examples=[["access_token", "refresh_token", "other_field"]], 1077 title="Extract Output", 1078 ) 1079 state: Optional[State] = Field( 1080 None, 1081 description="The DeclarativeOAuth Specific object to provide the criteria of how the `state` query param should be constructed,\nincluding length and complexity.", 1082 examples=[{"min": 7, "max": 128}], 1083 title="Configurable State Query Param", 1084 ) 1085 client_id_key: Optional[str] = Field( 1086 None, 1087 description="The DeclarativeOAuth Specific optional override to provide the custom `client_id` key name, if required by data-provider.", 1088 examples=["my_custom_client_id_key_name"], 1089 title="Client ID Key Override", 1090 ) 1091 client_secret_key: Optional[str] = Field( 1092 None, 1093 description="The DeclarativeOAuth Specific optional override to provide the custom `client_secret` key name, if required by data-provider.", 1094 examples=["my_custom_client_secret_key_name"], 1095 title="Client Secret Key Override", 1096 ) 1097 scope_key: Optional[str] = Field( 1098 None, 1099 description="The DeclarativeOAuth Specific optional override to provide the custom `scope` key name, if required by data-provider.", 1100 examples=["my_custom_scope_key_key_name"], 1101 title="Scopes Key Override", 1102 ) 1103 state_key: Optional[str] = Field( 1104 None, 1105 description="The DeclarativeOAuth Specific optional override to provide the custom `state` key name, if required by data-provider.", 1106 examples=["my_custom_state_key_key_name"], 1107 title="State Key Override", 1108 ) 1109 auth_code_key: Optional[str] = Field( 1110 None, 1111 description="The DeclarativeOAuth Specific optional override to provide the custom `code` key name to something like `auth_code` or `custom_auth_code`, if required by data-provider.", 1112 examples=["my_custom_auth_code_key_name"], 1113 title="Auth Code Key Override", 1114 ) 1115 redirect_uri_key: Optional[str] = Field( 1116 None, 1117 description="The DeclarativeOAuth Specific optional override to provide the custom `redirect_uri` key name to something like `callback_uri`, if required by data-provider.", 1118 examples=["my_custom_redirect_uri_key_name"], 1119 title="Redirect URI Key Override", 1120 ) 1121 1122 1123class OAuthConfigSpecification(BaseModel): 1124 class Config: 1125 extra = Extra.allow 1126 1127 oauth_user_input_from_connector_config_specification: Optional[Dict[str, Any]] = Field( 1128 None, 1129 description="OAuth specific blob. This is a Json Schema used to validate Json configurations used as input to OAuth.\nMust be a valid non-nested JSON that refers to properties from ConnectorSpecification.connectionSpecification\nusing special annotation 'path_in_connector_config'.\nThese are input values the user is entering through the UI to authenticate to the connector, that might also shared\nas inputs for syncing data via the connector.\nExamples:\nif no connector values is shared during oauth flow, oauth_user_input_from_connector_config_specification=[]\nif connector values such as 'app_id' inside the top level are used to generate the API url for the oauth flow,\n oauth_user_input_from_connector_config_specification={\n app_id: {\n type: string\n path_in_connector_config: ['app_id']\n }\n }\nif connector values such as 'info.app_id' nested inside another object are used to generate the API url for the oauth flow,\n oauth_user_input_from_connector_config_specification={\n app_id: {\n type: string\n path_in_connector_config: ['info', 'app_id']\n }\n }", 1130 examples=[ 1131 {"app_id": {"type": "string", "path_in_connector_config": ["app_id"]}}, 1132 { 1133 "app_id": { 1134 "type": "string", 1135 "path_in_connector_config": ["info", "app_id"], 1136 } 1137 }, 1138 ], 1139 title="OAuth user input", 1140 ) 1141 oauth_connector_input_specification: Optional[OauthConnectorInputSpecification] = Field( 1142 None, 1143 description='The DeclarativeOAuth specific blob.\nPertains to the fields defined by the connector relating to the OAuth flow.\n\nInterpolation capabilities:\n- The variables placeholders are declared as `{{my_var}}`.\n- The nested resolution variables like `{{ {{my_nested_var}} }}` is allowed as well.\n\n- The allowed interpolation context is:\n + base64Encoder - encode to `base64`, {{ {{my_var_a}}:{{my_var_b}} | base64Encoder }}\n + base64Decorer - decode from `base64` encoded string, {{ {{my_string_variable_or_string_value}} | base64Decoder }}\n + urlEncoder - encode the input string to URL-like format, {{ https://test.host.com/endpoint | urlEncoder}}\n + urlDecorer - decode the input url-encoded string into text format, {{ urlDecoder:https%3A%2F%2Fairbyte.io | urlDecoder}}\n + codeChallengeS256 - get the `codeChallenge` encoded value to provide additional data-provider specific authorisation values, {{ {{state_value}} | codeChallengeS256 }}\n\nExamples:\n - The TikTok Marketing DeclarativeOAuth spec:\n {\n "oauth_connector_input_specification": {\n "type": "object",\n "additionalProperties": false,\n "properties": {\n "consent_url": "https://ads.tiktok.com/marketing_api/auth?{{client_id_key}}={{client_id_value}}&{{redirect_uri_key}}={{ {{redirect_uri_value}} | urlEncoder}}&{{state_key}}={{state_value}}",\n "access_token_url": "https://business-api.tiktok.com/open_api/v1.3/oauth2/access_token/",\n "access_token_params": {\n "{{ auth_code_key }}": "{{ auth_code_value }}",\n "{{ client_id_key }}": "{{ client_id_value }}",\n "{{ client_secret_key }}": "{{ client_secret_value }}"\n },\n "access_token_headers": {\n "Content-Type": "application/json",\n "Accept": "application/json"\n },\n "extract_output": ["data.access_token"],\n "client_id_key": "app_id",\n "client_secret_key": "secret",\n "auth_code_key": "auth_code"\n }\n }\n }', 1144 title="DeclarativeOAuth Connector Specification", 1145 ) 1146 complete_oauth_output_specification: Optional[Dict[str, Any]] = Field( 1147 None, 1148 description="OAuth specific blob. This is a Json Schema used to validate Json configurations produced by the OAuth flows as they are\nreturned by the distant OAuth APIs.\nMust be a valid JSON describing the fields to merge back to `ConnectorSpecification.connectionSpecification`.\nFor each field, a special annotation `path_in_connector_config` can be specified to determine where to merge it,\nExamples:\n complete_oauth_output_specification={\n refresh_token: {\n type: string,\n path_in_connector_config: ['credentials', 'refresh_token']\n }\n }", 1149 examples=[ 1150 { 1151 "refresh_token": { 1152 "type": "string,", 1153 "path_in_connector_config": ["credentials", "refresh_token"], 1154 } 1155 } 1156 ], 1157 title="OAuth output specification", 1158 ) 1159 complete_oauth_server_input_specification: Optional[Dict[str, Any]] = Field( 1160 None, 1161 description="OAuth specific blob. This is a Json Schema used to validate Json configurations persisted as Airbyte Server configurations.\nMust be a valid non-nested JSON describing additional fields configured by the Airbyte Instance or Workspace Admins to be used by the\nserver when completing an OAuth flow (typically exchanging an auth code for refresh token).\nExamples:\n complete_oauth_server_input_specification={\n client_id: {\n type: string\n },\n client_secret: {\n type: string\n }\n }", 1162 examples=[{"client_id": {"type": "string"}, "client_secret": {"type": "string"}}], 1163 title="OAuth input specification", 1164 ) 1165 complete_oauth_server_output_specification: Optional[Dict[str, Any]] = Field( 1166 None, 1167 description="OAuth specific blob. This is a Json Schema used to validate Json configurations persisted as Airbyte Server configurations that\nalso need to be merged back into the connector configuration at runtime.\nThis is a subset configuration of `complete_oauth_server_input_specification` that filters fields out to retain only the ones that\nare necessary for the connector to function with OAuth. (some fields could be used during oauth flows but not needed afterwards, therefore\nthey would be listed in the `complete_oauth_server_input_specification` but not `complete_oauth_server_output_specification`)\nMust be a valid non-nested JSON describing additional fields configured by the Airbyte Instance or Workspace Admins to be used by the\nconnector when using OAuth flow APIs.\nThese fields are to be merged back to `ConnectorSpecification.connectionSpecification`.\nFor each field, a special annotation `path_in_connector_config` can be specified to determine where to merge it,\nExamples:\n complete_oauth_server_output_specification={\n client_id: {\n type: string,\n path_in_connector_config: ['credentials', 'client_id']\n },\n client_secret: {\n type: string,\n path_in_connector_config: ['credentials', 'client_secret']\n }\n }", 1168 examples=[ 1169 { 1170 "client_id": { 1171 "type": "string,", 1172 "path_in_connector_config": ["credentials", "client_id"], 1173 }, 1174 "client_secret": { 1175 "type": "string,", 1176 "path_in_connector_config": ["credentials", "client_secret"], 1177 }, 1178 } 1179 ], 1180 title="OAuth server output specification", 1181 ) 1182 1183 1184class OffsetIncrement(BaseModel): 1185 type: Literal["OffsetIncrement"] 1186 page_size: Optional[Union[int, str]] = Field( 1187 None, 1188 description="The number of records to include in each pages.", 1189 examples=[100, "{{ config['page_size'] }}"], 1190 title="Limit", 1191 ) 1192 inject_on_first_request: Optional[bool] = Field( 1193 False, 1194 description="Using the `offset` with value `0` during the first request", 1195 title="Inject Offset", 1196 ) 1197 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters") 1198 1199 1200class PageIncrement(BaseModel): 1201 type: Literal["PageIncrement"] 1202 page_size: Optional[Union[int, str]] = Field( 1203 None, 1204 description="The number of records to include in each pages.", 1205 examples=[100, "100", "{{ config['page_size'] }}"], 1206 title="Page Size", 1207 ) 1208 start_from_page: Optional[int] = Field( 1209 0, 1210 description="Index of the first page to request.", 1211 examples=[0, 1], 1212 title="Start From Page", 1213 ) 1214 inject_on_first_request: Optional[bool] = Field( 1215 False, 1216 description="Using the `page number` with value defined by `start_from_page` during the first request", 1217 title="Inject Page Number", 1218 ) 1219 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters") 1220 1221 1222class PrimaryKey(BaseModel): 1223 __root__: Union[str, List[str], List[List[str]]] = Field( 1224 ..., 1225 description="The stream field to be used to distinguish unique records. Can either be a single field, an array of fields representing a composite key, or an array of arrays representing a composite key where the fields are nested fields.", 1226 examples=["id", ["code", "type"]], 1227 title="Primary Key", 1228 ) 1229 1230 1231class PropertyLimitType(Enum): 1232 characters = "characters" 1233 property_count = "property_count" 1234 1235 1236class PropertyChunking(BaseModel): 1237 type: Literal["PropertyChunking"] 1238 property_limit_type: PropertyLimitType = Field( 1239 ..., 1240 description="The type used to determine the maximum number of properties per chunk", 1241 title="Property Limit Type", 1242 ) 1243 property_limit: Optional[int] = Field( 1244 None, 1245 description="The maximum amount of properties that can be retrieved per request according to the limit type.", 1246 title="Property Limit", 1247 ) 1248 record_merge_strategy: Optional[GroupByKeyMergeStrategy] = Field( 1249 None, 1250 description="Dictates how to records that require multiple requests to get all properties should be emitted to the destination", 1251 title="Record Merge Strategy", 1252 ) 1253 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters") 1254 1255 1256class RecordFilter(BaseModel): 1257 type: Literal["RecordFilter"] 1258 condition: Optional[str] = Field( 1259 "", 1260 description="The predicate to filter a record. Records will be removed if evaluated to False.", 1261 examples=[ 1262 "{{ record['created_at'] >= stream_interval['start_time'] }}", 1263 "{{ record.status in ['active', 'expired'] }}", 1264 ], 1265 title="Condition", 1266 ) 1267 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters") 1268 1269 1270class SchemaNormalization(Enum): 1271 Default = "Default" 1272 None_ = "None" 1273 1274 1275class RemoveFields(BaseModel): 1276 type: Literal["RemoveFields"] 1277 condition: Optional[str] = Field( 1278 "", 1279 description="The predicate to filter a property by a property value. Property will be removed if it is empty OR expression is evaluated to True.,", 1280 examples=[ 1281 "{{ property|string == '' }}", 1282 "{{ property is integer }}", 1283 "{{ property|length > 5 }}", 1284 "{{ property == 'some_string_to_match' }}", 1285 ], 1286 ) 1287 field_pointers: List[List[str]] = Field( 1288 ..., 1289 description="Array of paths defining the field to remove. Each item is an array whose field describe the path of a field to remove.", 1290 examples=[["tags"], [["content", "html"], ["content", "plain_text"]]], 1291 title="Field Paths", 1292 ) 1293 1294 1295class RequestPath(BaseModel): 1296 type: Literal["RequestPath"] 1297 1298 1299class InjectInto(Enum): 1300 request_parameter = "request_parameter" 1301 header = "header" 1302 body_data = "body_data" 1303 body_json = "body_json" 1304 1305 1306class RequestOption(BaseModel): 1307 type: Literal["RequestOption"] 1308 field_name: Optional[str] = Field( 1309 None, 1310 description="Configures which key should be used in the location that the descriptor is being injected into. We hope to eventually deprecate this field in favor of `field_path` for all request_options, but must currently maintain it for backwards compatibility in the Builder.", 1311 examples=["segment_id"], 1312 title="Field Name", 1313 ) 1314 field_path: Optional[List[str]] = Field( 1315 None, 1316 description="Configures a path to be used for nested structures in JSON body requests (e.g. GraphQL queries)", 1317 examples=[["data", "viewer", "id"]], 1318 title="Field Path", 1319 ) 1320 inject_into: InjectInto = Field( 1321 ..., 1322 description="Configures where the descriptor should be set on the HTTP requests. Note that request parameters that are already encoded in the URL path will not be duplicated.", 1323 examples=["request_parameter", "header", "body_data", "body_json"], 1324 title="Inject Into", 1325 ) 1326 1327 1328class Schemas(BaseModel): 1329 pass 1330 1331 class Config: 1332 extra = Extra.allow 1333 1334 1335class LegacySessionTokenAuthenticator(BaseModel): 1336 type: Literal["LegacySessionTokenAuthenticator"] 1337 header: str = Field( 1338 ..., 1339 description="The name of the session token header that will be injected in the request", 1340 examples=["X-Session"], 1341 title="Session Request Header", 1342 ) 1343 login_url: str = Field( 1344 ..., 1345 description="Path of the login URL (do not include the base URL)", 1346 examples=["session"], 1347 title="Login Path", 1348 ) 1349 session_token: Optional[str] = Field( 1350 None, 1351 description="Session token to use if using a pre-defined token. Not needed if authenticating with username + password pair", 1352 example=["{{ config['session_token'] }}"], 1353 title="Session Token", 1354 ) 1355 session_token_response_key: str = Field( 1356 ..., 1357 description="Name of the key of the session token to be extracted from the response", 1358 examples=["id"], 1359 title="Response Token Response Key", 1360 ) 1361 username: Optional[str] = Field( 1362 None, 1363 description="Username used to authenticate and obtain a session token", 1364 examples=[" {{ config['username'] }}"], 1365 title="Username", 1366 ) 1367 password: Optional[str] = Field( 1368 "", 1369 description="Password used to authenticate and obtain a session token", 1370 examples=["{{ config['password'] }}", ""], 1371 title="Password", 1372 ) 1373 validate_session_url: str = Field( 1374 ..., 1375 description="Path of the URL to use to validate that the session token is valid (do not include the base URL)", 1376 examples=["user/current"], 1377 title="Validate Session Path", 1378 ) 1379 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters") 1380 1381 1382class CsvDecoder(BaseModel): 1383 type: Literal["CsvDecoder"] 1384 encoding: Optional[str] = "utf-8" 1385 delimiter: Optional[str] = "," 1386 set_values_to_none: Optional[List[str]] = None 1387 1388 1389class AsyncJobStatusMap(BaseModel): 1390 type: Optional[Literal["AsyncJobStatusMap"]] = None 1391 running: List[str] 1392 completed: List[str] 1393 failed: List[str] 1394 timeout: List[str] 1395 1396 1397class ValueType(Enum): 1398 string = "string" 1399 number = "number" 1400 integer = "integer" 1401 boolean = "boolean" 1402 1403 1404class WaitTimeFromHeader(BaseModel): 1405 type: Literal["WaitTimeFromHeader"] 1406 header: str = Field( 1407 ..., 1408 description="The name of the response header defining how long to wait before retrying.", 1409 examples=["Retry-After"], 1410 title="Response Header Name", 1411 ) 1412 regex: Optional[str] = Field( 1413 None, 1414 description="Optional regex to apply on the header to extract its value. The regex should define a capture group defining the wait time.", 1415 examples=["([-+]?\\d+)"], 1416 title="Extraction Regex", 1417 ) 1418 max_waiting_time_in_seconds: Optional[float] = Field( 1419 None, 1420 description="Given the value extracted from the header is greater than this value, stop the stream.", 1421 examples=[3600], 1422 title="Max Waiting Time in Seconds", 1423 ) 1424 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters") 1425 1426 1427class WaitUntilTimeFromHeader(BaseModel): 1428 type: Literal["WaitUntilTimeFromHeader"] 1429 header: str = Field( 1430 ..., 1431 description="The name of the response header defining how long to wait before retrying.", 1432 examples=["wait_time"], 1433 title="Response Header", 1434 ) 1435 min_wait: Optional[Union[float, str]] = Field( 1436 None, 1437 description="Minimum time to wait before retrying.", 1438 examples=[10, "60"], 1439 title="Minimum Wait Time", 1440 ) 1441 regex: Optional[str] = Field( 1442 None, 1443 description="Optional regex to apply on the header to extract its value. The regex should define a capture group defining the wait time.", 1444 examples=["([-+]?\\d+)"], 1445 title="Extraction Regex", 1446 ) 1447 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters") 1448 1449 1450class ComponentMappingDefinition(BaseModel): 1451 type: Literal["ComponentMappingDefinition"] 1452 field_path: List[str] = Field( 1453 ..., 1454 description="A list of potentially nested fields indicating the full path where value will be added or updated.", 1455 examples=[ 1456 ["data"], 1457 ["data", "records"], 1458 ["data", 1, "name"], 1459 ["data", "{{ components_values.name }}"], 1460 ["data", "*", "record"], 1461 ["*", "**", "name"], 1462 ], 1463 title="Field Path", 1464 ) 1465 value: str = Field( 1466 ..., 1467 description="The dynamic or static value to assign to the key. Interpolated values can be used to dynamically determine the value during runtime.", 1468 examples=[ 1469 "{{ components_values['updates'] }}", 1470 "{{ components_values['MetaData']['LastUpdatedTime'] }}", 1471 "{{ config['segment_id'] }}", 1472 "{{ stream_slice['parent_id'] }}", 1473 "{{ stream_slice['extra_fields']['name'] }}", 1474 ], 1475 title="Value", 1476 ) 1477 value_type: Optional[ValueType] = Field( 1478 None, 1479 description="The expected data type of the value. If omitted, the type will be inferred from the value provided.", 1480 title="Value Type", 1481 ) 1482 create_or_update: Optional[bool] = Field( 1483 False, 1484 description="Determines whether to create a new path if it doesn't exist (true) or only update existing paths (false). When set to true, the resolver will create new paths in the stream template if they don't exist. When false (default), it will only update existing paths.", 1485 title="Create or Update", 1486 ) 1487 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters") 1488 1489 1490class StreamConfig(BaseModel): 1491 type: Literal["StreamConfig"] 1492 configs_pointer: List[str] = Field( 1493 ..., 1494 description="A list of potentially nested fields indicating the full path in source config file where streams configs located.", 1495 examples=[["data"], ["data", "streams"], ["data", "{{ parameters.name }}"]], 1496 title="Configs Pointer", 1497 ) 1498 default_values: Optional[List[Dict[str, Any]]] = Field( 1499 None, 1500 description="A list of default values, each matching the structure expected from the parsed component value.", 1501 title="Default Values", 1502 ) 1503 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters") 1504 1505 1506class ConfigComponentsResolver(BaseModel): 1507 type: Literal["ConfigComponentsResolver"] 1508 stream_config: Union[List[StreamConfig], StreamConfig] 1509 components_mapping: List[ComponentMappingDefinition] 1510 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters") 1511 1512 1513class StreamParametersDefinition(BaseModel): 1514 type: Literal["StreamParametersDefinition"] 1515 list_of_parameters_for_stream: List[Dict[str, Any]] = Field( 1516 ..., 1517 description="A list of object of parameters for stream, each object in the list represents params for one stream.", 1518 examples=[ 1519 [ 1520 { 1521 "name": "test stream", 1522 "$parameters": {"entity": "test entity"}, 1523 "primary_key": "test key", 1524 } 1525 ] 1526 ], 1527 title="Stream Parameters", 1528 ) 1529 1530 1531class ParametrizedComponentsResolver(BaseModel): 1532 type: Literal["ParametrizedComponentsResolver"] 1533 stream_parameters: StreamParametersDefinition 1534 components_mapping: List[ComponentMappingDefinition] 1535 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters") 1536 1537 1538class RequestBodyPlainText(BaseModel): 1539 type: Literal["RequestBodyPlainText"] 1540 value: str 1541 1542 1543class RequestBodyUrlEncodedForm(BaseModel): 1544 type: Literal["RequestBodyUrlEncodedForm"] 1545 value: Dict[str, str] 1546 1547 1548class RequestBodyJsonObject(BaseModel): 1549 type: Literal["RequestBodyJsonObject"] 1550 value: Dict[str, Any] 1551 1552 1553class RequestBodyGraphQlQuery(BaseModel): 1554 class Config: 1555 extra = Extra.allow 1556 1557 query: Dict[str, Any] = Field(..., description="The GraphQL query to be executed") 1558 1559 1560class ValidateAdheresToSchema(BaseModel): 1561 type: Literal["ValidateAdheresToSchema"] 1562 base_schema: Union[str, Dict[str, Any]] = Field( 1563 ..., 1564 description="The base JSON schema against which the user-provided schema will be validated.", 1565 examples=[ 1566 "{{ config['report_validation_schema'] }}", 1567 '\'{\n "$schema": "http://json-schema.org/draft-07/schema#",\n "title": "Person",\n "type": "object",\n "properties": {\n "name": {\n "type": "string",\n "description": "The person\'s name"\n },\n "age": {\n "type": "integer",\n "minimum": 0,\n "description": "The person\'s age"\n }\n },\n "required": ["name", "age"]\n}\'\n', 1568 { 1569 "$schema": "http://json-schema.org/draft-07/schema#", 1570 "title": "Person", 1571 "type": "object", 1572 "properties": { 1573 "name": {"type": "string", "description": "The person's name"}, 1574 "age": { 1575 "type": "integer", 1576 "minimum": 0, 1577 "description": "The person's age", 1578 }, 1579 }, 1580 "required": ["name", "age"], 1581 }, 1582 ], 1583 title="Base JSON Schema", 1584 ) 1585 1586 1587class CustomValidationStrategy(BaseModel): 1588 class Config: 1589 extra = Extra.allow 1590 1591 type: Literal["CustomValidationStrategy"] 1592 class_name: str = Field( 1593 ..., 1594 description="Fully-qualified name of the class that will be implementing the custom validation strategy. Has to be a sub class of ValidationStrategy. The format is `source_<name>.<package>.<class_name>`.", 1595 examples=["source_declarative_manifest.components.MyCustomValidationStrategy"], 1596 title="Class Name", 1597 ) 1598 1599 1600class ConfigRemapField(BaseModel): 1601 type: Literal["ConfigRemapField"] 1602 map: Union[Dict[str, Any], str] = Field( 1603 ..., 1604 description="A mapping of original values to new values. When a field value matches a key in this map, it will be replaced with the corresponding value.", 1605 examples=[ 1606 {"pending": "in_progress", "done": "completed", "cancelled": "terminated"}, 1607 "{{ config['status_mapping'] }}", 1608 ], 1609 title="Value Mapping", 1610 ) 1611 field_path: List[str] = Field( 1612 ..., 1613 description="The path to the field whose value should be remapped. Specified as a list of path components to navigate through nested objects.", 1614 examples=[ 1615 ["status"], 1616 ["data", "status"], 1617 ["data", "{{ config.name }}", "status"], 1618 ["data", "*", "status"], 1619 ], 1620 title="Field Path", 1621 ) 1622 1623 1624class ConfigRemoveFields(BaseModel): 1625 type: Literal["ConfigRemoveFields"] 1626 field_pointers: List[List[str]] = Field( 1627 ..., 1628 description="A list of field pointers to be removed from the config.", 1629 examples=[["tags"], [["content", "html"], ["content", "plain_text"]]], 1630 title="Field Pointers", 1631 ) 1632 condition: Optional[str] = Field( 1633 "", 1634 description="Fields will be removed if expression is evaluated to True.", 1635 examples=[ 1636 "{{ config['environemnt'] == 'sandbox' }}", 1637 "{{ property is integer }}", 1638 "{{ property|length > 5 }}", 1639 "{{ property == 'some_string_to_match' }}", 1640 ], 1641 ) 1642 1643 1644class AddedFieldDefinition(BaseModel): 1645 type: Literal["AddedFieldDefinition"] 1646 path: List[str] = Field( 1647 ..., 1648 description="List of strings defining the path where to add the value on the record.", 1649 examples=[["segment_id"], ["metadata", "segment_id"]], 1650 title="Path", 1651 ) 1652 value: str = Field( 1653 ..., 1654 description="Value of the new field. Use {{ record['existing_field'] }} syntax to refer to other fields in the record.", 1655 examples=[ 1656 "{{ record['updates'] }}", 1657 "{{ record['MetaData']['LastUpdatedTime'] }}", 1658 "{{ stream_partition['segment_id'] }}", 1659 ], 1660 title="Value", 1661 ) 1662 value_type: Optional[ValueType] = Field( 1663 None, 1664 description="Type of the value. If not specified, the type will be inferred from the value.", 1665 title="Value Type", 1666 ) 1667 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters") 1668 1669 1670class AddFields(BaseModel): 1671 type: Literal["AddFields"] 1672 fields: List[AddedFieldDefinition] = Field( 1673 ..., 1674 description="List of transformations (path and corresponding value) that will be added to the record.", 1675 title="Fields", 1676 ) 1677 condition: Optional[str] = Field( 1678 "", 1679 description="Fields will be added if expression is evaluated to True.", 1680 examples=[ 1681 "{{ property|string == '' }}", 1682 "{{ property is integer }}", 1683 "{{ property|length > 5 }}", 1684 "{{ property == 'some_string_to_match' }}", 1685 ], 1686 ) 1687 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters") 1688 1689 1690class ApiKeyAuthenticator(BaseModel): 1691 type: Literal["ApiKeyAuthenticator"] 1692 api_token: Optional[str] = Field( 1693 None, 1694 description="The API key to inject in the request. Fill it in the user inputs.", 1695 examples=["{{ config['api_key'] }}", "Token token={{ config['api_key'] }}"], 1696 title="API Key", 1697 ) 1698 header: Optional[str] = Field( 1699 None, 1700 description="The name of the HTTP header that will be set to the API key. This setting is deprecated, use inject_into instead. Header and inject_into can not be defined at the same time.", 1701 examples=["Authorization", "Api-Token", "X-Auth-Token"], 1702 title="Header Name", 1703 ) 1704 inject_into: Optional[RequestOption] = Field( 1705 None, 1706 description="Configure how the API Key will be sent in requests to the source API. Either inject_into or header has to be defined.", 1707 examples=[ 1708 {"inject_into": "header", "field_name": "Authorization"}, 1709 {"inject_into": "request_parameter", "field_name": "authKey"}, 1710 ], 1711 title="Inject API Key Into Outgoing HTTP Request", 1712 ) 1713 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters") 1714 1715 1716class AuthFlow(BaseModel): 1717 auth_flow_type: Optional[AuthFlowType] = Field( 1718 None, description="The type of auth to use", title="Auth flow type" 1719 ) 1720 predicate_key: Optional[List[str]] = Field( 1721 None, 1722 description="JSON path to a field in the connectorSpecification that should exist for the advanced auth to be applicable.", 1723 examples=[["credentials", "auth_type"]], 1724 title="Predicate key", 1725 ) 1726 predicate_value: Optional[str] = Field( 1727 None, 1728 description="Value of the predicate_key fields for the advanced auth to be applicable.", 1729 examples=["Oauth"], 1730 title="Predicate value", 1731 ) 1732 oauth_config_specification: Optional[OAuthConfigSpecification] = None 1733 1734 1735class CheckStream(BaseModel): 1736 type: Literal["CheckStream"] 1737 stream_names: Optional[List[str]] = Field( 1738 None, 1739 description="Names of the streams to try reading from when running a check operation.", 1740 examples=[["users"], ["users", "contacts"]], 1741 title="Stream Names", 1742 ) 1743 dynamic_streams_check_configs: Optional[List[DynamicStreamCheckConfig]] = None 1744 1745 1746class IncrementingCountCursor(BaseModel): 1747 type: Literal["IncrementingCountCursor"] 1748 cursor_field: str = Field( 1749 ..., 1750 description="The location of the value on a record that will be used as a bookmark during sync. To ensure no data loss, the API must return records in ascending order based on the cursor field. Nested fields are not supported, so the field must be at the top level of the record. You can use a combination of Add Field and Remove Field transformations to move the nested field to the top.", 1751 examples=["created_at", "{{ config['record_cursor'] }}"], 1752 title="Cursor Field", 1753 ) 1754 start_value: Optional[Union[str, int]] = Field( 1755 None, 1756 description="The value that determines the earliest record that should be synced.", 1757 examples=[0, "{{ config['start_value'] }}"], 1758 title="Start Value", 1759 ) 1760 start_value_option: Optional[RequestOption] = Field( 1761 None, 1762 description="Optionally configures how the start value will be sent in requests to the source API.", 1763 title="Inject Start Value Into Outgoing HTTP Request", 1764 ) 1765 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters") 1766 1767 1768class DatetimeBasedCursor(BaseModel): 1769 type: Literal["DatetimeBasedCursor"] 1770 clamping: Optional[Clamping] = Field( 1771 None, 1772 description="This option is used to adjust the upper and lower boundaries of each datetime window to beginning and end of the provided target period (day, week, month)", 1773 title="Date Range Clamping", 1774 ) 1775 cursor_field: str = Field( 1776 ..., 1777 description="The location of the value on a record that will be used as a bookmark during sync. To ensure no data loss, the API must return records in ascending order based on the cursor field. Nested fields are not supported, so the field must be at the top level of the record. You can use a combination of Add Field and Remove Field transformations to move the nested field to the top.", 1778 examples=["created_at", "{{ config['record_cursor'] }}"], 1779 title="Cursor Field", 1780 ) 1781 cursor_datetime_formats: Optional[List[str]] = Field( 1782 None, 1783 description="The possible formats for the cursor field, in order of preference. The first format that matches the cursor field value will be used to parse it. If not provided, the `datetime_format` will be used.", 1784 title="Cursor Datetime Formats", 1785 ) 1786 start_datetime: Union[MinMaxDatetime, str] = Field( 1787 ..., 1788 description="The datetime that determines the earliest record that should be synced.", 1789 examples=["2020-01-1T00:00:00Z", "{{ config['start_time'] }}"], 1790 title="Start Datetime", 1791 ) 1792 start_time_option: Optional[RequestOption] = Field( 1793 None, 1794 description="Optionally configures how the start datetime will be sent in requests to the source API.", 1795 title="Inject Start Time Into Outgoing HTTP Request", 1796 ) 1797 end_datetime: Optional[Union[MinMaxDatetime, str]] = Field( 1798 None, 1799 description="The datetime that determines the last record that should be synced. If not provided, `{{ now_utc() }}` will be used.", 1800 examples=["2021-01-1T00:00:00Z", "{{ now_utc() }}", "{{ day_delta(-1) }}"], 1801 title="End Datetime", 1802 ) 1803 end_time_option: Optional[RequestOption] = Field( 1804 None, 1805 description="Optionally configures how the end datetime will be sent in requests to the source API.", 1806 title="Inject End Time Into Outgoing HTTP Request", 1807 ) 1808 datetime_format: str = Field( 1809 ..., 1810 description="The datetime format used to format the datetime values that are sent in outgoing requests to the API. Use placeholders starting with \"%\" to describe the format the API is using. The following placeholders are available:\n * **%s**: Epoch unix timestamp - `1686218963`\n * **%s_as_float**: Epoch unix timestamp in seconds as float with microsecond precision - `1686218963.123456`\n * **%ms**: Epoch unix timestamp (milliseconds) - `1686218963123`\n * **%a**: Weekday (abbreviated) - `Sun`\n * **%A**: Weekday (full) - `Sunday`\n * **%w**: Weekday (decimal) - `0` (Sunday), `6` (Saturday)\n * **%d**: Day of the month (zero-padded) - `01`, `02`, ..., `31`\n * **%b**: Month (abbreviated) - `Jan`\n * **%B**: Month (full) - `January`\n * **%m**: Month (zero-padded) - `01`, `02`, ..., `12`\n * **%y**: Year (without century, zero-padded) - `00`, `01`, ..., `99`\n * **%Y**: Year (with century) - `0001`, `0002`, ..., `9999`\n * **%H**: Hour (24-hour, zero-padded) - `00`, `01`, ..., `23`\n * **%I**: Hour (12-hour, zero-padded) - `01`, `02`, ..., `12`\n * **%p**: AM/PM indicator\n * **%M**: Minute (zero-padded) - `00`, `01`, ..., `59`\n * **%S**: Second (zero-padded) - `00`, `01`, ..., `59`\n * **%f**: Microsecond (zero-padded to 6 digits) - `000000`\n * **%_ms**: Millisecond (zero-padded to 3 digits) - `000`\n * **%z**: UTC offset - `(empty)`, `+0000`, `-04:00`\n * **%Z**: Time zone name - `(empty)`, `UTC`, `GMT`\n * **%j**: Day of the year (zero-padded) - `001`, `002`, ..., `366`\n * **%U**: Week number of the year (starting Sunday) - `00`, ..., `53`\n * **%W**: Week number of the year (starting Monday) - `00`, ..., `53`\n * **%c**: Date and time - `Tue Aug 16 21:30:00 1988`\n * **%x**: Date standard format - `08/16/1988`\n * **%X**: Time standard format - `21:30:00`\n * **%%**: Literal '%' character\n\n Some placeholders depend on the locale of the underlying system - in most cases this locale is configured as en/US. For more information see the [Python documentation](https://docs.python.org/3/library/datetime.html#strftime-and-strptime-format-codes).\n", 1811 examples=["%Y-%m-%dT%H:%M:%S.%f%z", "%Y-%m-%d", "%s", "%ms", "%s_as_float"], 1812 title="Outgoing Datetime Format", 1813 ) 1814 cursor_granularity: Optional[str] = Field( 1815 None, 1816 description="Smallest increment the datetime_format has (ISO 8601 duration) that is used to ensure the start of a slice does not overlap with the end of the previous one, e.g. for %Y-%m-%d the granularity should be P1D, for %Y-%m-%dT%H:%M:%SZ the granularity should be PT1S. Given this field is provided, `step` needs to be provided as well.", 1817 examples=["PT1S"], 1818 title="Cursor Granularity", 1819 ) 1820 is_data_feed: Optional[bool] = Field( 1821 None, 1822 description="A data feed API is an API that does not allow filtering and paginates the content from the most recent to the least recent. Given this, the CDK needs to know when to stop paginating and this field will generate a stop condition for pagination.", 1823 title="Whether the target API is formatted as a data feed", 1824 ) 1825 is_client_side_incremental: Optional[bool] = Field( 1826 None, 1827 description="If the target API endpoint does not take cursor values to filter records and returns all records anyway, the connector with this cursor will filter out records locally, and only emit new records from the last sync, hence incremental. This means that all records would be read from the API, but only new records will be emitted to the destination.", 1828 title="Whether the target API does not support filtering and returns all data (the cursor filters records in the client instead of the API side)", 1829 ) 1830 is_compare_strictly: Optional[bool] = Field( 1831 False, 1832 description="Set to True if the target API does not accept queries where the start time equal the end time.", 1833 title="Whether to skip requests if the start time equals the end time", 1834 ) 1835 global_substream_cursor: Optional[bool] = Field( 1836 False, 1837 description="This setting optimizes performance when the parent stream has thousands of partitions by storing the cursor as a single value rather than per partition. Notably, the substream state is updated only at the end of the sync, which helps prevent data loss in case of a sync failure. See more info in the [docs](https://docs.airbyte.com/connector-development/config-based/understanding-the-yaml-file/incremental-syncs).", 1838 title="Whether to store cursor as one value instead of per partition", 1839 ) 1840 lookback_window: Optional[str] = Field( 1841 None, 1842 description="Time interval before the start_datetime to read data for, e.g. P1M for looking back one month.", 1843 examples=["P1D", "P{{ config['lookback_days'] }}D"], 1844 title="Lookback Window", 1845 ) 1846 partition_field_end: Optional[str] = Field( 1847 None, 1848 description="Name of the partition start time field.", 1849 examples=["ending_time"], 1850 title="Partition Field End", 1851 ) 1852 partition_field_start: Optional[str] = Field( 1853 None, 1854 description="Name of the partition end time field.", 1855 examples=["starting_time"], 1856 title="Partition Field Start", 1857 ) 1858 step: Optional[str] = Field( 1859 None, 1860 description="The size of the time window (ISO8601 duration). Given this field is provided, `cursor_granularity` needs to be provided as well.", 1861 examples=["P1W", "{{ config['step_increment'] }}"], 1862 title="Step", 1863 ) 1864 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters") 1865 1866 1867class FixedWindowCallRatePolicy(BaseModel): 1868 class Config: 1869 extra = Extra.allow 1870 1871 type: Literal["FixedWindowCallRatePolicy"] 1872 period: str = Field( 1873 ..., description="The time interval for the rate limit window.", title="Period" 1874 ) 1875 call_limit: int = Field( 1876 ..., 1877 description="The maximum number of calls allowed within the period.", 1878 title="Call Limit", 1879 ) 1880 matchers: List[HttpRequestRegexMatcher] = Field( 1881 ..., 1882 description="List of matchers that define which requests this policy applies to.", 1883 title="Matchers", 1884 ) 1885 1886 1887class MovingWindowCallRatePolicy(BaseModel): 1888 class Config: 1889 extra = Extra.allow 1890 1891 type: Literal["MovingWindowCallRatePolicy"] 1892 rates: List[Rate] = Field( 1893 ..., 1894 description="List of rates that define the call limits for different time intervals.", 1895 title="Rates", 1896 ) 1897 matchers: List[HttpRequestRegexMatcher] = Field( 1898 ..., 1899 description="List of matchers that define which requests this policy applies to.", 1900 title="Matchers", 1901 ) 1902 1903 1904class UnlimitedCallRatePolicy(BaseModel): 1905 class Config: 1906 extra = Extra.allow 1907 1908 type: Literal["UnlimitedCallRatePolicy"] 1909 matchers: List[HttpRequestRegexMatcher] = Field( 1910 ..., 1911 description="List of matchers that define which requests this policy applies to.", 1912 title="Matchers", 1913 ) 1914 1915 1916class DefaultErrorHandler(BaseModel): 1917 type: Literal["DefaultErrorHandler"] 1918 backoff_strategies: Optional[ 1919 List[ 1920 Union[ 1921 ConstantBackoffStrategy, 1922 ExponentialBackoffStrategy, 1923 WaitTimeFromHeader, 1924 WaitUntilTimeFromHeader, 1925 CustomBackoffStrategy, 1926 ] 1927 ] 1928 ] = Field( 1929 None, 1930 description="List of backoff strategies to use to determine how long to wait before retrying a retryable request.", 1931 title="Backoff Strategies", 1932 ) 1933 max_retries: Optional[int] = Field( 1934 5, 1935 description="The maximum number of time to retry a retryable request before giving up and failing.", 1936 examples=[5, 0, 10], 1937 title="Max Retry Count", 1938 ) 1939 response_filters: Optional[List[HttpResponseFilter]] = Field( 1940 None, 1941 description="List of response filters to iterate on when deciding how to handle an error. When using an array of multiple filters, the filters will be applied sequentially and the response will be selected if it matches any of the filter's predicate.", 1942 title="Response Filters", 1943 ) 1944 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters") 1945 1946 1947class DefaultPaginator(BaseModel): 1948 type: Literal["DefaultPaginator"] 1949 pagination_strategy: Union[ 1950 PageIncrement, OffsetIncrement, CursorPagination, CustomPaginationStrategy 1951 ] = Field( 1952 ..., 1953 description="Strategy defining how records are paginated.", 1954 title="Pagination Strategy", 1955 ) 1956 page_size_option: Optional[RequestOption] = Field( 1957 None, title="Inject Page Size Into Outgoing HTTP Request" 1958 ) 1959 page_token_option: Optional[Union[RequestOption, RequestPath]] = Field( 1960 None, title="Inject Page Token Into Outgoing HTTP Request" 1961 ) 1962 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters") 1963 1964 1965class SessionTokenRequestApiKeyAuthenticator(BaseModel): 1966 type: Literal["ApiKey"] 1967 inject_into: RequestOption = Field( 1968 ..., 1969 description="Configure how the API Key will be sent in requests to the source API.", 1970 examples=[ 1971 {"inject_into": "header", "field_name": "Authorization"}, 1972 {"inject_into": "request_parameter", "field_name": "authKey"}, 1973 ], 1974 title="Inject API Key Into Outgoing HTTP Request", 1975 ) 1976 1977 1978class ListPartitionRouter(BaseModel): 1979 type: Literal["ListPartitionRouter"] 1980 cursor_field: str = Field( 1981 ..., 1982 description='While iterating over list values, the name of field used to reference a list value. The partition value can be accessed with string interpolation. e.g. "{{ stream_partition[\'my_key\'] }}" where "my_key" is the value of the cursor_field.', 1983 examples=["section", "{{ config['section_key'] }}"], 1984 title="Current Partition Value Identifier", 1985 ) 1986 values: Union[str, List[str]] = Field( 1987 ..., 1988 description="The list of attributes being iterated over and used as input for the requests made to the source API.", 1989 examples=[["section_a", "section_b", "section_c"], "{{ config['sections'] }}"], 1990 title="Partition Values", 1991 ) 1992 request_option: Optional[RequestOption] = Field( 1993 None, 1994 description="A request option describing where the list value should be injected into and under what field name if applicable.", 1995 title="Inject Partition Value Into Outgoing HTTP Request", 1996 ) 1997 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters") 1998 1999 2000class RecordSelector(BaseModel): 2001 type: Literal["RecordSelector"] 2002 extractor: Union[DpathExtractor, CustomRecordExtractor] 2003 record_filter: Optional[Union[RecordFilter, CustomRecordFilter]] = Field( 2004 None, 2005 description="Responsible for filtering records to be emitted by the Source.", 2006 title="Record Filter", 2007 ) 2008 schema_normalization: Optional[Union[SchemaNormalization, CustomSchemaNormalization]] = Field( 2009 None, 2010 description="Responsible for normalization according to the schema.", 2011 title="Schema Normalization", 2012 ) 2013 transform_before_filtering: Optional[bool] = Field( 2014 None, 2015 description="If true, transformation will be applied before record filtering.", 2016 title="Transform Before Filtering", 2017 ) 2018 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters") 2019 2020 2021class GzipDecoder(BaseModel): 2022 type: Literal["GzipDecoder"] 2023 decoder: Union[CsvDecoder, GzipDecoder, JsonDecoder, JsonlDecoder] 2024 2025 2026class RequestBodyGraphQL(BaseModel): 2027 type: Literal["RequestBodyGraphQL"] 2028 value: RequestBodyGraphQlQuery 2029 2030 2031class DpathValidator(BaseModel): 2032 type: Literal["DpathValidator"] 2033 field_path: List[str] = Field( 2034 ..., 2035 description='List of potentially nested fields describing the full path of the field to validate. Use "*" to validate all values from an array.', 2036 examples=[ 2037 ["data"], 2038 ["data", "records"], 2039 ["data", "{{ parameters.name }}"], 2040 ["data", "*", "record"], 2041 ], 2042 title="Field Path", 2043 ) 2044 validation_strategy: Union[ValidateAdheresToSchema, CustomValidationStrategy] = Field( 2045 ..., 2046 description="The condition that the specified config value will be evaluated against", 2047 title="Validation Strategy", 2048 ) 2049 2050 2051class PredicateValidator(BaseModel): 2052 type: Literal["PredicateValidator"] 2053 value: Optional[Union[str, float, Dict[str, Any], List[Any], bool]] = Field( 2054 ..., 2055 description="The value to be validated. Can be a literal value or interpolated from configuration.", 2056 examples=[ 2057 "test-value", 2058 "{{ config['api_version'] }}", 2059 "{{ config['tenant_id'] }}", 2060 123, 2061 ], 2062 title="Value", 2063 ) 2064 validation_strategy: Union[ValidateAdheresToSchema, CustomValidationStrategy] = Field( 2065 ..., 2066 description="The validation strategy to apply to the value.", 2067 title="Validation Strategy", 2068 ) 2069 2070 2071class ConfigAddFields(BaseModel): 2072 type: Literal["ConfigAddFields"] 2073 fields: List[AddedFieldDefinition] = Field( 2074 ..., 2075 description="A list of transformations (path and corresponding value) that will be added to the config.", 2076 title="Fields", 2077 ) 2078 condition: Optional[str] = Field( 2079 "", 2080 description="Fields will be added if expression is evaluated to True.", 2081 examples=[ 2082 "{{ config['environemnt'] == 'sandbox' }}", 2083 "{{ property is integer }}", 2084 "{{ property|length > 5 }}", 2085 "{{ property == 'some_string_to_match' }}", 2086 ], 2087 ) 2088 2089 2090class CompositeErrorHandler(BaseModel): 2091 type: Literal["CompositeErrorHandler"] 2092 error_handlers: List[Union[CompositeErrorHandler, DefaultErrorHandler]] = Field( 2093 ..., 2094 description="List of error handlers to iterate on to determine how to handle a failed response.", 2095 title="Error Handlers", 2096 ) 2097 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters") 2098 2099 2100class HTTPAPIBudget(BaseModel): 2101 class Config: 2102 extra = Extra.allow 2103 2104 type: Literal["HTTPAPIBudget"] 2105 policies: List[ 2106 Union[ 2107 FixedWindowCallRatePolicy, 2108 MovingWindowCallRatePolicy, 2109 UnlimitedCallRatePolicy, 2110 ] 2111 ] = Field( 2112 ..., 2113 description="List of call rate policies that define how many calls are allowed.", 2114 title="Policies", 2115 ) 2116 ratelimit_reset_header: Optional[str] = Field( 2117 "ratelimit-reset", 2118 description="The HTTP response header name that indicates when the rate limit resets.", 2119 title="Rate Limit Reset Header", 2120 ) 2121 ratelimit_remaining_header: Optional[str] = Field( 2122 "ratelimit-remaining", 2123 description="The HTTP response header name that indicates the number of remaining allowed calls.", 2124 title="Rate Limit Remaining Header", 2125 ) 2126 status_codes_for_ratelimit_hit: Optional[List[int]] = Field( 2127 [429], 2128 description="List of HTTP status codes that indicate a rate limit has been hit.", 2129 title="Status Codes for Rate Limit Hit", 2130 ) 2131 2132 2133class ZipfileDecoder(BaseModel): 2134 class Config: 2135 extra = Extra.allow 2136 2137 type: Literal["ZipfileDecoder"] 2138 decoder: Union[CsvDecoder, GzipDecoder, JsonDecoder, JsonlDecoder] = Field( 2139 ..., 2140 description="Parser to parse the decompressed data from the zipfile(s).", 2141 title="Parser", 2142 ) 2143 2144 2145class ConfigMigration(BaseModel): 2146 type: Literal["ConfigMigration"] 2147 description: Optional[str] = Field( 2148 None, description="The description/purpose of the config migration." 2149 ) 2150 transformations: List[Union[ConfigRemapField, ConfigAddFields, ConfigRemoveFields]] = Field( 2151 ..., 2152 description="The list of transformations that will attempt to be applied on an incoming unmigrated config. The transformations will be applied in the order they are defined.", 2153 title="Transformations", 2154 ) 2155 2156 2157class ConfigNormalizationRules(BaseModel): 2158 class Config: 2159 extra = Extra.forbid 2160 2161 config_migrations: Optional[List[ConfigMigration]] = Field( 2162 [], 2163 description="The discrete migrations that will be applied on the incoming config. Each migration will be applied in the order they are defined.", 2164 title="Config Migrations", 2165 ) 2166 transformations: Optional[ 2167 List[Union[ConfigRemapField, ConfigAddFields, ConfigRemoveFields]] 2168 ] = Field( 2169 [], 2170 description="The list of transformations that will be applied on the incoming config at the start of each sync. The transformations will be applied in the order they are defined.", 2171 title="Transformations", 2172 ) 2173 validations: Optional[List[Union[DpathValidator, PredicateValidator]]] = Field( 2174 [], 2175 description="The list of validations that will be performed on the incoming config at the start of each sync.", 2176 title="Validations", 2177 ) 2178 2179 2180class Spec(BaseModel): 2181 type: Literal["Spec"] 2182 connection_specification: Dict[str, Any] = Field( 2183 ..., 2184 description="A connection specification describing how a the connector can be configured.", 2185 title="Connection Specification", 2186 ) 2187 documentation_url: Optional[str] = Field( 2188 None, 2189 description="URL of the connector's documentation page.", 2190 examples=["https://docs.airbyte.com/integrations/sources/dremio"], 2191 title="Documentation URL", 2192 ) 2193 advanced_auth: Optional[AuthFlow] = Field( 2194 None, 2195 description="Advanced specification for configuring the authentication flow.", 2196 title="Advanced Auth", 2197 ) 2198 config_normalization_rules: Optional[ConfigNormalizationRules] = Field( 2199 None, title="Config Normalization Rules" 2200 ) 2201 2202 2203class DeclarativeSource1(BaseModel): 2204 class Config: 2205 extra = Extra.forbid 2206 2207 type: Literal["DeclarativeSource"] 2208 check: Union[CheckStream, CheckDynamicStream] 2209 streams: List[Union[ConditionalStreams, DeclarativeStream, StateDelegatingStream]] 2210 dynamic_streams: Optional[List[DynamicDeclarativeStream]] = None 2211 version: str = Field( 2212 ..., 2213 description="The version of the Airbyte CDK used to build and test the source.", 2214 ) 2215 schemas: Optional[Schemas] = None 2216 definitions: Optional[Dict[str, Any]] = None 2217 spec: Optional[Spec] = None 2218 concurrency_level: Optional[ConcurrencyLevel] = None 2219 api_budget: Optional[HTTPAPIBudget] = None 2220 max_concurrent_async_job_count: Optional[Union[int, str]] = Field( 2221 None, 2222 description="Maximum number of concurrent asynchronous jobs to run. This property is only relevant for sources/streams that support asynchronous job execution through the AsyncRetriever (e.g. a report-based stream that initiates a job, polls the job status, and then fetches the job results). This is often set by the API's maximum number of concurrent jobs on the account level. Refer to the API's documentation for this information.", 2223 examples=[3, "{{ config['max_concurrent_async_job_count'] }}"], 2224 title="Maximum Concurrent Asynchronous Jobs", 2225 ) 2226 metadata: Optional[Dict[str, Any]] = Field( 2227 None, 2228 description="For internal Airbyte use only - DO NOT modify manually. Used by consumers of declarative manifests for storing related metadata.", 2229 ) 2230 description: Optional[str] = Field( 2231 None, 2232 description="A description of the connector. It will be presented on the Source documentation page.", 2233 ) 2234 2235 2236class DeclarativeSource2(BaseModel): 2237 class Config: 2238 extra = Extra.forbid 2239 2240 type: Literal["DeclarativeSource"] 2241 check: Union[CheckStream, CheckDynamicStream] 2242 streams: Optional[List[Union[ConditionalStreams, DeclarativeStream, StateDelegatingStream]]] = ( 2243 None 2244 ) 2245 dynamic_streams: List[DynamicDeclarativeStream] 2246 version: str = Field( 2247 ..., 2248 description="The version of the Airbyte CDK used to build and test the source.", 2249 ) 2250 schemas: Optional[Schemas] = None 2251 definitions: Optional[Dict[str, Any]] = None 2252 spec: Optional[Spec] = None 2253 concurrency_level: Optional[ConcurrencyLevel] = None 2254 api_budget: Optional[HTTPAPIBudget] = None 2255 max_concurrent_async_job_count: Optional[Union[int, str]] = Field( 2256 None, 2257 description="Maximum number of concurrent asynchronous jobs to run. This property is only relevant for sources/streams that support asynchronous job execution through the AsyncRetriever (e.g. a report-based stream that initiates a job, polls the job status, and then fetches the job results). This is often set by the API's maximum number of concurrent jobs on the account level. Refer to the API's documentation for this information.", 2258 examples=[3, "{{ config['max_concurrent_async_job_count'] }}"], 2259 title="Maximum Concurrent Asynchronous Jobs", 2260 ) 2261 metadata: Optional[Dict[str, Any]] = Field( 2262 None, 2263 description="For internal Airbyte use only - DO NOT modify manually. Used by consumers of declarative manifests for storing related metadata.", 2264 ) 2265 description: Optional[str] = Field( 2266 None, 2267 description="A description of the connector. It will be presented on the Source documentation page.", 2268 ) 2269 2270 2271class DeclarativeSource(BaseModel): 2272 class Config: 2273 extra = Extra.forbid 2274 2275 __root__: Union[DeclarativeSource1, DeclarativeSource2] = Field( 2276 ..., 2277 description="An API source that extracts data according to its declarative components.", 2278 title="DeclarativeSource", 2279 ) 2280 2281 2282class SelectiveAuthenticator(BaseModel): 2283 class Config: 2284 extra = Extra.allow 2285 2286 type: Literal["SelectiveAuthenticator"] 2287 authenticator_selection_path: List[str] = Field( 2288 ..., 2289 description="Path of the field in config with selected authenticator name", 2290 examples=[["auth"], ["auth", "type"]], 2291 title="Authenticator Selection Path", 2292 ) 2293 authenticators: Dict[ 2294 str, 2295 Union[ 2296 ApiKeyAuthenticator, 2297 BasicHttpAuthenticator, 2298 BearerAuthenticator, 2299 OAuthAuthenticator, 2300 JwtAuthenticator, 2301 SessionTokenAuthenticator, 2302 LegacySessionTokenAuthenticator, 2303 CustomAuthenticator, 2304 NoAuth, 2305 ], 2306 ] = Field( 2307 ..., 2308 description="Authenticators to select from.", 2309 examples=[ 2310 { 2311 "authenticators": { 2312 "token": "#/definitions/ApiKeyAuthenticator", 2313 "oauth": "#/definitions/OAuthAuthenticator", 2314 "jwt": "#/definitions/JwtAuthenticator", 2315 } 2316 } 2317 ], 2318 title="Authenticators", 2319 ) 2320 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters") 2321 2322 2323class ConditionalStreams(BaseModel): 2324 type: Literal["ConditionalStreams"] 2325 condition: str = Field( 2326 ..., 2327 description="Condition that will be evaluated to determine if a set of streams should be available.", 2328 examples=["{{ config['is_sandbox'] }}"], 2329 title="Condition", 2330 ) 2331 streams: List[DeclarativeStream] = Field( 2332 ..., 2333 description="Streams that will be used during an operation based on the condition.", 2334 title="Streams", 2335 ) 2336 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters") 2337 2338 2339class FileUploader(BaseModel): 2340 type: Literal["FileUploader"] 2341 requester: Union[HttpRequester, CustomRequester] = Field( 2342 ..., 2343 description="Requester component that describes how to prepare HTTP requests to send to the source API.", 2344 ) 2345 download_target_extractor: Union[DpathExtractor, CustomRecordExtractor] = Field( 2346 ..., 2347 description="Responsible for fetching the url where the file is located. This is applied on each records and not on the HTTP response", 2348 ) 2349 file_extractor: Optional[Union[DpathExtractor, CustomRecordExtractor]] = Field( 2350 None, 2351 description="Responsible for fetching the content of the file. If not defined, the assumption is that the whole response body is the file content", 2352 ) 2353 filename_extractor: Optional[str] = Field( 2354 None, 2355 description="Defines the name to store the file. Stream name is automatically added to the file path. File unique ID can be used to avoid overwriting files. Random UUID will be used if the extractor is not provided.", 2356 examples=[ 2357 "{{ record.id }}/{{ record.file_name }}/", 2358 "{{ record.id }}_{{ record.file_name }}/", 2359 ], 2360 ) 2361 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters") 2362 2363 2364class DeclarativeStream(BaseModel): 2365 class Config: 2366 extra = Extra.allow 2367 2368 type: Literal["DeclarativeStream"] 2369 name: Optional[str] = Field("", description="The stream name.", example=["Users"], title="Name") 2370 retriever: Union[SimpleRetriever, AsyncRetriever, CustomRetriever] = Field( 2371 ..., 2372 description="Component used to coordinate how records are extracted across stream slices and request pages.", 2373 title="Retriever", 2374 ) 2375 incremental_sync: Optional[ 2376 Union[DatetimeBasedCursor, IncrementingCountCursor, CustomIncrementalSync] 2377 ] = Field( 2378 None, 2379 description="Component used to fetch data incrementally based on a time field in the data.", 2380 title="Incremental Sync", 2381 ) 2382 primary_key: Optional[PrimaryKey] = Field("", title="Primary Key") 2383 schema_loader: Optional[ 2384 Union[ 2385 InlineSchemaLoader, 2386 DynamicSchemaLoader, 2387 JsonFileSchemaLoader, 2388 List[ 2389 Union[ 2390 InlineSchemaLoader, 2391 DynamicSchemaLoader, 2392 JsonFileSchemaLoader, 2393 CustomSchemaLoader, 2394 ] 2395 ], 2396 CustomSchemaLoader, 2397 ] 2398 ] = Field( 2399 None, 2400 description="One or many schema loaders can be used to retrieve the schema for the current stream. When multiple schema loaders are defined, schema properties will be merged together. Schema loaders defined first taking precedence in the event of a conflict.", 2401 title="Schema Loader", 2402 ) 2403 transformations: Optional[ 2404 List[ 2405 Union[ 2406 AddFields, 2407 RemoveFields, 2408 KeysToLower, 2409 KeysToSnakeCase, 2410 FlattenFields, 2411 DpathFlattenFields, 2412 KeysReplace, 2413 CustomTransformation, 2414 ] 2415 ] 2416 ] = Field( 2417 None, 2418 description="A list of transformations to be applied to each output record.", 2419 title="Transformations", 2420 ) 2421 state_migrations: Optional[ 2422 List[Union[LegacyToPerPartitionStateMigration, CustomStateMigration]] 2423 ] = Field( 2424 [], 2425 description="Array of state migrations to be applied on the input state", 2426 title="State Migrations", 2427 ) 2428 file_uploader: Optional[FileUploader] = Field( 2429 None, 2430 description="(experimental) Describes how to fetch a file", 2431 title="File Uploader", 2432 ) 2433 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters") 2434 2435 2436class SessionTokenAuthenticator(BaseModel): 2437 type: Literal["SessionTokenAuthenticator"] 2438 login_requester: HttpRequester = Field( 2439 ..., 2440 description="Description of the request to perform to obtain a session token to perform data requests. The response body is expected to be a JSON object with a session token property.", 2441 examples=[ 2442 { 2443 "type": "HttpRequester", 2444 "url_base": "https://my_api.com", 2445 "path": "/login", 2446 "authenticator": { 2447 "type": "BasicHttpAuthenticator", 2448 "username": "{{ config.username }}", 2449 "password": "{{ config.password }}", 2450 }, 2451 } 2452 ], 2453 title="Login Requester", 2454 ) 2455 session_token_path: List[str] = Field( 2456 ..., 2457 description="The path in the response body returned from the login requester to the session token.", 2458 examples=[["access_token"], ["result", "token"]], 2459 title="Session Token Path", 2460 ) 2461 expiration_duration: Optional[str] = Field( 2462 None, 2463 description="The duration in ISO 8601 duration notation after which the session token expires, starting from the time it was obtained. Omitting it will result in the session token being refreshed for every request.", 2464 examples=["PT1H", "P1D"], 2465 title="Expiration Duration", 2466 ) 2467 request_authentication: Union[ 2468 SessionTokenRequestApiKeyAuthenticator, SessionTokenRequestBearerAuthenticator 2469 ] = Field( 2470 ..., 2471 description="Authentication method to use for requests sent to the API, specifying how to inject the session token.", 2472 title="Data Request Authentication", 2473 ) 2474 decoder: Optional[Union[JsonDecoder, XmlDecoder]] = Field( 2475 None, description="Component used to decode the response.", title="Decoder" 2476 ) 2477 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters") 2478 2479 2480class HttpRequester(BaseModelWithDeprecations): 2481 type: Literal["HttpRequester"] 2482 url_base: Optional[str] = Field( 2483 None, 2484 deprecated=True, 2485 deprecation_message="Use `url` field instead.", 2486 description="Deprecated, use the `url` instead. Base URL of the API source. Do not put sensitive information (e.g. API tokens) into this field - Use the Authenticator component for this.", 2487 examples=[ 2488 "https://connect.squareup.com/v2", 2489 "{{ config['base_url'] or 'https://app.posthog.com'}}/api", 2490 "https://connect.squareup.com/v2/quotes/{{ stream_partition['id'] }}/quote_line_groups", 2491 "https://example.com/api/v1/resource/{{ next_page_token['id'] }}", 2492 ], 2493 title="API Base URL", 2494 ) 2495 url: Optional[str] = Field( 2496 None, 2497 description="The URL of the source API endpoint. Do not put sensitive information (e.g. API tokens) into this field - Use the Authenticator component for this.", 2498 examples=[ 2499 "https://connect.squareup.com/v2", 2500 "{{ config['url'] or 'https://app.posthog.com'}}/api", 2501 "https://connect.squareup.com/v2/quotes/{{ stream_partition['id'] }}/quote_line_groups", 2502 "https://example.com/api/v1/resource/{{ next_page_token['id'] }}", 2503 ], 2504 title="API Endpoint URL", 2505 ) 2506 path: Optional[str] = Field( 2507 None, 2508 deprecated=True, 2509 deprecation_message="Use `url` field instead.", 2510 description="Deprecated, use the `url` instead. Path the specific API endpoint that this stream represents. Do not put sensitive information (e.g. API tokens) into this field - Use the Authenticator component for this.", 2511 examples=[ 2512 "/products", 2513 "/quotes/{{ stream_partition['id'] }}/quote_line_groups", 2514 "/trades/{{ config['symbol_id'] }}/history", 2515 ], 2516 title="URL Path", 2517 ) 2518 http_method: Optional[HttpMethod] = Field( 2519 HttpMethod.GET, 2520 description="The HTTP method used to fetch data from the source (can be GET or POST).", 2521 examples=["GET", "POST"], 2522 title="HTTP Method", 2523 ) 2524 authenticator: Optional[ 2525 Union[ 2526 ApiKeyAuthenticator, 2527 BasicHttpAuthenticator, 2528 BearerAuthenticator, 2529 OAuthAuthenticator, 2530 JwtAuthenticator, 2531 SessionTokenAuthenticator, 2532 SelectiveAuthenticator, 2533 CustomAuthenticator, 2534 NoAuth, 2535 LegacySessionTokenAuthenticator, 2536 ] 2537 ] = Field( 2538 None, 2539 description="Authentication method to use for requests sent to the API.", 2540 title="Authenticator", 2541 ) 2542 fetch_properties_from_endpoint: Optional[PropertiesFromEndpoint] = Field( 2543 None, 2544 description="Allows for retrieving a dynamic set of properties from an API endpoint which can be injected into outbound request using the stream_partition.extra_fields.", 2545 title="Fetch Properties from Endpoint", 2546 ) 2547 request_parameters: Optional[Union[Dict[str, Union[str, QueryProperties]], str]] = Field( 2548 None, 2549 description="Specifies the query parameters that should be set on an outgoing HTTP request given the inputs.", 2550 examples=[ 2551 {"unit": "day"}, 2552 { 2553 "query": 'last_event_time BETWEEN TIMESTAMP "{{ stream_interval.start_time }}" AND TIMESTAMP "{{ stream_interval.end_time }}"' 2554 }, 2555 {"searchIn": "{{ ','.join(config.get('search_in', [])) }}"}, 2556 {"sort_by[asc]": "updated_at"}, 2557 ], 2558 title="Query Parameters", 2559 ) 2560 request_headers: Optional[Union[Dict[str, str], str]] = Field( 2561 None, 2562 description="Return any non-auth headers. Authentication headers will overwrite any overlapping headers returned from this method.", 2563 examples=[{"Output-Format": "JSON"}, {"Version": "{{ config['version'] }}"}], 2564 title="Request Headers", 2565 ) 2566 request_body_data: Optional[Union[Dict[str, str], str]] = Field( 2567 None, 2568 deprecated=True, 2569 deprecation_message="Use `request_body` field instead.", 2570 description="Specifies how to populate the body of the request with a non-JSON payload. Plain text will be sent as is, whereas objects will be converted to a urlencoded form.", 2571 examples=[ 2572 '[{"clause": {"type": "timestamp", "operator": 10, "parameters":\n [{"value": {{ stream_interval[\'start_time\'] | int * 1000 }} }]\n }, "orderBy": 1, "columnName": "Timestamp"}]/\n' 2573 ], 2574 title="Request Body Payload (Non-JSON)", 2575 ) 2576 request_body_json: Optional[Union[Dict[str, Any], str]] = Field( 2577 None, 2578 deprecated=True, 2579 deprecation_message="Use `request_body` field instead.", 2580 description="Specifies how to populate the body of the request with a JSON payload. Can contain nested objects.", 2581 examples=[ 2582 {"sort_order": "ASC", "sort_field": "CREATED_AT"}, 2583 {"key": "{{ config['value'] }}"}, 2584 {"sort": {"field": "updated_at", "order": "ascending"}}, 2585 ], 2586 title="Request Body JSON Payload", 2587 ) 2588 request_body: Optional[ 2589 Union[ 2590 RequestBodyPlainText, 2591 RequestBodyUrlEncodedForm, 2592 RequestBodyJsonObject, 2593 RequestBodyGraphQL, 2594 ] 2595 ] = Field( 2596 None, 2597 description="Specifies how to populate the body of the request with a payload. Can contain nested objects.", 2598 examples=[ 2599 { 2600 "type": "RequestBodyJsonObject", 2601 "value": {"sort_order": "ASC", "sort_field": "CREATED_AT"}, 2602 }, 2603 { 2604 "type": "RequestBodyJsonObject", 2605 "value": {"key": "{{ config['value'] }}"}, 2606 }, 2607 { 2608 "type": "RequestBodyJsonObject", 2609 "value": {"sort": {"field": "updated_at", "order": "ascending"}}, 2610 }, 2611 {"type": "RequestBodyPlainText", "value": "plain_text_body"}, 2612 { 2613 "type": "RequestBodyUrlEncodedForm", 2614 "value": {"param1": "value1", "param2": "{{ config['param2_value'] }}"}, 2615 }, 2616 { 2617 "type": "RequestBodyGraphQL", 2618 "value": { 2619 "query": { 2620 "param1": "value1", 2621 "param2": "{{ config['param2_value'] }}", 2622 } 2623 }, 2624 }, 2625 ], 2626 title="Request Body", 2627 ) 2628 error_handler: Optional[ 2629 Union[DefaultErrorHandler, CompositeErrorHandler, CustomErrorHandler] 2630 ] = Field( 2631 None, 2632 description="Error handler component that defines how to handle errors.", 2633 title="Error Handler", 2634 ) 2635 use_cache: Optional[bool] = Field( 2636 False, 2637 description="Enables stream requests caching. This field is automatically set by the CDK.", 2638 title="Use Cache", 2639 ) 2640 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters") 2641 2642 2643class DynamicSchemaLoader(BaseModel): 2644 type: Literal["DynamicSchemaLoader"] 2645 retriever: Union[SimpleRetriever, AsyncRetriever, CustomRetriever] = Field( 2646 ..., 2647 description="Component used to coordinate how records are extracted across stream slices and request pages.", 2648 title="Retriever", 2649 ) 2650 schema_filter: Optional[Union[RecordFilter, CustomRecordFilter]] = Field( 2651 None, 2652 description="Responsible for filtering fields to be added to json schema.", 2653 title="Schema Filter", 2654 ) 2655 schema_transformations: Optional[ 2656 List[ 2657 Union[ 2658 AddFields, 2659 RemoveFields, 2660 KeysToLower, 2661 KeysToSnakeCase, 2662 FlattenFields, 2663 DpathFlattenFields, 2664 KeysReplace, 2665 CustomTransformation, 2666 ] 2667 ] 2668 ] = Field( 2669 None, 2670 description="A list of transformations to be applied to the schema.", 2671 title="Schema Transformations", 2672 ) 2673 schema_type_identifier: SchemaTypeIdentifier 2674 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters") 2675 2676 2677class ParentStreamConfig(BaseModel): 2678 type: Literal["ParentStreamConfig"] 2679 lazy_read_pointer: Optional[List[str]] = Field( 2680 [], 2681 description="If set, this will enable lazy reading, using the initial read of parent records to extract child records.", 2682 title="Lazy Read Pointer", 2683 ) 2684 parent_key: str = Field( 2685 ..., 2686 description="The primary key of records from the parent stream that will be used during the retrieval of records for the current substream. This parent identifier field is typically a characteristic of the child records being extracted from the source API.", 2687 examples=["id", "{{ config['parent_record_id'] }}"], 2688 title="Parent Key", 2689 ) 2690 stream: Union[DeclarativeStream, StateDelegatingStream] = Field( 2691 ..., description="Reference to the parent stream.", title="Parent Stream" 2692 ) 2693 partition_field: str = Field( 2694 ..., 2695 description="While iterating over parent records during a sync, the parent_key value can be referenced by using this field.", 2696 examples=["parent_id", "{{ config['parent_partition_field'] }}"], 2697 title="Current Parent Key Value Identifier", 2698 ) 2699 request_option: Optional[RequestOption] = Field( 2700 None, 2701 description="A request option describing where the parent key value should be injected into and under what field name if applicable.", 2702 title="Request Option", 2703 ) 2704 incremental_dependency: Optional[bool] = Field( 2705 False, 2706 description="Indicates whether the parent stream should be read incrementally based on updates in the child stream.", 2707 title="Incremental Dependency", 2708 ) 2709 extra_fields: Optional[List[List[str]]] = Field( 2710 None, 2711 description="Array of field paths to include as additional fields in the stream slice. Each path is an array of strings representing keys to access fields in the respective parent record. Accessible via `stream_slice.extra_fields`. Missing fields are set to `None`.", 2712 title="Extra Fields", 2713 ) 2714 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters") 2715 2716 2717class PropertiesFromEndpoint(BaseModel): 2718 type: Literal["PropertiesFromEndpoint"] 2719 property_field_path: List[str] = Field( 2720 ..., 2721 description="Describes the path to the field that should be extracted", 2722 examples=[["name"]], 2723 ) 2724 retriever: Union[SimpleRetriever, CustomRetriever] = Field( 2725 ..., 2726 description="Requester component that describes how to fetch the properties to query from a remote API endpoint.", 2727 ) 2728 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters") 2729 2730 2731class QueryProperties(BaseModel): 2732 type: Literal["QueryProperties"] 2733 property_list: Union[List[str], PropertiesFromEndpoint] = Field( 2734 ..., 2735 description="The set of properties that will be queried for in the outbound request. This can either be statically defined or dynamic based on an API endpoint", 2736 title="Property List", 2737 ) 2738 always_include_properties: Optional[List[str]] = Field( 2739 None, 2740 description="The list of properties that should be included in every set of properties when multiple chunks of properties are being requested.", 2741 title="Always Include Properties", 2742 ) 2743 property_chunking: Optional[PropertyChunking] = Field( 2744 None, 2745 description="Defines how query properties will be grouped into smaller sets for APIs with limitations on the number of properties fetched per API request.", 2746 title="Property Chunking", 2747 ) 2748 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters") 2749 2750 2751class StateDelegatingStream(BaseModel): 2752 type: Literal["StateDelegatingStream"] 2753 name: str = Field(..., description="The stream name.", example=["Users"], title="Name") 2754 full_refresh_stream: DeclarativeStream = Field( 2755 ..., 2756 description="Component used to coordinate how records are extracted across stream slices and request pages when the state is empty or not provided.", 2757 title="Full Refresh Stream", 2758 ) 2759 incremental_stream: DeclarativeStream = Field( 2760 ..., 2761 description="Component used to coordinate how records are extracted across stream slices and request pages when the state provided.", 2762 title="Incremental Stream", 2763 ) 2764 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters") 2765 2766 2767class SimpleRetriever(BaseModel): 2768 type: Literal["SimpleRetriever"] 2769 requester: Union[HttpRequester, CustomRequester] = Field( 2770 ..., 2771 description="Requester component that describes how to prepare HTTP requests to send to the source API.", 2772 ) 2773 decoder: Optional[ 2774 Union[ 2775 JsonDecoder, 2776 XmlDecoder, 2777 CsvDecoder, 2778 JsonlDecoder, 2779 GzipDecoder, 2780 IterableDecoder, 2781 ZipfileDecoder, 2782 CustomDecoder, 2783 ] 2784 ] = Field( 2785 None, 2786 description="Component decoding the response so records can be extracted.", 2787 title="HTTP Response Format", 2788 ) 2789 record_selector: RecordSelector = Field( 2790 ..., 2791 description="Component that describes how to extract records from a HTTP response.", 2792 ) 2793 paginator: Optional[Union[DefaultPaginator, NoPagination]] = Field( 2794 None, 2795 description="Paginator component that describes how to navigate through the API's pages.", 2796 ) 2797 ignore_stream_slicer_parameters_on_paginated_requests: Optional[bool] = Field( 2798 False, 2799 description="If true, the partition router and incremental request options will be ignored when paginating requests. Request options set directly on the requester will not be ignored.", 2800 ) 2801 partition_router: Optional[ 2802 Union[ 2803 ListPartitionRouter, 2804 SubstreamPartitionRouter, 2805 GroupingPartitionRouter, 2806 CustomPartitionRouter, 2807 List[ 2808 Union[ 2809 ListPartitionRouter, 2810 SubstreamPartitionRouter, 2811 GroupingPartitionRouter, 2812 CustomPartitionRouter, 2813 ] 2814 ], 2815 ] 2816 ] = Field( 2817 [], 2818 description="PartitionRouter component that describes how to partition the stream, enabling incremental syncs and checkpointing.", 2819 title="Partition Router", 2820 ) 2821 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters") 2822 2823 2824class AsyncRetriever(BaseModel): 2825 type: Literal["AsyncRetriever"] 2826 record_selector: RecordSelector = Field( 2827 ..., 2828 description="Component that describes how to extract records from a HTTP response.", 2829 ) 2830 status_mapping: AsyncJobStatusMap = Field( 2831 ..., description="Async Job Status to Airbyte CDK Async Job Status mapping." 2832 ) 2833 status_extractor: Union[DpathExtractor, CustomRecordExtractor] = Field( 2834 ..., description="Responsible for fetching the actual status of the async job." 2835 ) 2836 download_target_extractor: Union[DpathExtractor, CustomRecordExtractor] = Field( 2837 ..., 2838 description="Responsible for fetching the final result `urls` provided by the completed / finished / ready async job.", 2839 ) 2840 download_extractor: Optional[ 2841 Union[DpathExtractor, CustomRecordExtractor, ResponseToFileExtractor] 2842 ] = Field(None, description="Responsible for fetching the records from provided urls.") 2843 creation_requester: Union[HttpRequester, CustomRequester] = Field( 2844 ..., 2845 description="Requester component that describes how to prepare HTTP requests to send to the source API to create the async server-side job.", 2846 ) 2847 polling_requester: Union[HttpRequester, CustomRequester] = Field( 2848 ..., 2849 description="Requester component that describes how to prepare HTTP requests to send to the source API to fetch the status of the running async job.", 2850 ) 2851 polling_job_timeout: Optional[Union[int, str]] = Field( 2852 None, 2853 description="The time in minutes after which the single Async Job should be considered as Timed Out.", 2854 ) 2855 download_target_requester: Optional[Union[HttpRequester, CustomRequester]] = Field( 2856 None, 2857 description="Requester component that describes how to prepare HTTP requests to send to the source API to extract the url from polling response by the completed async job.", 2858 ) 2859 download_requester: Union[HttpRequester, CustomRequester] = Field( 2860 ..., 2861 description="Requester component that describes how to prepare HTTP requests to send to the source API to download the data provided by the completed async job.", 2862 ) 2863 download_paginator: Optional[Union[DefaultPaginator, NoPagination]] = Field( 2864 None, 2865 description="Paginator component that describes how to navigate through the API's pages during download.", 2866 ) 2867 abort_requester: Optional[Union[HttpRequester, CustomRequester]] = Field( 2868 None, 2869 description="Requester component that describes how to prepare HTTP requests to send to the source API to abort a job once it is timed out from the source's perspective.", 2870 ) 2871 delete_requester: Optional[Union[HttpRequester, CustomRequester]] = Field( 2872 None, 2873 description="Requester component that describes how to prepare HTTP requests to send to the source API to delete a job once the records are extracted.", 2874 ) 2875 partition_router: Optional[ 2876 Union[ 2877 ListPartitionRouter, 2878 SubstreamPartitionRouter, 2879 GroupingPartitionRouter, 2880 CustomPartitionRouter, 2881 List[ 2882 Union[ 2883 ListPartitionRouter, 2884 SubstreamPartitionRouter, 2885 GroupingPartitionRouter, 2886 CustomPartitionRouter, 2887 ] 2888 ], 2889 ] 2890 ] = Field( 2891 [], 2892 description="PartitionRouter component that describes how to partition the stream, enabling incremental syncs and checkpointing.", 2893 title="Partition Router", 2894 ) 2895 decoder: Optional[ 2896 Union[ 2897 CsvDecoder, 2898 GzipDecoder, 2899 JsonDecoder, 2900 JsonlDecoder, 2901 IterableDecoder, 2902 XmlDecoder, 2903 ZipfileDecoder, 2904 CustomDecoder, 2905 ] 2906 ] = Field( 2907 None, 2908 description="Component decoding the response so records can be extracted.", 2909 title="HTTP Response Format", 2910 ) 2911 download_decoder: Optional[ 2912 Union[ 2913 CsvDecoder, 2914 GzipDecoder, 2915 JsonDecoder, 2916 JsonlDecoder, 2917 IterableDecoder, 2918 XmlDecoder, 2919 ZipfileDecoder, 2920 CustomDecoder, 2921 ] 2922 ] = Field( 2923 None, 2924 description="Component decoding the download response so records can be extracted.", 2925 title="Download HTTP Response Format", 2926 ) 2927 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters") 2928 2929 2930class SubstreamPartitionRouter(BaseModel): 2931 type: Literal["SubstreamPartitionRouter"] 2932 parent_stream_configs: List[ParentStreamConfig] = Field( 2933 ..., 2934 description="Specifies which parent streams are being iterated over and how parent records should be used to partition the child stream data set.", 2935 title="Parent Stream Configs", 2936 ) 2937 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters") 2938 2939 2940class GroupingPartitionRouter(BaseModel): 2941 type: Literal["GroupingPartitionRouter"] 2942 group_size: int = Field( 2943 ..., 2944 description="The number of partitions to include in each group. This determines how many partition values are batched together in a single slice.", 2945 examples=[10, 50], 2946 title="Group Size", 2947 ) 2948 underlying_partition_router: Union[ 2949 ListPartitionRouter, SubstreamPartitionRouter, CustomPartitionRouter 2950 ] = Field( 2951 ..., 2952 description="The partition router whose output will be grouped. This can be any valid partition router component.", 2953 title="Underlying Partition Router", 2954 ) 2955 deduplicate: Optional[bool] = Field( 2956 True, 2957 description="If true, ensures that partitions are unique within each group by removing duplicates based on the partition key.", 2958 title="Deduplicate Partitions", 2959 ) 2960 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters") 2961 2962 2963class HttpComponentsResolver(BaseModel): 2964 type: Literal["HttpComponentsResolver"] 2965 retriever: Union[SimpleRetriever, AsyncRetriever, CustomRetriever] = Field( 2966 ..., 2967 description="Component used to coordinate how records are extracted across stream slices and request pages.", 2968 title="Retriever", 2969 ) 2970 components_mapping: List[ComponentMappingDefinition] 2971 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters") 2972 2973 2974class DynamicDeclarativeStream(BaseModel): 2975 type: Literal["DynamicDeclarativeStream"] 2976 name: Optional[str] = Field( 2977 "", description="The dynamic stream name.", example=["Tables"], title="Name" 2978 ) 2979 stream_template: Union[DeclarativeStream, StateDelegatingStream] = Field( 2980 ..., description="Reference to the stream template.", title="Stream Template" 2981 ) 2982 components_resolver: Union[ 2983 HttpComponentsResolver, ConfigComponentsResolver, ParametrizedComponentsResolver 2984 ] = Field( 2985 ..., 2986 description="Component resolve and populates stream templates with components values.", 2987 title="Components Resolver", 2988 ) 2989 2990 2991ComplexFieldType.update_forward_refs() 2992GzipDecoder.update_forward_refs() 2993CompositeErrorHandler.update_forward_refs() 2994DeclarativeSource1.update_forward_refs() 2995DeclarativeSource2.update_forward_refs() 2996SelectiveAuthenticator.update_forward_refs() 2997ConditionalStreams.update_forward_refs() 2998FileUploader.update_forward_refs() 2999DeclarativeStream.update_forward_refs() 3000SessionTokenAuthenticator.update_forward_refs() 3001HttpRequester.update_forward_refs() 3002DynamicSchemaLoader.update_forward_refs() 3003ParentStreamConfig.update_forward_refs() 3004PropertiesFromEndpoint.update_forward_refs() 3005SimpleRetriever.update_forward_refs() 3006AsyncRetriever.update_forward_refs()
class
AuthFlowType(enum.Enum):
An enumeration.
oauth2_0 =
<AuthFlowType.oauth2_0: 'oauth2.0'>
oauth1_0 =
<AuthFlowType.oauth1_0: 'oauth1.0'>
class
BasicHttpAuthenticator(pydantic.v1.main.BaseModel):
22class BasicHttpAuthenticator(BaseModel): 23 type: Literal["BasicHttpAuthenticator"] 24 username: str = Field( 25 ..., 26 description="The username that will be combined with the password, base64 encoded and used to make requests. Fill it in the user inputs.", 27 examples=["{{ config['username'] }}", "{{ config['api_key'] }}"], 28 title="Username", 29 ) 30 password: Optional[str] = Field( 31 "", 32 description="The password that will be combined with the username, base64 encoded and used to make requests. Fill it in the user inputs.", 33 examples=["{{ config['password'] }}", ""], 34 title="Password", 35 ) 36 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters")
class
BearerAuthenticator(pydantic.v1.main.BaseModel):
39class BearerAuthenticator(BaseModel): 40 type: Literal["BearerAuthenticator"] 41 api_token: str = Field( 42 ..., 43 description="Token to inject as request header for authenticating with the API.", 44 examples=["{{ config['api_key'] }}", "{{ config['token'] }}"], 45 title="Bearer Token", 46 ) 47 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters")
class
DynamicStreamCheckConfig(pydantic.v1.main.BaseModel):
50class DynamicStreamCheckConfig(BaseModel): 51 type: Literal["DynamicStreamCheckConfig"] 52 dynamic_stream_name: str = Field( 53 ..., description="The dynamic stream name.", title="Dynamic Stream Name" 54 ) 55 stream_count: Optional[int] = Field( 56 0, 57 description="The number of streams to attempt reading from during a check operation. If `stream_count` exceeds the total number of available streams, the minimum of the two values will be used.", 58 title="Stream Count", 59 )
class
CheckDynamicStream(pydantic.v1.main.BaseModel):
62class CheckDynamicStream(BaseModel): 63 type: Literal["CheckDynamicStream"] 64 stream_count: int = Field( 65 ..., 66 description="Numbers of the streams to try reading from when running a check operation.", 67 title="Stream Count", 68 ) 69 use_check_availability: Optional[bool] = Field( 70 True, 71 description="Enables stream check availability. This field is automatically set by the CDK.", 72 title="Use Check Availability", 73 )
class
ConcurrencyLevel(pydantic.v1.main.BaseModel):
76class ConcurrencyLevel(BaseModel): 77 type: Optional[Literal["ConcurrencyLevel"]] = None 78 default_concurrency: Union[int, str] = Field( 79 ..., 80 description="The amount of concurrency that will applied during a sync. This value can be hardcoded or user-defined in the config if different users have varying volume thresholds in the target API.", 81 examples=[10, "{{ config['num_workers'] or 10 }}"], 82 title="Default Concurrency", 83 ) 84 max_concurrency: Optional[int] = Field( 85 None, 86 description="The maximum level of concurrency that will be used during a sync. This becomes a required field when the default_concurrency derives from the config, because it serves as a safeguard against a user-defined threshold that is too high.", 87 examples=[20, 100], 88 title="Max Concurrency", 89 ) 90 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters")
class
ConstantBackoffStrategy(pydantic.v1.main.BaseModel):
93class ConstantBackoffStrategy(BaseModel): 94 type: Literal["ConstantBackoffStrategy"] 95 backoff_time_in_seconds: Union[float, str] = Field( 96 ..., 97 description="Backoff time in seconds.", 98 examples=[30, 30.5, "{{ config['backoff_time'] }}"], 99 title="Backoff Time", 100 ) 101 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters")
class
CursorPagination(pydantic.v1.main.BaseModel):
104class CursorPagination(BaseModel): 105 type: Literal["CursorPagination"] 106 cursor_value: str = Field( 107 ..., 108 description="Value of the cursor defining the next page to fetch.", 109 examples=[ 110 "{{ headers.link.next.cursor }}", 111 "{{ last_record['key'] }}", 112 "{{ response['nextPage'] }}", 113 ], 114 title="Cursor Value", 115 ) 116 page_size: Optional[int] = Field( 117 None, 118 description="The number of records to include in each pages.", 119 examples=[100], 120 title="Page Size", 121 ) 122 stop_condition: Optional[str] = Field( 123 None, 124 description="Template string evaluating when to stop paginating.", 125 examples=[ 126 "{{ response.data.has_more is false }}", 127 "{{ 'next' not in headers['link'] }}", 128 ], 129 title="Stop Condition", 130 ) 131 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters")
class
CustomAuthenticator(pydantic.v1.main.BaseModel):
134class CustomAuthenticator(BaseModel): 135 class Config: 136 extra = Extra.allow 137 138 type: Literal["CustomAuthenticator"] 139 class_name: str = Field( 140 ..., 141 description="Fully-qualified name of the class that will be implementing the custom authentication strategy. Has to be a sub class of DeclarativeAuthenticator. The format is `source_<name>.<package>.<class_name>`.", 142 examples=["source_railz.components.ShortLivedTokenAuthenticator"], 143 title="Class Name", 144 ) 145 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters")
class
CustomAuthenticator.Config:
class
CustomBackoffStrategy(pydantic.v1.main.BaseModel):
148class CustomBackoffStrategy(BaseModel): 149 class Config: 150 extra = Extra.allow 151 152 type: Literal["CustomBackoffStrategy"] 153 class_name: str = Field( 154 ..., 155 description="Fully-qualified name of the class that will be implementing the custom backoff strategy. The format is `source_<name>.<package>.<class_name>`.", 156 examples=["source_railz.components.MyCustomBackoffStrategy"], 157 title="Class Name", 158 ) 159 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters")
class
CustomBackoffStrategy.Config:
class
CustomErrorHandler(pydantic.v1.main.BaseModel):
162class CustomErrorHandler(BaseModel): 163 class Config: 164 extra = Extra.allow 165 166 type: Literal["CustomErrorHandler"] 167 class_name: str = Field( 168 ..., 169 description="Fully-qualified name of the class that will be implementing the custom error handler. The format is `source_<name>.<package>.<class_name>`.", 170 examples=["source_railz.components.MyCustomErrorHandler"], 171 title="Class Name", 172 ) 173 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters")
class
CustomErrorHandler.Config:
class
CustomIncrementalSync(pydantic.v1.main.BaseModel):
176class CustomIncrementalSync(BaseModel): 177 class Config: 178 extra = Extra.allow 179 180 type: Literal["CustomIncrementalSync"] 181 class_name: str = Field( 182 ..., 183 description="Fully-qualified name of the class that will be implementing the custom incremental sync. The format is `source_<name>.<package>.<class_name>`.", 184 examples=["source_railz.components.MyCustomIncrementalSync"], 185 title="Class Name", 186 ) 187 cursor_field: str = Field( 188 ..., 189 description="The location of the value on a record that will be used as a bookmark during sync.", 190 ) 191 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters")
class
CustomIncrementalSync.Config:
class
CustomPaginationStrategy(pydantic.v1.main.BaseModel):
194class CustomPaginationStrategy(BaseModel): 195 class Config: 196 extra = Extra.allow 197 198 type: Literal["CustomPaginationStrategy"] 199 class_name: str = Field( 200 ..., 201 description="Fully-qualified name of the class that will be implementing the custom pagination strategy. The format is `source_<name>.<package>.<class_name>`.", 202 examples=["source_railz.components.MyCustomPaginationStrategy"], 203 title="Class Name", 204 ) 205 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters")
class
CustomPaginationStrategy.Config:
class
CustomRecordExtractor(pydantic.v1.main.BaseModel):
208class CustomRecordExtractor(BaseModel): 209 class Config: 210 extra = Extra.allow 211 212 type: Literal["CustomRecordExtractor"] 213 class_name: str = Field( 214 ..., 215 description="Fully-qualified name of the class that will be implementing the custom record extraction strategy. The format is `source_<name>.<package>.<class_name>`.", 216 examples=["source_railz.components.MyCustomRecordExtractor"], 217 title="Class Name", 218 ) 219 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters")
class
CustomRecordExtractor.Config:
class
CustomRecordFilter(pydantic.v1.main.BaseModel):
222class CustomRecordFilter(BaseModel): 223 class Config: 224 extra = Extra.allow 225 226 type: Literal["CustomRecordFilter"] 227 class_name: str = Field( 228 ..., 229 description="Fully-qualified name of the class that will be implementing the custom record filter strategy. The format is `source_<name>.<package>.<class_name>`.", 230 examples=["source_railz.components.MyCustomCustomRecordFilter"], 231 title="Class Name", 232 ) 233 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters")
class
CustomRecordFilter.Config:
class
CustomRequester(pydantic.v1.main.BaseModel):
236class CustomRequester(BaseModel): 237 class Config: 238 extra = Extra.allow 239 240 type: Literal["CustomRequester"] 241 class_name: str = Field( 242 ..., 243 description="Fully-qualified name of the class that will be implementing the custom requester strategy. The format is `source_<name>.<package>.<class_name>`.", 244 examples=["source_railz.components.MyCustomRecordExtractor"], 245 title="Class Name", 246 ) 247 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters")
class
CustomRequester.Config:
class
CustomRetriever(pydantic.v1.main.BaseModel):
250class CustomRetriever(BaseModel): 251 class Config: 252 extra = Extra.allow 253 254 type: Literal["CustomRetriever"] 255 class_name: str = Field( 256 ..., 257 description="Fully-qualified name of the class that will be implementing the custom retriever strategy. The format is `source_<name>.<package>.<class_name>`.", 258 examples=["source_railz.components.MyCustomRetriever"], 259 title="Class Name", 260 ) 261 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters")
class
CustomRetriever.Config:
class
CustomPartitionRouter(pydantic.v1.main.BaseModel):
264class CustomPartitionRouter(BaseModel): 265 class Config: 266 extra = Extra.allow 267 268 type: Literal["CustomPartitionRouter"] 269 class_name: str = Field( 270 ..., 271 description="Fully-qualified name of the class that will be implementing the custom partition router. The format is `source_<name>.<package>.<class_name>`.", 272 examples=["source_railz.components.MyCustomPartitionRouter"], 273 title="Class Name", 274 ) 275 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters")
class
CustomPartitionRouter.Config:
class
CustomSchemaLoader(pydantic.v1.main.BaseModel):
278class CustomSchemaLoader(BaseModel): 279 class Config: 280 extra = Extra.allow 281 282 type: Literal["CustomSchemaLoader"] 283 class_name: str = Field( 284 ..., 285 description="Fully-qualified name of the class that will be implementing the custom schema loader. The format is `source_<name>.<package>.<class_name>`.", 286 examples=["source_railz.components.MyCustomSchemaLoader"], 287 title="Class Name", 288 ) 289 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters")
class
CustomSchemaLoader.Config:
class
CustomSchemaNormalization(pydantic.v1.main.BaseModel):
292class CustomSchemaNormalization(BaseModel): 293 class Config: 294 extra = Extra.allow 295 296 type: Literal["CustomSchemaNormalization"] 297 class_name: str = Field( 298 ..., 299 description="Fully-qualified name of the class that will be implementing the custom normalization. The format is `source_<name>.<package>.<class_name>`.", 300 examples=[ 301 "source_amazon_seller_partner.components.LedgerDetailedViewReportsTypeTransformer" 302 ], 303 title="Class Name", 304 ) 305 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters")
class
CustomSchemaNormalization.Config:
class
CustomStateMigration(pydantic.v1.main.BaseModel):
308class CustomStateMigration(BaseModel): 309 class Config: 310 extra = Extra.allow 311 312 type: Literal["CustomStateMigration"] 313 class_name: str = Field( 314 ..., 315 description="Fully-qualified name of the class that will be implementing the custom state migration. The format is `source_<name>.<package>.<class_name>`.", 316 examples=["source_railz.components.MyCustomStateMigration"], 317 title="Class Name", 318 ) 319 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters")
class
CustomStateMigration.Config:
class
CustomTransformation(pydantic.v1.main.BaseModel):
322class CustomTransformation(BaseModel): 323 class Config: 324 extra = Extra.allow 325 326 type: Literal["CustomTransformation"] 327 class_name: str = Field( 328 ..., 329 description="Fully-qualified name of the class that will be implementing the custom transformation. The format is `source_<name>.<package>.<class_name>`.", 330 examples=["source_railz.components.MyCustomTransformation"], 331 title="Class Name", 332 ) 333 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters")
class
CustomTransformation.Config:
class
LegacyToPerPartitionStateMigration(pydantic.v1.main.BaseModel):
class
LegacyToPerPartitionStateMigration.Config:
class
Clamping(pydantic.v1.main.BaseModel):
class
Algorithm(enum.Enum):
353class Algorithm(Enum): 354 HS256 = "HS256" 355 HS384 = "HS384" 356 HS512 = "HS512" 357 ES256 = "ES256" 358 ES256K = "ES256K" 359 ES384 = "ES384" 360 ES512 = "ES512" 361 RS256 = "RS256" 362 RS384 = "RS384" 363 RS512 = "RS512" 364 PS256 = "PS256" 365 PS384 = "PS384" 366 PS512 = "PS512" 367 EdDSA = "EdDSA"
An enumeration.
HS256 =
<Algorithm.HS256: 'HS256'>
HS384 =
<Algorithm.HS384: 'HS384'>
HS512 =
<Algorithm.HS512: 'HS512'>
ES256 =
<Algorithm.ES256: 'ES256'>
ES256K =
<Algorithm.ES256K: 'ES256K'>
ES384 =
<Algorithm.ES384: 'ES384'>
ES512 =
<Algorithm.ES512: 'ES512'>
RS256 =
<Algorithm.RS256: 'RS256'>
RS384 =
<Algorithm.RS384: 'RS384'>
RS512 =
<Algorithm.RS512: 'RS512'>
PS256 =
<Algorithm.PS256: 'PS256'>
PS384 =
<Algorithm.PS384: 'PS384'>
PS512 =
<Algorithm.PS512: 'PS512'>
EdDSA =
<Algorithm.EdDSA: 'EdDSA'>
class
JwtHeaders(pydantic.v1.main.BaseModel):
370class JwtHeaders(BaseModel): 371 class Config: 372 extra = Extra.forbid 373 374 kid: Optional[str] = Field( 375 None, 376 description="Private key ID for user account.", 377 examples=["{{ config['kid'] }}"], 378 title="Key Identifier", 379 ) 380 typ: Optional[str] = Field( 381 "JWT", 382 description="The media type of the complete JWT.", 383 examples=["JWT"], 384 title="Type", 385 ) 386 cty: Optional[str] = Field( 387 None, 388 description="Content type of JWT header.", 389 examples=["JWT"], 390 title="Content Type", 391 )
class
JwtHeaders.Config:
class
JwtPayload(pydantic.v1.main.BaseModel):
394class JwtPayload(BaseModel): 395 class Config: 396 extra = Extra.forbid 397 398 iss: Optional[str] = Field( 399 None, 400 description="The user/principal that issued the JWT. Commonly a value unique to the user.", 401 examples=["{{ config['iss'] }}"], 402 title="Issuer", 403 ) 404 sub: Optional[str] = Field( 405 None, 406 description="The subject of the JWT. Commonly defined by the API.", 407 title="Subject", 408 ) 409 aud: Optional[str] = Field( 410 None, 411 description="The recipient that the JWT is intended for. Commonly defined by the API.", 412 examples=["appstoreconnect-v1"], 413 title="Audience", 414 )
class
JwtPayload.Config:
class
JwtAuthenticator(pydantic.v1.main.BaseModel):
417class JwtAuthenticator(BaseModel): 418 type: Literal["JwtAuthenticator"] 419 secret_key: str = Field( 420 ..., 421 description="Secret used to sign the JSON web token.", 422 examples=["{{ config['secret_key'] }}"], 423 title="Secret Key", 424 ) 425 base64_encode_secret_key: Optional[bool] = Field( 426 False, 427 description='When set to true, the secret key will be base64 encoded prior to being encoded as part of the JWT. Only set to "true" when required by the API.', 428 title="Base64-encode Secret Key", 429 ) 430 algorithm: Algorithm = Field( 431 ..., 432 description="Algorithm used to sign the JSON web token.", 433 examples=["ES256", "HS256", "RS256", "{{ config['algorithm'] }}"], 434 title="Algorithm", 435 ) 436 token_duration: Optional[int] = Field( 437 1200, 438 description="The amount of time in seconds a JWT token can be valid after being issued.", 439 examples=[1200, 3600], 440 title="Token Duration", 441 ) 442 header_prefix: Optional[str] = Field( 443 None, 444 description="The prefix to be used within the Authentication header.", 445 examples=["Bearer", "Basic"], 446 title="Header Prefix", 447 ) 448 jwt_headers: Optional[JwtHeaders] = Field( 449 None, 450 description="JWT headers used when signing JSON web token.", 451 title="JWT Headers", 452 ) 453 additional_jwt_headers: Optional[Dict[str, Any]] = Field( 454 None, 455 description="Additional headers to be included with the JWT headers object.", 456 title="Additional JWT Headers", 457 ) 458 jwt_payload: Optional[JwtPayload] = Field( 459 None, 460 description="JWT Payload used when signing JSON web token.", 461 title="JWT Payload", 462 ) 463 additional_jwt_payload: Optional[Dict[str, Any]] = Field( 464 None, 465 description="Additional properties to be added to the JWT payload.", 466 title="Additional JWT Payload Properties", 467 ) 468 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters")
algorithm: Algorithm
jwt_headers: Optional[JwtHeaders]
jwt_payload: Optional[JwtPayload]
class
RefreshTokenUpdater(pydantic.v1.main.BaseModel):
471class RefreshTokenUpdater(BaseModel): 472 refresh_token_name: Optional[str] = Field( 473 "refresh_token", 474 description="The name of the property which contains the updated refresh token in the response from the token refresh endpoint.", 475 examples=["refresh_token"], 476 title="Refresh Token Property Name", 477 ) 478 access_token_config_path: Optional[List[str]] = Field( 479 ["credentials", "access_token"], 480 description="Config path to the access token. Make sure the field actually exists in the config.", 481 examples=[["credentials", "access_token"], ["access_token"]], 482 title="Config Path To Access Token", 483 ) 484 refresh_token_config_path: Optional[List[str]] = Field( 485 ["credentials", "refresh_token"], 486 description="Config path to the access token. Make sure the field actually exists in the config.", 487 examples=[["credentials", "refresh_token"], ["refresh_token"]], 488 title="Config Path To Refresh Token", 489 ) 490 token_expiry_date_config_path: Optional[List[str]] = Field( 491 ["credentials", "token_expiry_date"], 492 description="Config path to the expiry date. Make sure actually exists in the config.", 493 examples=[["credentials", "token_expiry_date"]], 494 title="Config Path To Expiry Date", 495 ) 496 refresh_token_error_status_codes: Optional[List[int]] = Field( 497 [], 498 description="Status Codes to Identify refresh token error in response (Refresh Token Error Key and Refresh Token Error Values should be also specified). Responses with one of the error status code and containing an error value will be flagged as a config error", 499 examples=[[400, 500]], 500 title="Refresh Token Error Status Codes", 501 ) 502 refresh_token_error_key: Optional[str] = Field( 503 "", 504 description="Key to Identify refresh token error in response (Refresh Token Error Status Codes and Refresh Token Error Values should be also specified).", 505 examples=["error"], 506 title="Refresh Token Error Key", 507 ) 508 refresh_token_error_values: Optional[List[str]] = Field( 509 [], 510 description='List of values to check for exception during token refresh process. Used to check if the error found in the response matches the key from the Refresh Token Error Key field (e.g. response={"error": "invalid_grant"}). Only responses with one of the error status code and containing an error value will be flagged as a config error', 511 examples=[["invalid_grant", "invalid_permissions"]], 512 title="Refresh Token Error Values", 513 )
class
OAuthAuthenticator(pydantic.v1.main.BaseModel):
516class OAuthAuthenticator(BaseModel): 517 type: Literal["OAuthAuthenticator"] 518 client_id_name: Optional[str] = Field( 519 "client_id", 520 description="The name of the property to use to refresh the `access_token`.", 521 examples=["custom_app_id"], 522 title="Client ID Property Name", 523 ) 524 client_id: Optional[str] = Field( 525 None, 526 description="The OAuth client ID. Fill it in the user inputs.", 527 examples=["{{ config['client_id }}", "{{ config['credentials']['client_id }}"], 528 title="Client ID", 529 ) 530 client_secret_name: Optional[str] = Field( 531 "client_secret", 532 description="The name of the property to use to refresh the `access_token`.", 533 examples=["custom_app_secret"], 534 title="Client Secret Property Name", 535 ) 536 client_secret: Optional[str] = Field( 537 None, 538 description="The OAuth client secret. Fill it in the user inputs.", 539 examples=[ 540 "{{ config['client_secret }}", 541 "{{ config['credentials']['client_secret }}", 542 ], 543 title="Client Secret", 544 ) 545 refresh_token_name: Optional[str] = Field( 546 "refresh_token", 547 description="The name of the property to use to refresh the `access_token`.", 548 examples=["custom_app_refresh_value"], 549 title="Refresh Token Property Name", 550 ) 551 refresh_token: Optional[str] = Field( 552 None, 553 description="Credential artifact used to get a new access token.", 554 examples=[ 555 "{{ config['refresh_token'] }}", 556 "{{ config['credentials]['refresh_token'] }}", 557 ], 558 title="Refresh Token", 559 ) 560 token_refresh_endpoint: Optional[str] = Field( 561 None, 562 description="The full URL to call to obtain a new access token.", 563 examples=["https://connect.squareup.com/oauth2/token"], 564 title="Token Refresh Endpoint", 565 ) 566 access_token_name: Optional[str] = Field( 567 "access_token", 568 description="The name of the property which contains the access token in the response from the token refresh endpoint.", 569 examples=["access_token"], 570 title="Access Token Property Name", 571 ) 572 access_token_value: Optional[str] = Field( 573 None, 574 description="The value of the access_token to bypass the token refreshing using `refresh_token`.", 575 examples=["secret_access_token_value"], 576 title="Access Token Value", 577 ) 578 expires_in_name: Optional[str] = Field( 579 "expires_in", 580 description="The name of the property which contains the expiry date in the response from the token refresh endpoint.", 581 examples=["expires_in"], 582 title="Token Expiry Property Name", 583 ) 584 grant_type_name: Optional[str] = Field( 585 "grant_type", 586 description="The name of the property to use to refresh the `access_token`.", 587 examples=["custom_grant_type"], 588 title="Grant Type Property Name", 589 ) 590 grant_type: Optional[str] = Field( 591 "refresh_token", 592 description="Specifies the OAuth2 grant type. If set to refresh_token, the refresh_token needs to be provided as well. For client_credentials, only client id and secret are required. Other grant types are not officially supported.", 593 examples=["refresh_token", "client_credentials"], 594 title="Grant Type", 595 ) 596 refresh_request_body: Optional[Dict[str, Any]] = Field( 597 None, 598 description="Body of the request sent to get a new access token.", 599 examples=[ 600 { 601 "applicationId": "{{ config['application_id'] }}", 602 "applicationSecret": "{{ config['application_secret'] }}", 603 "token": "{{ config['token'] }}", 604 } 605 ], 606 title="Refresh Request Body", 607 ) 608 refresh_request_headers: Optional[Dict[str, Any]] = Field( 609 None, 610 description="Headers of the request sent to get a new access token.", 611 examples=[ 612 { 613 "Authorization": "<AUTH_TOKEN>", 614 "Content-Type": "application/x-www-form-urlencoded", 615 } 616 ], 617 title="Refresh Request Headers", 618 ) 619 scopes: Optional[List[str]] = Field( 620 None, 621 description="List of scopes that should be granted to the access token.", 622 examples=[["crm.list.read", "crm.objects.contacts.read", "crm.schema.contacts.read"]], 623 title="Scopes", 624 ) 625 token_expiry_date: Optional[str] = Field( 626 None, 627 description="The access token expiry date.", 628 examples=["2023-04-06T07:12:10.421833+00:00", 1680842386], 629 title="Token Expiry Date", 630 ) 631 token_expiry_date_format: Optional[str] = Field( 632 None, 633 description="The format of the time to expiration datetime. Provide it if the time is returned as a date-time string instead of seconds.", 634 examples=["%Y-%m-%d %H:%M:%S.%f+00:00"], 635 title="Token Expiry Date Format", 636 ) 637 refresh_token_updater: Optional[RefreshTokenUpdater] = Field( 638 None, 639 description="When the token updater is defined, new refresh tokens, access tokens and the access token expiry date are written back from the authentication response to the config object. This is important if the refresh token can only used once.", 640 title="Token Updater", 641 ) 642 profile_assertion: Optional[JwtAuthenticator] = Field( 643 None, 644 description="The authenticator being used to authenticate the client authenticator.", 645 title="Profile Assertion", 646 ) 647 use_profile_assertion: Optional[bool] = Field( 648 False, 649 description="Enable using profile assertion as a flow for OAuth authorization.", 650 title="Use Profile Assertion", 651 ) 652 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters")
refresh_token_updater: Optional[RefreshTokenUpdater]
profile_assertion: Optional[JwtAuthenticator]
class
Rate(pydantic.v1.main.BaseModel):
655class Rate(BaseModel): 656 class Config: 657 extra = Extra.allow 658 659 limit: Union[int, str] = Field( 660 ..., 661 description="The maximum number of calls allowed within the interval.", 662 title="Limit", 663 ) 664 interval: str = Field( 665 ..., 666 description="The time interval for the rate limit.", 667 examples=["PT1H", "P1D"], 668 title="Interval", 669 )
class
Rate.Config:
class
HttpRequestRegexMatcher(pydantic.v1.main.BaseModel):
672class HttpRequestRegexMatcher(BaseModel): 673 class Config: 674 extra = Extra.allow 675 676 method: Optional[str] = Field( 677 None, description="The HTTP method to match (e.g., GET, POST).", title="Method" 678 ) 679 url_base: Optional[str] = Field( 680 None, 681 description='The base URL (scheme and host, e.g. "https://api.example.com") to match.', 682 title="URL Base", 683 ) 684 url_path_pattern: Optional[str] = Field( 685 None, 686 description="A regular expression pattern to match the URL path.", 687 title="URL Path Pattern", 688 ) 689 params: Optional[Dict[str, Any]] = Field( 690 None, description="The query parameters to match.", title="Parameters" 691 ) 692 headers: Optional[Dict[str, Any]] = Field( 693 None, description="The headers to match.", title="Headers" 694 )
class
HttpRequestRegexMatcher.Config:
class
DpathExtractor(pydantic.v1.main.BaseModel):
697class DpathExtractor(BaseModel): 698 type: Literal["DpathExtractor"] 699 field_path: List[str] = Field( 700 ..., 701 description='List of potentially nested fields describing the full path of the field to extract. Use "*" to extract all values from an array. See more info in the [docs](https://docs.airbyte.com/connector-development/config-based/understanding-the-yaml-file/record-selector).', 702 examples=[ 703 ["data"], 704 ["data", "records"], 705 ["data", "{{ parameters.name }}"], 706 ["data", "*", "record"], 707 ], 708 title="Field Path", 709 ) 710 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters")
class
ResponseToFileExtractor(pydantic.v1.main.BaseModel):
class
ExponentialBackoffStrategy(pydantic.v1.main.BaseModel):
718class ExponentialBackoffStrategy(BaseModel): 719 type: Literal["ExponentialBackoffStrategy"] 720 factor: Optional[Union[float, str]] = Field( 721 5, 722 description="Multiplicative constant applied on each retry.", 723 examples=[5, 5.5, "10"], 724 title="Factor", 725 ) 726 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters")
class
GroupByKeyMergeStrategy(pydantic.v1.main.BaseModel):
729class GroupByKeyMergeStrategy(BaseModel): 730 type: Literal["GroupByKeyMergeStrategy"] 731 key: Union[str, List[str]] = Field( 732 ..., 733 description="The name of the field on the record whose value will be used to group properties that were retrieved through multiple API requests.", 734 examples=["id", ["parent_id", "end_date"]], 735 title="Key", 736 ) 737 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters")
class
SessionTokenRequestBearerAuthenticator(pydantic.v1.main.BaseModel):
class
HttpMethod(enum.Enum):
An enumeration.
GET =
<HttpMethod.GET: 'GET'>
POST =
<HttpMethod.POST: 'POST'>
class
Action(enum.Enum):
749class Action(Enum): 750 SUCCESS = "SUCCESS" 751 FAIL = "FAIL" 752 RETRY = "RETRY" 753 IGNORE = "IGNORE" 754 RATE_LIMITED = "RATE_LIMITED"
An enumeration.
SUCCESS =
<Action.SUCCESS: 'SUCCESS'>
FAIL =
<Action.FAIL: 'FAIL'>
RETRY =
<Action.RETRY: 'RETRY'>
IGNORE =
<Action.IGNORE: 'IGNORE'>
RATE_LIMITED =
<Action.RATE_LIMITED: 'RATE_LIMITED'>
class
FailureType(enum.Enum):
757class FailureType(Enum): 758 system_error = "system_error" 759 config_error = "config_error" 760 transient_error = "transient_error"
An enumeration.
system_error =
<FailureType.system_error: 'system_error'>
config_error =
<FailureType.config_error: 'config_error'>
transient_error =
<FailureType.transient_error: 'transient_error'>
class
HttpResponseFilter(pydantic.v1.main.BaseModel):
763class HttpResponseFilter(BaseModel): 764 type: Literal["HttpResponseFilter"] 765 action: Optional[Action] = Field( 766 None, 767 description="Action to execute if a response matches the filter.", 768 examples=["SUCCESS", "FAIL", "RETRY", "IGNORE", "RATE_LIMITED"], 769 title="Action", 770 ) 771 failure_type: Optional[FailureType] = Field( 772 None, 773 description="Failure type of traced exception if a response matches the filter.", 774 examples=["system_error", "config_error", "transient_error"], 775 title="Failure Type", 776 ) 777 error_message: Optional[str] = Field( 778 None, 779 description="Error Message to display if the response matches the filter.", 780 title="Error Message", 781 ) 782 error_message_contains: Optional[str] = Field( 783 None, 784 description="Match the response if its error message contains the substring.", 785 example=["This API operation is not enabled for this site"], 786 title="Error Message Substring", 787 ) 788 http_codes: Optional[List[int]] = Field( 789 None, 790 description="Match the response if its HTTP code is included in this list.", 791 examples=[[420, 429], [500]], 792 title="HTTP Codes", 793 unique_items=True, 794 ) 795 predicate: Optional[str] = Field( 796 None, 797 description="Match the response if the predicate evaluates to true.", 798 examples=[ 799 "{{ 'Too much requests' in response }}", 800 "{{ 'error_code' in response and response['error_code'] == 'ComplexityException' }}", 801 ], 802 title="Predicate", 803 ) 804 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters")
action: Optional[Action]
failure_type: Optional[FailureType]
class
ComplexFieldType(pydantic.v1.main.BaseModel):
807class ComplexFieldType(BaseModel): 808 field_type: str 809 items: Optional[Union[str, ComplexFieldType]] = None
items: Union[str, ComplexFieldType, NoneType]
class
TypesMap(pydantic.v1.main.BaseModel):
812class TypesMap(BaseModel): 813 target_type: Union[str, List[str], ComplexFieldType] 814 current_type: Union[str, List[str]] 815 condition: Optional[str] = None
target_type: Union[str, List[str], ComplexFieldType]
class
SchemaTypeIdentifier(pydantic.v1.main.BaseModel):
818class SchemaTypeIdentifier(BaseModel): 819 type: Optional[Literal["SchemaTypeIdentifier"]] = None 820 schema_pointer: Optional[List[str]] = Field( 821 [], 822 description="List of nested fields defining the schema field path to extract. Defaults to [].", 823 title="Schema Path", 824 ) 825 key_pointer: List[str] = Field( 826 ..., 827 description="List of potentially nested fields describing the full path of the field key to extract.", 828 title="Key Path", 829 ) 830 type_pointer: Optional[List[str]] = Field( 831 None, 832 description="List of potentially nested fields describing the full path of the field type to extract.", 833 title="Type Path", 834 ) 835 types_mapping: Optional[List[TypesMap]] = None 836 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters")
types_mapping: Optional[List[TypesMap]]
class
InlineSchemaLoader(pydantic.v1.main.BaseModel):
839class InlineSchemaLoader(BaseModel): 840 type: Literal["InlineSchemaLoader"] 841 schema_: Optional[Dict[str, Any]] = Field( 842 None, 843 alias="schema", 844 description='Describes a streams\' schema. Refer to the <a href="https://docs.airbyte.com/understanding-airbyte/supported-data-types/">Data Types documentation</a> for more details on which types are valid.', 845 title="Schema", 846 )
class
JsonFileSchemaLoader(pydantic.v1.main.BaseModel):
849class JsonFileSchemaLoader(BaseModel): 850 type: Literal["JsonFileSchemaLoader"] 851 file_path: Optional[str] = Field( 852 None, 853 description="Path to the JSON file defining the schema. The path is relative to the connector module's root.", 854 example=["./schemas/users.json"], 855 title="File Path", 856 ) 857 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters")
class
JsonDecoder(pydantic.v1.main.BaseModel):
class
JsonlDecoder(pydantic.v1.main.BaseModel):
class
KeysToLower(pydantic.v1.main.BaseModel):
class
KeysToSnakeCase(pydantic.v1.main.BaseModel):
class
FlattenFields(pydantic.v1.main.BaseModel):
878class FlattenFields(BaseModel): 879 type: Literal["FlattenFields"] 880 flatten_lists: Optional[bool] = Field( 881 True, 882 description="Whether to flatten lists or leave it as is. Default is True.", 883 title="Flatten Lists", 884 ) 885 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters")
class
KeyTransformation(pydantic.v1.main.BaseModel):
888class KeyTransformation(BaseModel): 889 type: Literal["KeyTransformation"] 890 prefix: Optional[str] = Field( 891 None, 892 description="Prefix to add for object keys. If not provided original keys remain unchanged.", 893 examples=["flattened_"], 894 title="Key Prefix", 895 ) 896 suffix: Optional[str] = Field( 897 None, 898 description="Suffix to add for object keys. If not provided original keys remain unchanged.", 899 examples=["_flattened"], 900 title="Key Suffix", 901 )
class
DpathFlattenFields(pydantic.v1.main.BaseModel):
904class DpathFlattenFields(BaseModel): 905 type: Literal["DpathFlattenFields"] 906 field_path: List[str] = Field( 907 ..., 908 description="A path to field that needs to be flattened.", 909 examples=[["data"], ["data", "*", "field"]], 910 title="Field Path", 911 ) 912 delete_origin_value: Optional[bool] = Field( 913 None, 914 description="Whether to delete the origin value or keep it. Default is False.", 915 title="Delete Origin Value", 916 ) 917 replace_record: Optional[bool] = Field( 918 None, 919 description="Whether to replace the origin record or not. Default is False.", 920 title="Replace Origin Record", 921 ) 922 key_transformation: Optional[KeyTransformation] = Field( 923 None, 924 description="Transformation for object keys. If not provided, original key will be used.", 925 title="Key transformation", 926 ) 927 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters")
key_transformation: Optional[KeyTransformation]
class
KeysReplace(pydantic.v1.main.BaseModel):
930class KeysReplace(BaseModel): 931 type: Literal["KeysReplace"] 932 old: str = Field( 933 ..., 934 description="Old value to replace.", 935 examples=[ 936 " ", 937 "{{ record.id }}", 938 "{{ config['id'] }}", 939 "{{ stream_slice['id'] }}", 940 ], 941 title="Old value", 942 ) 943 new: str = Field( 944 ..., 945 description="New value to set.", 946 examples=[ 947 "_", 948 "{{ record.id }}", 949 "{{ config['id'] }}", 950 "{{ stream_slice['id'] }}", 951 ], 952 title="New value", 953 ) 954 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters")
class
IterableDecoder(pydantic.v1.main.BaseModel):
class
XmlDecoder(pydantic.v1.main.BaseModel):
class
CustomDecoder(pydantic.v1.main.BaseModel):
965class CustomDecoder(BaseModel): 966 class Config: 967 extra = Extra.allow 968 969 type: Literal["CustomDecoder"] 970 class_name: str = Field( 971 ..., 972 description="Fully-qualified name of the class that will be implementing the custom decoding. Has to be a sub class of Decoder. The format is `source_<name>.<package>.<class_name>`.", 973 examples=["source_amazon_ads.components.GzipJsonlDecoder"], 974 title="Class Name", 975 ) 976 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters")
class
CustomDecoder.Config:
class
MinMaxDatetime(pydantic.v1.main.BaseModel):
979class MinMaxDatetime(BaseModel): 980 type: Literal["MinMaxDatetime"] 981 datetime: str = Field( 982 ..., 983 description="Datetime value.", 984 examples=["2021-01-01", "2021-01-01T00:00:00Z", "{{ config['start_time'] }}"], 985 title="Datetime", 986 ) 987 datetime_format: Optional[str] = Field( 988 "", 989 description='Format of the datetime value. Defaults to "%Y-%m-%dT%H:%M:%S.%f%z" if left empty. Use placeholders starting with "%" to describe the format the API is using. The following placeholders are available:\n * **%s**: Epoch unix timestamp - `1686218963`\n * **%s_as_float**: Epoch unix timestamp in seconds as float with microsecond precision - `1686218963.123456`\n * **%ms**: Epoch unix timestamp - `1686218963123`\n * **%a**: Weekday (abbreviated) - `Sun`\n * **%A**: Weekday (full) - `Sunday`\n * **%w**: Weekday (decimal) - `0` (Sunday), `6` (Saturday)\n * **%d**: Day of the month (zero-padded) - `01`, `02`, ..., `31`\n * **%b**: Month (abbreviated) - `Jan`\n * **%B**: Month (full) - `January`\n * **%m**: Month (zero-padded) - `01`, `02`, ..., `12`\n * **%y**: Year (without century, zero-padded) - `00`, `01`, ..., `99`\n * **%Y**: Year (with century) - `0001`, `0002`, ..., `9999`\n * **%H**: Hour (24-hour, zero-padded) - `00`, `01`, ..., `23`\n * **%I**: Hour (12-hour, zero-padded) - `01`, `02`, ..., `12`\n * **%p**: AM/PM indicator\n * **%M**: Minute (zero-padded) - `00`, `01`, ..., `59`\n * **%S**: Second (zero-padded) - `00`, `01`, ..., `59`\n * **%f**: Microsecond (zero-padded to 6 digits) - `000000`, `000001`, ..., `999999`\n * **%_ms**: Millisecond (zero-padded to 3 digits) - `000`, `001`, ..., `999`\n * **%z**: UTC offset - `(empty)`, `+0000`, `-04:00`\n * **%Z**: Time zone name - `(empty)`, `UTC`, `GMT`\n * **%j**: Day of the year (zero-padded) - `001`, `002`, ..., `366`\n * **%U**: Week number of the year (Sunday as first day) - `00`, `01`, ..., `53`\n * **%W**: Week number of the year (Monday as first day) - `00`, `01`, ..., `53`\n * **%c**: Date and time representation - `Tue Aug 16 21:30:00 1988`\n * **%x**: Date representation - `08/16/1988`\n * **%X**: Time representation - `21:30:00`\n * **%%**: Literal \'%\' character\n\n Some placeholders depend on the locale of the underlying system - in most cases this locale is configured as en/US. For more information see the [Python documentation](https://docs.python.org/3/library/datetime.html#strftime-and-strptime-format-codes).\n', 990 examples=["%Y-%m-%dT%H:%M:%S.%f%z", "%Y-%m-%d", "%s"], 991 title="Datetime Format", 992 ) 993 max_datetime: Optional[str] = Field( 994 None, 995 description="Ceiling applied on the datetime value. Must be formatted with the datetime_format field.", 996 examples=["2021-01-01T00:00:00Z", "2021-01-01"], 997 title="Max Datetime", 998 ) 999 min_datetime: Optional[str] = Field( 1000 None, 1001 description="Floor applied on the datetime value. Must be formatted with the datetime_format field.", 1002 examples=["2010-01-01T00:00:00Z", "2010-01-01"], 1003 title="Min Datetime", 1004 ) 1005 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters")
class
NoAuth(pydantic.v1.main.BaseModel):
class
NoPagination(pydantic.v1.main.BaseModel):
class
State(pydantic.v1.main.BaseModel):
class
State.Config:
class
OauthConnectorInputSpecification(pydantic.v1.main.BaseModel):
1025class OauthConnectorInputSpecification(BaseModel): 1026 class Config: 1027 extra = Extra.allow 1028 1029 consent_url: str = Field( 1030 ..., 1031 description="The DeclarativeOAuth Specific string URL string template to initiate the authentication.\nThe placeholders are replaced during the processing to provide neccessary values.", 1032 examples=[ 1033 "https://domain.host.com/marketing_api/auth?{{client_id_key}}={{client_id_value}}&{{redirect_uri_key}}={{{{redirect_uri_value}} | urlEncoder}}&{{state_key}}={{state_value}}", 1034 "https://endpoint.host.com/oauth2/authorize?{{client_id_key}}={{client_id_value}}&{{redirect_uri_key}}={{{{redirect_uri_value}} | urlEncoder}}&{{scope_key}}={{{{scope_value}} | urlEncoder}}&{{state_key}}={{state_value}}&subdomain={{subdomain}}", 1035 ], 1036 title="Consent URL", 1037 ) 1038 scope: Optional[str] = Field( 1039 None, 1040 description="The DeclarativeOAuth Specific string of the scopes needed to be grant for authenticated user.", 1041 examples=["user:read user:read_orders workspaces:read"], 1042 title="Scopes", 1043 ) 1044 access_token_url: str = Field( 1045 ..., 1046 description="The DeclarativeOAuth Specific URL templated string to obtain the `access_token`, `refresh_token` etc.\nThe placeholders are replaced during the processing to provide neccessary values.", 1047 examples=[ 1048 "https://auth.host.com/oauth2/token?{{client_id_key}}={{client_id_value}}&{{client_secret_key}}={{client_secret_value}}&{{auth_code_key}}={{auth_code_value}}&{{redirect_uri_key}}={{{{redirect_uri_value}} | urlEncoder}}" 1049 ], 1050 title="Access Token URL", 1051 ) 1052 access_token_headers: Optional[Dict[str, Any]] = Field( 1053 None, 1054 description="The DeclarativeOAuth Specific optional headers to inject while exchanging the `auth_code` to `access_token` during `completeOAuthFlow` step.", 1055 examples=[ 1056 { 1057 "Authorization": "Basic {{ {{ client_id_value }}:{{ client_secret_value }} | base64Encoder }}" 1058 } 1059 ], 1060 title="Access Token Headers", 1061 ) 1062 access_token_params: Optional[Dict[str, Any]] = Field( 1063 None, 1064 description="The DeclarativeOAuth Specific optional query parameters to inject while exchanging the `auth_code` to `access_token` during `completeOAuthFlow` step.\nWhen this property is provided, the query params will be encoded as `Json` and included in the outgoing API request.", 1065 examples=[ 1066 { 1067 "{{ auth_code_key }}": "{{ auth_code_value }}", 1068 "{{ client_id_key }}": "{{ client_id_value }}", 1069 "{{ client_secret_key }}": "{{ client_secret_value }}", 1070 } 1071 ], 1072 title="Access Token Query Params (Json Encoded)", 1073 ) 1074 extract_output: Optional[List[str]] = Field( 1075 None, 1076 description="The DeclarativeOAuth Specific list of strings to indicate which keys should be extracted and returned back to the input config.", 1077 examples=[["access_token", "refresh_token", "other_field"]], 1078 title="Extract Output", 1079 ) 1080 state: Optional[State] = Field( 1081 None, 1082 description="The DeclarativeOAuth Specific object to provide the criteria of how the `state` query param should be constructed,\nincluding length and complexity.", 1083 examples=[{"min": 7, "max": 128}], 1084 title="Configurable State Query Param", 1085 ) 1086 client_id_key: Optional[str] = Field( 1087 None, 1088 description="The DeclarativeOAuth Specific optional override to provide the custom `client_id` key name, if required by data-provider.", 1089 examples=["my_custom_client_id_key_name"], 1090 title="Client ID Key Override", 1091 ) 1092 client_secret_key: Optional[str] = Field( 1093 None, 1094 description="The DeclarativeOAuth Specific optional override to provide the custom `client_secret` key name, if required by data-provider.", 1095 examples=["my_custom_client_secret_key_name"], 1096 title="Client Secret Key Override", 1097 ) 1098 scope_key: Optional[str] = Field( 1099 None, 1100 description="The DeclarativeOAuth Specific optional override to provide the custom `scope` key name, if required by data-provider.", 1101 examples=["my_custom_scope_key_key_name"], 1102 title="Scopes Key Override", 1103 ) 1104 state_key: Optional[str] = Field( 1105 None, 1106 description="The DeclarativeOAuth Specific optional override to provide the custom `state` key name, if required by data-provider.", 1107 examples=["my_custom_state_key_key_name"], 1108 title="State Key Override", 1109 ) 1110 auth_code_key: Optional[str] = Field( 1111 None, 1112 description="The DeclarativeOAuth Specific optional override to provide the custom `code` key name to something like `auth_code` or `custom_auth_code`, if required by data-provider.", 1113 examples=["my_custom_auth_code_key_name"], 1114 title="Auth Code Key Override", 1115 ) 1116 redirect_uri_key: Optional[str] = Field( 1117 None, 1118 description="The DeclarativeOAuth Specific optional override to provide the custom `redirect_uri` key name to something like `callback_uri`, if required by data-provider.", 1119 examples=["my_custom_redirect_uri_key_name"], 1120 title="Redirect URI Key Override", 1121 )
state: Optional[State]
class
OauthConnectorInputSpecification.Config:
class
OAuthConfigSpecification(pydantic.v1.main.BaseModel):
1124class OAuthConfigSpecification(BaseModel): 1125 class Config: 1126 extra = Extra.allow 1127 1128 oauth_user_input_from_connector_config_specification: Optional[Dict[str, Any]] = Field( 1129 None, 1130 description="OAuth specific blob. This is a Json Schema used to validate Json configurations used as input to OAuth.\nMust be a valid non-nested JSON that refers to properties from ConnectorSpecification.connectionSpecification\nusing special annotation 'path_in_connector_config'.\nThese are input values the user is entering through the UI to authenticate to the connector, that might also shared\nas inputs for syncing data via the connector.\nExamples:\nif no connector values is shared during oauth flow, oauth_user_input_from_connector_config_specification=[]\nif connector values such as 'app_id' inside the top level are used to generate the API url for the oauth flow,\n oauth_user_input_from_connector_config_specification={\n app_id: {\n type: string\n path_in_connector_config: ['app_id']\n }\n }\nif connector values such as 'info.app_id' nested inside another object are used to generate the API url for the oauth flow,\n oauth_user_input_from_connector_config_specification={\n app_id: {\n type: string\n path_in_connector_config: ['info', 'app_id']\n }\n }", 1131 examples=[ 1132 {"app_id": {"type": "string", "path_in_connector_config": ["app_id"]}}, 1133 { 1134 "app_id": { 1135 "type": "string", 1136 "path_in_connector_config": ["info", "app_id"], 1137 } 1138 }, 1139 ], 1140 title="OAuth user input", 1141 ) 1142 oauth_connector_input_specification: Optional[OauthConnectorInputSpecification] = Field( 1143 None, 1144 description='The DeclarativeOAuth specific blob.\nPertains to the fields defined by the connector relating to the OAuth flow.\n\nInterpolation capabilities:\n- The variables placeholders are declared as `{{my_var}}`.\n- The nested resolution variables like `{{ {{my_nested_var}} }}` is allowed as well.\n\n- The allowed interpolation context is:\n + base64Encoder - encode to `base64`, {{ {{my_var_a}}:{{my_var_b}} | base64Encoder }}\n + base64Decorer - decode from `base64` encoded string, {{ {{my_string_variable_or_string_value}} | base64Decoder }}\n + urlEncoder - encode the input string to URL-like format, {{ https://test.host.com/endpoint | urlEncoder}}\n + urlDecorer - decode the input url-encoded string into text format, {{ urlDecoder:https%3A%2F%2Fairbyte.io | urlDecoder}}\n + codeChallengeS256 - get the `codeChallenge` encoded value to provide additional data-provider specific authorisation values, {{ {{state_value}} | codeChallengeS256 }}\n\nExamples:\n - The TikTok Marketing DeclarativeOAuth spec:\n {\n "oauth_connector_input_specification": {\n "type": "object",\n "additionalProperties": false,\n "properties": {\n "consent_url": "https://ads.tiktok.com/marketing_api/auth?{{client_id_key}}={{client_id_value}}&{{redirect_uri_key}}={{ {{redirect_uri_value}} | urlEncoder}}&{{state_key}}={{state_value}}",\n "access_token_url": "https://business-api.tiktok.com/open_api/v1.3/oauth2/access_token/",\n "access_token_params": {\n "{{ auth_code_key }}": "{{ auth_code_value }}",\n "{{ client_id_key }}": "{{ client_id_value }}",\n "{{ client_secret_key }}": "{{ client_secret_value }}"\n },\n "access_token_headers": {\n "Content-Type": "application/json",\n "Accept": "application/json"\n },\n "extract_output": ["data.access_token"],\n "client_id_key": "app_id",\n "client_secret_key": "secret",\n "auth_code_key": "auth_code"\n }\n }\n }', 1145 title="DeclarativeOAuth Connector Specification", 1146 ) 1147 complete_oauth_output_specification: Optional[Dict[str, Any]] = Field( 1148 None, 1149 description="OAuth specific blob. This is a Json Schema used to validate Json configurations produced by the OAuth flows as they are\nreturned by the distant OAuth APIs.\nMust be a valid JSON describing the fields to merge back to `ConnectorSpecification.connectionSpecification`.\nFor each field, a special annotation `path_in_connector_config` can be specified to determine where to merge it,\nExamples:\n complete_oauth_output_specification={\n refresh_token: {\n type: string,\n path_in_connector_config: ['credentials', 'refresh_token']\n }\n }", 1150 examples=[ 1151 { 1152 "refresh_token": { 1153 "type": "string,", 1154 "path_in_connector_config": ["credentials", "refresh_token"], 1155 } 1156 } 1157 ], 1158 title="OAuth output specification", 1159 ) 1160 complete_oauth_server_input_specification: Optional[Dict[str, Any]] = Field( 1161 None, 1162 description="OAuth specific blob. This is a Json Schema used to validate Json configurations persisted as Airbyte Server configurations.\nMust be a valid non-nested JSON describing additional fields configured by the Airbyte Instance or Workspace Admins to be used by the\nserver when completing an OAuth flow (typically exchanging an auth code for refresh token).\nExamples:\n complete_oauth_server_input_specification={\n client_id: {\n type: string\n },\n client_secret: {\n type: string\n }\n }", 1163 examples=[{"client_id": {"type": "string"}, "client_secret": {"type": "string"}}], 1164 title="OAuth input specification", 1165 ) 1166 complete_oauth_server_output_specification: Optional[Dict[str, Any]] = Field( 1167 None, 1168 description="OAuth specific blob. This is a Json Schema used to validate Json configurations persisted as Airbyte Server configurations that\nalso need to be merged back into the connector configuration at runtime.\nThis is a subset configuration of `complete_oauth_server_input_specification` that filters fields out to retain only the ones that\nare necessary for the connector to function with OAuth. (some fields could be used during oauth flows but not needed afterwards, therefore\nthey would be listed in the `complete_oauth_server_input_specification` but not `complete_oauth_server_output_specification`)\nMust be a valid non-nested JSON describing additional fields configured by the Airbyte Instance or Workspace Admins to be used by the\nconnector when using OAuth flow APIs.\nThese fields are to be merged back to `ConnectorSpecification.connectionSpecification`.\nFor each field, a special annotation `path_in_connector_config` can be specified to determine where to merge it,\nExamples:\n complete_oauth_server_output_specification={\n client_id: {\n type: string,\n path_in_connector_config: ['credentials', 'client_id']\n },\n client_secret: {\n type: string,\n path_in_connector_config: ['credentials', 'client_secret']\n }\n }", 1169 examples=[ 1170 { 1171 "client_id": { 1172 "type": "string,", 1173 "path_in_connector_config": ["credentials", "client_id"], 1174 }, 1175 "client_secret": { 1176 "type": "string,", 1177 "path_in_connector_config": ["credentials", "client_secret"], 1178 }, 1179 } 1180 ], 1181 title="OAuth server output specification", 1182 )
oauth_connector_input_specification: Optional[OauthConnectorInputSpecification]
class
OAuthConfigSpecification.Config:
class
OffsetIncrement(pydantic.v1.main.BaseModel):
1185class OffsetIncrement(BaseModel): 1186 type: Literal["OffsetIncrement"] 1187 page_size: Optional[Union[int, str]] = Field( 1188 None, 1189 description="The number of records to include in each pages.", 1190 examples=[100, "{{ config['page_size'] }}"], 1191 title="Limit", 1192 ) 1193 inject_on_first_request: Optional[bool] = Field( 1194 False, 1195 description="Using the `offset` with value `0` during the first request", 1196 title="Inject Offset", 1197 ) 1198 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters")
class
PageIncrement(pydantic.v1.main.BaseModel):
1201class PageIncrement(BaseModel): 1202 type: Literal["PageIncrement"] 1203 page_size: Optional[Union[int, str]] = Field( 1204 None, 1205 description="The number of records to include in each pages.", 1206 examples=[100, "100", "{{ config['page_size'] }}"], 1207 title="Page Size", 1208 ) 1209 start_from_page: Optional[int] = Field( 1210 0, 1211 description="Index of the first page to request.", 1212 examples=[0, 1], 1213 title="Start From Page", 1214 ) 1215 inject_on_first_request: Optional[bool] = Field( 1216 False, 1217 description="Using the `page number` with value defined by `start_from_page` during the first request", 1218 title="Inject Page Number", 1219 ) 1220 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters")
class
PrimaryKey(pydantic.v1.main.BaseModel):
1223class PrimaryKey(BaseModel): 1224 __root__: Union[str, List[str], List[List[str]]] = Field( 1225 ..., 1226 description="The stream field to be used to distinguish unique records. Can either be a single field, an array of fields representing a composite key, or an array of arrays representing a composite key where the fields are nested fields.", 1227 examples=["id", ["code", "type"]], 1228 title="Primary Key", 1229 )
class
PropertyLimitType(enum.Enum):
1232class PropertyLimitType(Enum): 1233 characters = "characters" 1234 property_count = "property_count"
An enumeration.
characters =
<PropertyLimitType.characters: 'characters'>
property_count =
<PropertyLimitType.property_count: 'property_count'>
class
PropertyChunking(pydantic.v1.main.BaseModel):
1237class PropertyChunking(BaseModel): 1238 type: Literal["PropertyChunking"] 1239 property_limit_type: PropertyLimitType = Field( 1240 ..., 1241 description="The type used to determine the maximum number of properties per chunk", 1242 title="Property Limit Type", 1243 ) 1244 property_limit: Optional[int] = Field( 1245 None, 1246 description="The maximum amount of properties that can be retrieved per request according to the limit type.", 1247 title="Property Limit", 1248 ) 1249 record_merge_strategy: Optional[GroupByKeyMergeStrategy] = Field( 1250 None, 1251 description="Dictates how to records that require multiple requests to get all properties should be emitted to the destination", 1252 title="Record Merge Strategy", 1253 ) 1254 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters")
property_limit_type: PropertyLimitType
record_merge_strategy: Optional[GroupByKeyMergeStrategy]
class
RecordFilter(pydantic.v1.main.BaseModel):
1257class RecordFilter(BaseModel): 1258 type: Literal["RecordFilter"] 1259 condition: Optional[str] = Field( 1260 "", 1261 description="The predicate to filter a record. Records will be removed if evaluated to False.", 1262 examples=[ 1263 "{{ record['created_at'] >= stream_interval['start_time'] }}", 1264 "{{ record.status in ['active', 'expired'] }}", 1265 ], 1266 title="Condition", 1267 ) 1268 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters")
class
SchemaNormalization(enum.Enum):
An enumeration.
Default =
<SchemaNormalization.Default: 'Default'>
None_ =
<SchemaNormalization.None_: 'None'>
class
RemoveFields(pydantic.v1.main.BaseModel):
1276class RemoveFields(BaseModel): 1277 type: Literal["RemoveFields"] 1278 condition: Optional[str] = Field( 1279 "", 1280 description="The predicate to filter a property by a property value. Property will be removed if it is empty OR expression is evaluated to True.,", 1281 examples=[ 1282 "{{ property|string == '' }}", 1283 "{{ property is integer }}", 1284 "{{ property|length > 5 }}", 1285 "{{ property == 'some_string_to_match' }}", 1286 ], 1287 ) 1288 field_pointers: List[List[str]] = Field( 1289 ..., 1290 description="Array of paths defining the field to remove. Each item is an array whose field describe the path of a field to remove.", 1291 examples=[["tags"], [["content", "html"], ["content", "plain_text"]]], 1292 title="Field Paths", 1293 )
class
RequestPath(pydantic.v1.main.BaseModel):
class
InjectInto(enum.Enum):
1300class InjectInto(Enum): 1301 request_parameter = "request_parameter" 1302 header = "header" 1303 body_data = "body_data" 1304 body_json = "body_json"
An enumeration.
request_parameter =
<InjectInto.request_parameter: 'request_parameter'>
header =
<InjectInto.header: 'header'>
body_data =
<InjectInto.body_data: 'body_data'>
body_json =
<InjectInto.body_json: 'body_json'>
class
RequestOption(pydantic.v1.main.BaseModel):
1307class RequestOption(BaseModel): 1308 type: Literal["RequestOption"] 1309 field_name: Optional[str] = Field( 1310 None, 1311 description="Configures which key should be used in the location that the descriptor is being injected into. We hope to eventually deprecate this field in favor of `field_path` for all request_options, but must currently maintain it for backwards compatibility in the Builder.", 1312 examples=["segment_id"], 1313 title="Field Name", 1314 ) 1315 field_path: Optional[List[str]] = Field( 1316 None, 1317 description="Configures a path to be used for nested structures in JSON body requests (e.g. GraphQL queries)", 1318 examples=[["data", "viewer", "id"]], 1319 title="Field Path", 1320 ) 1321 inject_into: InjectInto = Field( 1322 ..., 1323 description="Configures where the descriptor should be set on the HTTP requests. Note that request parameters that are already encoded in the URL path will not be duplicated.", 1324 examples=["request_parameter", "header", "body_data", "body_json"], 1325 title="Inject Into", 1326 )
inject_into: InjectInto
class
Schemas(pydantic.v1.main.BaseModel):
class
Schemas.Config:
class
LegacySessionTokenAuthenticator(pydantic.v1.main.BaseModel):
1336class LegacySessionTokenAuthenticator(BaseModel): 1337 type: Literal["LegacySessionTokenAuthenticator"] 1338 header: str = Field( 1339 ..., 1340 description="The name of the session token header that will be injected in the request", 1341 examples=["X-Session"], 1342 title="Session Request Header", 1343 ) 1344 login_url: str = Field( 1345 ..., 1346 description="Path of the login URL (do not include the base URL)", 1347 examples=["session"], 1348 title="Login Path", 1349 ) 1350 session_token: Optional[str] = Field( 1351 None, 1352 description="Session token to use if using a pre-defined token. Not needed if authenticating with username + password pair", 1353 example=["{{ config['session_token'] }}"], 1354 title="Session Token", 1355 ) 1356 session_token_response_key: str = Field( 1357 ..., 1358 description="Name of the key of the session token to be extracted from the response", 1359 examples=["id"], 1360 title="Response Token Response Key", 1361 ) 1362 username: Optional[str] = Field( 1363 None, 1364 description="Username used to authenticate and obtain a session token", 1365 examples=[" {{ config['username'] }}"], 1366 title="Username", 1367 ) 1368 password: Optional[str] = Field( 1369 "", 1370 description="Password used to authenticate and obtain a session token", 1371 examples=["{{ config['password'] }}", ""], 1372 title="Password", 1373 ) 1374 validate_session_url: str = Field( 1375 ..., 1376 description="Path of the URL to use to validate that the session token is valid (do not include the base URL)", 1377 examples=["user/current"], 1378 title="Validate Session Path", 1379 ) 1380 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters")
class
CsvDecoder(pydantic.v1.main.BaseModel):
class
AsyncJobStatusMap(pydantic.v1.main.BaseModel):
class
ValueType(enum.Enum):
1398class ValueType(Enum): 1399 string = "string" 1400 number = "number" 1401 integer = "integer" 1402 boolean = "boolean"
An enumeration.
string =
<ValueType.string: 'string'>
number =
<ValueType.number: 'number'>
integer =
<ValueType.integer: 'integer'>
boolean =
<ValueType.boolean: 'boolean'>
class
WaitTimeFromHeader(pydantic.v1.main.BaseModel):
1405class WaitTimeFromHeader(BaseModel): 1406 type: Literal["WaitTimeFromHeader"] 1407 header: str = Field( 1408 ..., 1409 description="The name of the response header defining how long to wait before retrying.", 1410 examples=["Retry-After"], 1411 title="Response Header Name", 1412 ) 1413 regex: Optional[str] = Field( 1414 None, 1415 description="Optional regex to apply on the header to extract its value. The regex should define a capture group defining the wait time.", 1416 examples=["([-+]?\\d+)"], 1417 title="Extraction Regex", 1418 ) 1419 max_waiting_time_in_seconds: Optional[float] = Field( 1420 None, 1421 description="Given the value extracted from the header is greater than this value, stop the stream.", 1422 examples=[3600], 1423 title="Max Waiting Time in Seconds", 1424 ) 1425 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters")
class
WaitUntilTimeFromHeader(pydantic.v1.main.BaseModel):
1428class WaitUntilTimeFromHeader(BaseModel): 1429 type: Literal["WaitUntilTimeFromHeader"] 1430 header: str = Field( 1431 ..., 1432 description="The name of the response header defining how long to wait before retrying.", 1433 examples=["wait_time"], 1434 title="Response Header", 1435 ) 1436 min_wait: Optional[Union[float, str]] = Field( 1437 None, 1438 description="Minimum time to wait before retrying.", 1439 examples=[10, "60"], 1440 title="Minimum Wait Time", 1441 ) 1442 regex: Optional[str] = Field( 1443 None, 1444 description="Optional regex to apply on the header to extract its value. The regex should define a capture group defining the wait time.", 1445 examples=["([-+]?\\d+)"], 1446 title="Extraction Regex", 1447 ) 1448 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters")
class
ComponentMappingDefinition(pydantic.v1.main.BaseModel):
1451class ComponentMappingDefinition(BaseModel): 1452 type: Literal["ComponentMappingDefinition"] 1453 field_path: List[str] = Field( 1454 ..., 1455 description="A list of potentially nested fields indicating the full path where value will be added or updated.", 1456 examples=[ 1457 ["data"], 1458 ["data", "records"], 1459 ["data", 1, "name"], 1460 ["data", "{{ components_values.name }}"], 1461 ["data", "*", "record"], 1462 ["*", "**", "name"], 1463 ], 1464 title="Field Path", 1465 ) 1466 value: str = Field( 1467 ..., 1468 description="The dynamic or static value to assign to the key. Interpolated values can be used to dynamically determine the value during runtime.", 1469 examples=[ 1470 "{{ components_values['updates'] }}", 1471 "{{ components_values['MetaData']['LastUpdatedTime'] }}", 1472 "{{ config['segment_id'] }}", 1473 "{{ stream_slice['parent_id'] }}", 1474 "{{ stream_slice['extra_fields']['name'] }}", 1475 ], 1476 title="Value", 1477 ) 1478 value_type: Optional[ValueType] = Field( 1479 None, 1480 description="The expected data type of the value. If omitted, the type will be inferred from the value provided.", 1481 title="Value Type", 1482 ) 1483 create_or_update: Optional[bool] = Field( 1484 False, 1485 description="Determines whether to create a new path if it doesn't exist (true) or only update existing paths (false). When set to true, the resolver will create new paths in the stream template if they don't exist. When false (default), it will only update existing paths.", 1486 title="Create or Update", 1487 ) 1488 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters")
value_type: Optional[ValueType]
class
StreamConfig(pydantic.v1.main.BaseModel):
1491class StreamConfig(BaseModel): 1492 type: Literal["StreamConfig"] 1493 configs_pointer: List[str] = Field( 1494 ..., 1495 description="A list of potentially nested fields indicating the full path in source config file where streams configs located.", 1496 examples=[["data"], ["data", "streams"], ["data", "{{ parameters.name }}"]], 1497 title="Configs Pointer", 1498 ) 1499 default_values: Optional[List[Dict[str, Any]]] = Field( 1500 None, 1501 description="A list of default values, each matching the structure expected from the parsed component value.", 1502 title="Default Values", 1503 ) 1504 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters")
class
ConfigComponentsResolver(pydantic.v1.main.BaseModel):
1507class ConfigComponentsResolver(BaseModel): 1508 type: Literal["ConfigComponentsResolver"] 1509 stream_config: Union[List[StreamConfig], StreamConfig] 1510 components_mapping: List[ComponentMappingDefinition] 1511 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters")
stream_config: Union[List[StreamConfig], StreamConfig]
components_mapping: List[ComponentMappingDefinition]
class
StreamParametersDefinition(pydantic.v1.main.BaseModel):
1514class StreamParametersDefinition(BaseModel): 1515 type: Literal["StreamParametersDefinition"] 1516 list_of_parameters_for_stream: List[Dict[str, Any]] = Field( 1517 ..., 1518 description="A list of object of parameters for stream, each object in the list represents params for one stream.", 1519 examples=[ 1520 [ 1521 { 1522 "name": "test stream", 1523 "$parameters": {"entity": "test entity"}, 1524 "primary_key": "test key", 1525 } 1526 ] 1527 ], 1528 title="Stream Parameters", 1529 )
class
ParametrizedComponentsResolver(pydantic.v1.main.BaseModel):
1532class ParametrizedComponentsResolver(BaseModel): 1533 type: Literal["ParametrizedComponentsResolver"] 1534 stream_parameters: StreamParametersDefinition 1535 components_mapping: List[ComponentMappingDefinition] 1536 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters")
stream_parameters: StreamParametersDefinition
components_mapping: List[ComponentMappingDefinition]
class
RequestBodyPlainText(pydantic.v1.main.BaseModel):
class
RequestBodyUrlEncodedForm(pydantic.v1.main.BaseModel):
class
RequestBodyJsonObject(pydantic.v1.main.BaseModel):
class
RequestBodyGraphQlQuery(pydantic.v1.main.BaseModel):
class
RequestBodyGraphQlQuery.Config:
class
ValidateAdheresToSchema(pydantic.v1.main.BaseModel):
1561class ValidateAdheresToSchema(BaseModel): 1562 type: Literal["ValidateAdheresToSchema"] 1563 base_schema: Union[str, Dict[str, Any]] = Field( 1564 ..., 1565 description="The base JSON schema against which the user-provided schema will be validated.", 1566 examples=[ 1567 "{{ config['report_validation_schema'] }}", 1568 '\'{\n "$schema": "http://json-schema.org/draft-07/schema#",\n "title": "Person",\n "type": "object",\n "properties": {\n "name": {\n "type": "string",\n "description": "The person\'s name"\n },\n "age": {\n "type": "integer",\n "minimum": 0,\n "description": "The person\'s age"\n }\n },\n "required": ["name", "age"]\n}\'\n', 1569 { 1570 "$schema": "http://json-schema.org/draft-07/schema#", 1571 "title": "Person", 1572 "type": "object", 1573 "properties": { 1574 "name": {"type": "string", "description": "The person's name"}, 1575 "age": { 1576 "type": "integer", 1577 "minimum": 0, 1578 "description": "The person's age", 1579 }, 1580 }, 1581 "required": ["name", "age"], 1582 }, 1583 ], 1584 title="Base JSON Schema", 1585 )
class
CustomValidationStrategy(pydantic.v1.main.BaseModel):
1588class CustomValidationStrategy(BaseModel): 1589 class Config: 1590 extra = Extra.allow 1591 1592 type: Literal["CustomValidationStrategy"] 1593 class_name: str = Field( 1594 ..., 1595 description="Fully-qualified name of the class that will be implementing the custom validation strategy. Has to be a sub class of ValidationStrategy. The format is `source_<name>.<package>.<class_name>`.", 1596 examples=["source_declarative_manifest.components.MyCustomValidationStrategy"], 1597 title="Class Name", 1598 )
class
CustomValidationStrategy.Config:
class
ConfigRemapField(pydantic.v1.main.BaseModel):
1601class ConfigRemapField(BaseModel): 1602 type: Literal["ConfigRemapField"] 1603 map: Union[Dict[str, Any], str] = Field( 1604 ..., 1605 description="A mapping of original values to new values. When a field value matches a key in this map, it will be replaced with the corresponding value.", 1606 examples=[ 1607 {"pending": "in_progress", "done": "completed", "cancelled": "terminated"}, 1608 "{{ config['status_mapping'] }}", 1609 ], 1610 title="Value Mapping", 1611 ) 1612 field_path: List[str] = Field( 1613 ..., 1614 description="The path to the field whose value should be remapped. Specified as a list of path components to navigate through nested objects.", 1615 examples=[ 1616 ["status"], 1617 ["data", "status"], 1618 ["data", "{{ config.name }}", "status"], 1619 ["data", "*", "status"], 1620 ], 1621 title="Field Path", 1622 )
class
ConfigRemoveFields(pydantic.v1.main.BaseModel):
1625class ConfigRemoveFields(BaseModel): 1626 type: Literal["ConfigRemoveFields"] 1627 field_pointers: List[List[str]] = Field( 1628 ..., 1629 description="A list of field pointers to be removed from the config.", 1630 examples=[["tags"], [["content", "html"], ["content", "plain_text"]]], 1631 title="Field Pointers", 1632 ) 1633 condition: Optional[str] = Field( 1634 "", 1635 description="Fields will be removed if expression is evaluated to True.", 1636 examples=[ 1637 "{{ config['environemnt'] == 'sandbox' }}", 1638 "{{ property is integer }}", 1639 "{{ property|length > 5 }}", 1640 "{{ property == 'some_string_to_match' }}", 1641 ], 1642 )
class
AddedFieldDefinition(pydantic.v1.main.BaseModel):
1645class AddedFieldDefinition(BaseModel): 1646 type: Literal["AddedFieldDefinition"] 1647 path: List[str] = Field( 1648 ..., 1649 description="List of strings defining the path where to add the value on the record.", 1650 examples=[["segment_id"], ["metadata", "segment_id"]], 1651 title="Path", 1652 ) 1653 value: str = Field( 1654 ..., 1655 description="Value of the new field. Use {{ record['existing_field'] }} syntax to refer to other fields in the record.", 1656 examples=[ 1657 "{{ record['updates'] }}", 1658 "{{ record['MetaData']['LastUpdatedTime'] }}", 1659 "{{ stream_partition['segment_id'] }}", 1660 ], 1661 title="Value", 1662 ) 1663 value_type: Optional[ValueType] = Field( 1664 None, 1665 description="Type of the value. If not specified, the type will be inferred from the value.", 1666 title="Value Type", 1667 ) 1668 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters")
value_type: Optional[ValueType]
class
AddFields(pydantic.v1.main.BaseModel):
1671class AddFields(BaseModel): 1672 type: Literal["AddFields"] 1673 fields: List[AddedFieldDefinition] = Field( 1674 ..., 1675 description="List of transformations (path and corresponding value) that will be added to the record.", 1676 title="Fields", 1677 ) 1678 condition: Optional[str] = Field( 1679 "", 1680 description="Fields will be added if expression is evaluated to True.", 1681 examples=[ 1682 "{{ property|string == '' }}", 1683 "{{ property is integer }}", 1684 "{{ property|length > 5 }}", 1685 "{{ property == 'some_string_to_match' }}", 1686 ], 1687 ) 1688 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters")
fields: List[AddedFieldDefinition]
class
ApiKeyAuthenticator(pydantic.v1.main.BaseModel):
1691class ApiKeyAuthenticator(BaseModel): 1692 type: Literal["ApiKeyAuthenticator"] 1693 api_token: Optional[str] = Field( 1694 None, 1695 description="The API key to inject in the request. Fill it in the user inputs.", 1696 examples=["{{ config['api_key'] }}", "Token token={{ config['api_key'] }}"], 1697 title="API Key", 1698 ) 1699 header: Optional[str] = Field( 1700 None, 1701 description="The name of the HTTP header that will be set to the API key. This setting is deprecated, use inject_into instead. Header and inject_into can not be defined at the same time.", 1702 examples=["Authorization", "Api-Token", "X-Auth-Token"], 1703 title="Header Name", 1704 ) 1705 inject_into: Optional[RequestOption] = Field( 1706 None, 1707 description="Configure how the API Key will be sent in requests to the source API. Either inject_into or header has to be defined.", 1708 examples=[ 1709 {"inject_into": "header", "field_name": "Authorization"}, 1710 {"inject_into": "request_parameter", "field_name": "authKey"}, 1711 ], 1712 title="Inject API Key Into Outgoing HTTP Request", 1713 ) 1714 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters")
inject_into: Optional[RequestOption]
class
AuthFlow(pydantic.v1.main.BaseModel):
1717class AuthFlow(BaseModel): 1718 auth_flow_type: Optional[AuthFlowType] = Field( 1719 None, description="The type of auth to use", title="Auth flow type" 1720 ) 1721 predicate_key: Optional[List[str]] = Field( 1722 None, 1723 description="JSON path to a field in the connectorSpecification that should exist for the advanced auth to be applicable.", 1724 examples=[["credentials", "auth_type"]], 1725 title="Predicate key", 1726 ) 1727 predicate_value: Optional[str] = Field( 1728 None, 1729 description="Value of the predicate_key fields for the advanced auth to be applicable.", 1730 examples=["Oauth"], 1731 title="Predicate value", 1732 ) 1733 oauth_config_specification: Optional[OAuthConfigSpecification] = None
auth_flow_type: Optional[AuthFlowType]
oauth_config_specification: Optional[OAuthConfigSpecification]
class
CheckStream(pydantic.v1.main.BaseModel):
1736class CheckStream(BaseModel): 1737 type: Literal["CheckStream"] 1738 stream_names: Optional[List[str]] = Field( 1739 None, 1740 description="Names of the streams to try reading from when running a check operation.", 1741 examples=[["users"], ["users", "contacts"]], 1742 title="Stream Names", 1743 ) 1744 dynamic_streams_check_configs: Optional[List[DynamicStreamCheckConfig]] = None
dynamic_streams_check_configs: Optional[List[DynamicStreamCheckConfig]]
class
IncrementingCountCursor(pydantic.v1.main.BaseModel):
1747class IncrementingCountCursor(BaseModel): 1748 type: Literal["IncrementingCountCursor"] 1749 cursor_field: str = Field( 1750 ..., 1751 description="The location of the value on a record that will be used as a bookmark during sync. To ensure no data loss, the API must return records in ascending order based on the cursor field. Nested fields are not supported, so the field must be at the top level of the record. You can use a combination of Add Field and Remove Field transformations to move the nested field to the top.", 1752 examples=["created_at", "{{ config['record_cursor'] }}"], 1753 title="Cursor Field", 1754 ) 1755 start_value: Optional[Union[str, int]] = Field( 1756 None, 1757 description="The value that determines the earliest record that should be synced.", 1758 examples=[0, "{{ config['start_value'] }}"], 1759 title="Start Value", 1760 ) 1761 start_value_option: Optional[RequestOption] = Field( 1762 None, 1763 description="Optionally configures how the start value will be sent in requests to the source API.", 1764 title="Inject Start Value Into Outgoing HTTP Request", 1765 ) 1766 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters")
start_value_option: Optional[RequestOption]
class
DatetimeBasedCursor(pydantic.v1.main.BaseModel):
1769class DatetimeBasedCursor(BaseModel): 1770 type: Literal["DatetimeBasedCursor"] 1771 clamping: Optional[Clamping] = Field( 1772 None, 1773 description="This option is used to adjust the upper and lower boundaries of each datetime window to beginning and end of the provided target period (day, week, month)", 1774 title="Date Range Clamping", 1775 ) 1776 cursor_field: str = Field( 1777 ..., 1778 description="The location of the value on a record that will be used as a bookmark during sync. To ensure no data loss, the API must return records in ascending order based on the cursor field. Nested fields are not supported, so the field must be at the top level of the record. You can use a combination of Add Field and Remove Field transformations to move the nested field to the top.", 1779 examples=["created_at", "{{ config['record_cursor'] }}"], 1780 title="Cursor Field", 1781 ) 1782 cursor_datetime_formats: Optional[List[str]] = Field( 1783 None, 1784 description="The possible formats for the cursor field, in order of preference. The first format that matches the cursor field value will be used to parse it. If not provided, the `datetime_format` will be used.", 1785 title="Cursor Datetime Formats", 1786 ) 1787 start_datetime: Union[MinMaxDatetime, str] = Field( 1788 ..., 1789 description="The datetime that determines the earliest record that should be synced.", 1790 examples=["2020-01-1T00:00:00Z", "{{ config['start_time'] }}"], 1791 title="Start Datetime", 1792 ) 1793 start_time_option: Optional[RequestOption] = Field( 1794 None, 1795 description="Optionally configures how the start datetime will be sent in requests to the source API.", 1796 title="Inject Start Time Into Outgoing HTTP Request", 1797 ) 1798 end_datetime: Optional[Union[MinMaxDatetime, str]] = Field( 1799 None, 1800 description="The datetime that determines the last record that should be synced. If not provided, `{{ now_utc() }}` will be used.", 1801 examples=["2021-01-1T00:00:00Z", "{{ now_utc() }}", "{{ day_delta(-1) }}"], 1802 title="End Datetime", 1803 ) 1804 end_time_option: Optional[RequestOption] = Field( 1805 None, 1806 description="Optionally configures how the end datetime will be sent in requests to the source API.", 1807 title="Inject End Time Into Outgoing HTTP Request", 1808 ) 1809 datetime_format: str = Field( 1810 ..., 1811 description="The datetime format used to format the datetime values that are sent in outgoing requests to the API. Use placeholders starting with \"%\" to describe the format the API is using. The following placeholders are available:\n * **%s**: Epoch unix timestamp - `1686218963`\n * **%s_as_float**: Epoch unix timestamp in seconds as float with microsecond precision - `1686218963.123456`\n * **%ms**: Epoch unix timestamp (milliseconds) - `1686218963123`\n * **%a**: Weekday (abbreviated) - `Sun`\n * **%A**: Weekday (full) - `Sunday`\n * **%w**: Weekday (decimal) - `0` (Sunday), `6` (Saturday)\n * **%d**: Day of the month (zero-padded) - `01`, `02`, ..., `31`\n * **%b**: Month (abbreviated) - `Jan`\n * **%B**: Month (full) - `January`\n * **%m**: Month (zero-padded) - `01`, `02`, ..., `12`\n * **%y**: Year (without century, zero-padded) - `00`, `01`, ..., `99`\n * **%Y**: Year (with century) - `0001`, `0002`, ..., `9999`\n * **%H**: Hour (24-hour, zero-padded) - `00`, `01`, ..., `23`\n * **%I**: Hour (12-hour, zero-padded) - `01`, `02`, ..., `12`\n * **%p**: AM/PM indicator\n * **%M**: Minute (zero-padded) - `00`, `01`, ..., `59`\n * **%S**: Second (zero-padded) - `00`, `01`, ..., `59`\n * **%f**: Microsecond (zero-padded to 6 digits) - `000000`\n * **%_ms**: Millisecond (zero-padded to 3 digits) - `000`\n * **%z**: UTC offset - `(empty)`, `+0000`, `-04:00`\n * **%Z**: Time zone name - `(empty)`, `UTC`, `GMT`\n * **%j**: Day of the year (zero-padded) - `001`, `002`, ..., `366`\n * **%U**: Week number of the year (starting Sunday) - `00`, ..., `53`\n * **%W**: Week number of the year (starting Monday) - `00`, ..., `53`\n * **%c**: Date and time - `Tue Aug 16 21:30:00 1988`\n * **%x**: Date standard format - `08/16/1988`\n * **%X**: Time standard format - `21:30:00`\n * **%%**: Literal '%' character\n\n Some placeholders depend on the locale of the underlying system - in most cases this locale is configured as en/US. For more information see the [Python documentation](https://docs.python.org/3/library/datetime.html#strftime-and-strptime-format-codes).\n", 1812 examples=["%Y-%m-%dT%H:%M:%S.%f%z", "%Y-%m-%d", "%s", "%ms", "%s_as_float"], 1813 title="Outgoing Datetime Format", 1814 ) 1815 cursor_granularity: Optional[str] = Field( 1816 None, 1817 description="Smallest increment the datetime_format has (ISO 8601 duration) that is used to ensure the start of a slice does not overlap with the end of the previous one, e.g. for %Y-%m-%d the granularity should be P1D, for %Y-%m-%dT%H:%M:%SZ the granularity should be PT1S. Given this field is provided, `step` needs to be provided as well.", 1818 examples=["PT1S"], 1819 title="Cursor Granularity", 1820 ) 1821 is_data_feed: Optional[bool] = Field( 1822 None, 1823 description="A data feed API is an API that does not allow filtering and paginates the content from the most recent to the least recent. Given this, the CDK needs to know when to stop paginating and this field will generate a stop condition for pagination.", 1824 title="Whether the target API is formatted as a data feed", 1825 ) 1826 is_client_side_incremental: Optional[bool] = Field( 1827 None, 1828 description="If the target API endpoint does not take cursor values to filter records and returns all records anyway, the connector with this cursor will filter out records locally, and only emit new records from the last sync, hence incremental. This means that all records would be read from the API, but only new records will be emitted to the destination.", 1829 title="Whether the target API does not support filtering and returns all data (the cursor filters records in the client instead of the API side)", 1830 ) 1831 is_compare_strictly: Optional[bool] = Field( 1832 False, 1833 description="Set to True if the target API does not accept queries where the start time equal the end time.", 1834 title="Whether to skip requests if the start time equals the end time", 1835 ) 1836 global_substream_cursor: Optional[bool] = Field( 1837 False, 1838 description="This setting optimizes performance when the parent stream has thousands of partitions by storing the cursor as a single value rather than per partition. Notably, the substream state is updated only at the end of the sync, which helps prevent data loss in case of a sync failure. See more info in the [docs](https://docs.airbyte.com/connector-development/config-based/understanding-the-yaml-file/incremental-syncs).", 1839 title="Whether to store cursor as one value instead of per partition", 1840 ) 1841 lookback_window: Optional[str] = Field( 1842 None, 1843 description="Time interval before the start_datetime to read data for, e.g. P1M for looking back one month.", 1844 examples=["P1D", "P{{ config['lookback_days'] }}D"], 1845 title="Lookback Window", 1846 ) 1847 partition_field_end: Optional[str] = Field( 1848 None, 1849 description="Name of the partition start time field.", 1850 examples=["ending_time"], 1851 title="Partition Field End", 1852 ) 1853 partition_field_start: Optional[str] = Field( 1854 None, 1855 description="Name of the partition end time field.", 1856 examples=["starting_time"], 1857 title="Partition Field Start", 1858 ) 1859 step: Optional[str] = Field( 1860 None, 1861 description="The size of the time window (ISO8601 duration). Given this field is provided, `cursor_granularity` needs to be provided as well.", 1862 examples=["P1W", "{{ config['step_increment'] }}"], 1863 title="Step", 1864 ) 1865 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters")
clamping: Optional[Clamping]
start_datetime: Union[MinMaxDatetime, str]
start_time_option: Optional[RequestOption]
end_datetime: Union[MinMaxDatetime, str, NoneType]
end_time_option: Optional[RequestOption]
class
FixedWindowCallRatePolicy(pydantic.v1.main.BaseModel):
1868class FixedWindowCallRatePolicy(BaseModel): 1869 class Config: 1870 extra = Extra.allow 1871 1872 type: Literal["FixedWindowCallRatePolicy"] 1873 period: str = Field( 1874 ..., description="The time interval for the rate limit window.", title="Period" 1875 ) 1876 call_limit: int = Field( 1877 ..., 1878 description="The maximum number of calls allowed within the period.", 1879 title="Call Limit", 1880 ) 1881 matchers: List[HttpRequestRegexMatcher] = Field( 1882 ..., 1883 description="List of matchers that define which requests this policy applies to.", 1884 title="Matchers", 1885 )
matchers: List[HttpRequestRegexMatcher]
class
FixedWindowCallRatePolicy.Config:
class
MovingWindowCallRatePolicy(pydantic.v1.main.BaseModel):
1888class MovingWindowCallRatePolicy(BaseModel): 1889 class Config: 1890 extra = Extra.allow 1891 1892 type: Literal["MovingWindowCallRatePolicy"] 1893 rates: List[Rate] = Field( 1894 ..., 1895 description="List of rates that define the call limits for different time intervals.", 1896 title="Rates", 1897 ) 1898 matchers: List[HttpRequestRegexMatcher] = Field( 1899 ..., 1900 description="List of matchers that define which requests this policy applies to.", 1901 title="Matchers", 1902 )
rates: List[Rate]
matchers: List[HttpRequestRegexMatcher]
class
MovingWindowCallRatePolicy.Config:
class
UnlimitedCallRatePolicy(pydantic.v1.main.BaseModel):
1905class UnlimitedCallRatePolicy(BaseModel): 1906 class Config: 1907 extra = Extra.allow 1908 1909 type: Literal["UnlimitedCallRatePolicy"] 1910 matchers: List[HttpRequestRegexMatcher] = Field( 1911 ..., 1912 description="List of matchers that define which requests this policy applies to.", 1913 title="Matchers", 1914 )
matchers: List[HttpRequestRegexMatcher]
class
UnlimitedCallRatePolicy.Config:
class
DefaultErrorHandler(pydantic.v1.main.BaseModel):
1917class DefaultErrorHandler(BaseModel): 1918 type: Literal["DefaultErrorHandler"] 1919 backoff_strategies: Optional[ 1920 List[ 1921 Union[ 1922 ConstantBackoffStrategy, 1923 ExponentialBackoffStrategy, 1924 WaitTimeFromHeader, 1925 WaitUntilTimeFromHeader, 1926 CustomBackoffStrategy, 1927 ] 1928 ] 1929 ] = Field( 1930 None, 1931 description="List of backoff strategies to use to determine how long to wait before retrying a retryable request.", 1932 title="Backoff Strategies", 1933 ) 1934 max_retries: Optional[int] = Field( 1935 5, 1936 description="The maximum number of time to retry a retryable request before giving up and failing.", 1937 examples=[5, 0, 10], 1938 title="Max Retry Count", 1939 ) 1940 response_filters: Optional[List[HttpResponseFilter]] = Field( 1941 None, 1942 description="List of response filters to iterate on when deciding how to handle an error. When using an array of multiple filters, the filters will be applied sequentially and the response will be selected if it matches any of the filter's predicate.", 1943 title="Response Filters", 1944 ) 1945 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters")
backoff_strategies: Optional[List[Union[ConstantBackoffStrategy, ExponentialBackoffStrategy, WaitTimeFromHeader, WaitUntilTimeFromHeader, CustomBackoffStrategy]]]
response_filters: Optional[List[HttpResponseFilter]]
class
DefaultPaginator(pydantic.v1.main.BaseModel):
1948class DefaultPaginator(BaseModel): 1949 type: Literal["DefaultPaginator"] 1950 pagination_strategy: Union[ 1951 PageIncrement, OffsetIncrement, CursorPagination, CustomPaginationStrategy 1952 ] = Field( 1953 ..., 1954 description="Strategy defining how records are paginated.", 1955 title="Pagination Strategy", 1956 ) 1957 page_size_option: Optional[RequestOption] = Field( 1958 None, title="Inject Page Size Into Outgoing HTTP Request" 1959 ) 1960 page_token_option: Optional[Union[RequestOption, RequestPath]] = Field( 1961 None, title="Inject Page Token Into Outgoing HTTP Request" 1962 ) 1963 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters")
pagination_strategy: Union[PageIncrement, OffsetIncrement, CursorPagination, CustomPaginationStrategy]
page_size_option: Optional[RequestOption]
page_token_option: Union[RequestOption, RequestPath, NoneType]
class
SessionTokenRequestApiKeyAuthenticator(pydantic.v1.main.BaseModel):
1966class SessionTokenRequestApiKeyAuthenticator(BaseModel): 1967 type: Literal["ApiKey"] 1968 inject_into: RequestOption = Field( 1969 ..., 1970 description="Configure how the API Key will be sent in requests to the source API.", 1971 examples=[ 1972 {"inject_into": "header", "field_name": "Authorization"}, 1973 {"inject_into": "request_parameter", "field_name": "authKey"}, 1974 ], 1975 title="Inject API Key Into Outgoing HTTP Request", 1976 )
inject_into: RequestOption
class
ListPartitionRouter(pydantic.v1.main.BaseModel):
1979class ListPartitionRouter(BaseModel): 1980 type: Literal["ListPartitionRouter"] 1981 cursor_field: str = Field( 1982 ..., 1983 description='While iterating over list values, the name of field used to reference a list value. The partition value can be accessed with string interpolation. e.g. "{{ stream_partition[\'my_key\'] }}" where "my_key" is the value of the cursor_field.', 1984 examples=["section", "{{ config['section_key'] }}"], 1985 title="Current Partition Value Identifier", 1986 ) 1987 values: Union[str, List[str]] = Field( 1988 ..., 1989 description="The list of attributes being iterated over and used as input for the requests made to the source API.", 1990 examples=[["section_a", "section_b", "section_c"], "{{ config['sections'] }}"], 1991 title="Partition Values", 1992 ) 1993 request_option: Optional[RequestOption] = Field( 1994 None, 1995 description="A request option describing where the list value should be injected into and under what field name if applicable.", 1996 title="Inject Partition Value Into Outgoing HTTP Request", 1997 ) 1998 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters")
request_option: Optional[RequestOption]
class
RecordSelector(pydantic.v1.main.BaseModel):
2001class RecordSelector(BaseModel): 2002 type: Literal["RecordSelector"] 2003 extractor: Union[DpathExtractor, CustomRecordExtractor] 2004 record_filter: Optional[Union[RecordFilter, CustomRecordFilter]] = Field( 2005 None, 2006 description="Responsible for filtering records to be emitted by the Source.", 2007 title="Record Filter", 2008 ) 2009 schema_normalization: Optional[Union[SchemaNormalization, CustomSchemaNormalization]] = Field( 2010 None, 2011 description="Responsible for normalization according to the schema.", 2012 title="Schema Normalization", 2013 ) 2014 transform_before_filtering: Optional[bool] = Field( 2015 None, 2016 description="If true, transformation will be applied before record filtering.", 2017 title="Transform Before Filtering", 2018 ) 2019 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters")
extractor: Union[DpathExtractor, CustomRecordExtractor]
record_filter: Union[RecordFilter, CustomRecordFilter, NoneType]
schema_normalization: Union[SchemaNormalization, CustomSchemaNormalization, NoneType]
class
GzipDecoder(pydantic.v1.main.BaseModel):
2022class GzipDecoder(BaseModel): 2023 type: Literal["GzipDecoder"] 2024 decoder: Union[CsvDecoder, GzipDecoder, JsonDecoder, JsonlDecoder]
decoder: Union[CsvDecoder, GzipDecoder, JsonDecoder, JsonlDecoder]
class
RequestBodyGraphQL(pydantic.v1.main.BaseModel):
2027class RequestBodyGraphQL(BaseModel): 2028 type: Literal["RequestBodyGraphQL"] 2029 value: RequestBodyGraphQlQuery
value: RequestBodyGraphQlQuery
class
DpathValidator(pydantic.v1.main.BaseModel):
2032class DpathValidator(BaseModel): 2033 type: Literal["DpathValidator"] 2034 field_path: List[str] = Field( 2035 ..., 2036 description='List of potentially nested fields describing the full path of the field to validate. Use "*" to validate all values from an array.', 2037 examples=[ 2038 ["data"], 2039 ["data", "records"], 2040 ["data", "{{ parameters.name }}"], 2041 ["data", "*", "record"], 2042 ], 2043 title="Field Path", 2044 ) 2045 validation_strategy: Union[ValidateAdheresToSchema, CustomValidationStrategy] = Field( 2046 ..., 2047 description="The condition that the specified config value will be evaluated against", 2048 title="Validation Strategy", 2049 )
validation_strategy: Union[ValidateAdheresToSchema, CustomValidationStrategy]
class
PredicateValidator(pydantic.v1.main.BaseModel):
2052class PredicateValidator(BaseModel): 2053 type: Literal["PredicateValidator"] 2054 value: Optional[Union[str, float, Dict[str, Any], List[Any], bool]] = Field( 2055 ..., 2056 description="The value to be validated. Can be a literal value or interpolated from configuration.", 2057 examples=[ 2058 "test-value", 2059 "{{ config['api_version'] }}", 2060 "{{ config['tenant_id'] }}", 2061 123, 2062 ], 2063 title="Value", 2064 ) 2065 validation_strategy: Union[ValidateAdheresToSchema, CustomValidationStrategy] = Field( 2066 ..., 2067 description="The validation strategy to apply to the value.", 2068 title="Validation Strategy", 2069 )
validation_strategy: Union[ValidateAdheresToSchema, CustomValidationStrategy]
class
ConfigAddFields(pydantic.v1.main.BaseModel):
2072class ConfigAddFields(BaseModel): 2073 type: Literal["ConfigAddFields"] 2074 fields: List[AddedFieldDefinition] = Field( 2075 ..., 2076 description="A list of transformations (path and corresponding value) that will be added to the config.", 2077 title="Fields", 2078 ) 2079 condition: Optional[str] = Field( 2080 "", 2081 description="Fields will be added if expression is evaluated to True.", 2082 examples=[ 2083 "{{ config['environemnt'] == 'sandbox' }}", 2084 "{{ property is integer }}", 2085 "{{ property|length > 5 }}", 2086 "{{ property == 'some_string_to_match' }}", 2087 ], 2088 )
fields: List[AddedFieldDefinition]
class
CompositeErrorHandler(pydantic.v1.main.BaseModel):
2091class CompositeErrorHandler(BaseModel): 2092 type: Literal["CompositeErrorHandler"] 2093 error_handlers: List[Union[CompositeErrorHandler, DefaultErrorHandler]] = Field( 2094 ..., 2095 description="List of error handlers to iterate on to determine how to handle a failed response.", 2096 title="Error Handlers", 2097 ) 2098 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters")
error_handlers: List[Union[CompositeErrorHandler, DefaultErrorHandler]]
class
HTTPAPIBudget(pydantic.v1.main.BaseModel):
2101class HTTPAPIBudget(BaseModel): 2102 class Config: 2103 extra = Extra.allow 2104 2105 type: Literal["HTTPAPIBudget"] 2106 policies: List[ 2107 Union[ 2108 FixedWindowCallRatePolicy, 2109 MovingWindowCallRatePolicy, 2110 UnlimitedCallRatePolicy, 2111 ] 2112 ] = Field( 2113 ..., 2114 description="List of call rate policies that define how many calls are allowed.", 2115 title="Policies", 2116 ) 2117 ratelimit_reset_header: Optional[str] = Field( 2118 "ratelimit-reset", 2119 description="The HTTP response header name that indicates when the rate limit resets.", 2120 title="Rate Limit Reset Header", 2121 ) 2122 ratelimit_remaining_header: Optional[str] = Field( 2123 "ratelimit-remaining", 2124 description="The HTTP response header name that indicates the number of remaining allowed calls.", 2125 title="Rate Limit Remaining Header", 2126 ) 2127 status_codes_for_ratelimit_hit: Optional[List[int]] = Field( 2128 [429], 2129 description="List of HTTP status codes that indicate a rate limit has been hit.", 2130 title="Status Codes for Rate Limit Hit", 2131 )
policies: List[Union[FixedWindowCallRatePolicy, MovingWindowCallRatePolicy, UnlimitedCallRatePolicy]]
class
HTTPAPIBudget.Config:
class
ZipfileDecoder(pydantic.v1.main.BaseModel):
2134class ZipfileDecoder(BaseModel): 2135 class Config: 2136 extra = Extra.allow 2137 2138 type: Literal["ZipfileDecoder"] 2139 decoder: Union[CsvDecoder, GzipDecoder, JsonDecoder, JsonlDecoder] = Field( 2140 ..., 2141 description="Parser to parse the decompressed data from the zipfile(s).", 2142 title="Parser", 2143 )
decoder: Union[CsvDecoder, GzipDecoder, JsonDecoder, JsonlDecoder]
class
ZipfileDecoder.Config:
class
ConfigMigration(pydantic.v1.main.BaseModel):
2146class ConfigMigration(BaseModel): 2147 type: Literal["ConfigMigration"] 2148 description: Optional[str] = Field( 2149 None, description="The description/purpose of the config migration." 2150 ) 2151 transformations: List[Union[ConfigRemapField, ConfigAddFields, ConfigRemoveFields]] = Field( 2152 ..., 2153 description="The list of transformations that will attempt to be applied on an incoming unmigrated config. The transformations will be applied in the order they are defined.", 2154 title="Transformations", 2155 )
transformations: List[Union[ConfigRemapField, ConfigAddFields, ConfigRemoveFields]]
class
ConfigNormalizationRules(pydantic.v1.main.BaseModel):
2158class ConfigNormalizationRules(BaseModel): 2159 class Config: 2160 extra = Extra.forbid 2161 2162 config_migrations: Optional[List[ConfigMigration]] = Field( 2163 [], 2164 description="The discrete migrations that will be applied on the incoming config. Each migration will be applied in the order they are defined.", 2165 title="Config Migrations", 2166 ) 2167 transformations: Optional[ 2168 List[Union[ConfigRemapField, ConfigAddFields, ConfigRemoveFields]] 2169 ] = Field( 2170 [], 2171 description="The list of transformations that will be applied on the incoming config at the start of each sync. The transformations will be applied in the order they are defined.", 2172 title="Transformations", 2173 ) 2174 validations: Optional[List[Union[DpathValidator, PredicateValidator]]] = Field( 2175 [], 2176 description="The list of validations that will be performed on the incoming config at the start of each sync.", 2177 title="Validations", 2178 )
config_migrations: Optional[List[ConfigMigration]]
transformations: Optional[List[Union[ConfigRemapField, ConfigAddFields, ConfigRemoveFields]]]
validations: Optional[List[Union[DpathValidator, PredicateValidator]]]
class
ConfigNormalizationRules.Config:
class
Spec(pydantic.v1.main.BaseModel):
2181class Spec(BaseModel): 2182 type: Literal["Spec"] 2183 connection_specification: Dict[str, Any] = Field( 2184 ..., 2185 description="A connection specification describing how a the connector can be configured.", 2186 title="Connection Specification", 2187 ) 2188 documentation_url: Optional[str] = Field( 2189 None, 2190 description="URL of the connector's documentation page.", 2191 examples=["https://docs.airbyte.com/integrations/sources/dremio"], 2192 title="Documentation URL", 2193 ) 2194 advanced_auth: Optional[AuthFlow] = Field( 2195 None, 2196 description="Advanced specification for configuring the authentication flow.", 2197 title="Advanced Auth", 2198 ) 2199 config_normalization_rules: Optional[ConfigNormalizationRules] = Field( 2200 None, title="Config Normalization Rules" 2201 )
advanced_auth: Optional[AuthFlow]
config_normalization_rules: Optional[ConfigNormalizationRules]
class
DeclarativeSource1(pydantic.v1.main.BaseModel):
2204class DeclarativeSource1(BaseModel): 2205 class Config: 2206 extra = Extra.forbid 2207 2208 type: Literal["DeclarativeSource"] 2209 check: Union[CheckStream, CheckDynamicStream] 2210 streams: List[Union[ConditionalStreams, DeclarativeStream, StateDelegatingStream]] 2211 dynamic_streams: Optional[List[DynamicDeclarativeStream]] = None 2212 version: str = Field( 2213 ..., 2214 description="The version of the Airbyte CDK used to build and test the source.", 2215 ) 2216 schemas: Optional[Schemas] = None 2217 definitions: Optional[Dict[str, Any]] = None 2218 spec: Optional[Spec] = None 2219 concurrency_level: Optional[ConcurrencyLevel] = None 2220 api_budget: Optional[HTTPAPIBudget] = None 2221 max_concurrent_async_job_count: Optional[Union[int, str]] = Field( 2222 None, 2223 description="Maximum number of concurrent asynchronous jobs to run. This property is only relevant for sources/streams that support asynchronous job execution through the AsyncRetriever (e.g. a report-based stream that initiates a job, polls the job status, and then fetches the job results). This is often set by the API's maximum number of concurrent jobs on the account level. Refer to the API's documentation for this information.", 2224 examples=[3, "{{ config['max_concurrent_async_job_count'] }}"], 2225 title="Maximum Concurrent Asynchronous Jobs", 2226 ) 2227 metadata: Optional[Dict[str, Any]] = Field( 2228 None, 2229 description="For internal Airbyte use only - DO NOT modify manually. Used by consumers of declarative manifests for storing related metadata.", 2230 ) 2231 description: Optional[str] = Field( 2232 None, 2233 description="A description of the connector. It will be presented on the Source documentation page.", 2234 )
check: Union[CheckStream, CheckDynamicStream]
streams: List[Union[ConditionalStreams, DeclarativeStream, StateDelegatingStream]]
dynamic_streams: Optional[List[DynamicDeclarativeStream]]
schemas: Optional[Schemas]
spec: Optional[Spec]
concurrency_level: Optional[ConcurrencyLevel]
api_budget: Optional[HTTPAPIBudget]
class
DeclarativeSource1.Config:
class
DeclarativeSource2(pydantic.v1.main.BaseModel):
2237class DeclarativeSource2(BaseModel): 2238 class Config: 2239 extra = Extra.forbid 2240 2241 type: Literal["DeclarativeSource"] 2242 check: Union[CheckStream, CheckDynamicStream] 2243 streams: Optional[List[Union[ConditionalStreams, DeclarativeStream, StateDelegatingStream]]] = ( 2244 None 2245 ) 2246 dynamic_streams: List[DynamicDeclarativeStream] 2247 version: str = Field( 2248 ..., 2249 description="The version of the Airbyte CDK used to build and test the source.", 2250 ) 2251 schemas: Optional[Schemas] = None 2252 definitions: Optional[Dict[str, Any]] = None 2253 spec: Optional[Spec] = None 2254 concurrency_level: Optional[ConcurrencyLevel] = None 2255 api_budget: Optional[HTTPAPIBudget] = None 2256 max_concurrent_async_job_count: Optional[Union[int, str]] = Field( 2257 None, 2258 description="Maximum number of concurrent asynchronous jobs to run. This property is only relevant for sources/streams that support asynchronous job execution through the AsyncRetriever (e.g. a report-based stream that initiates a job, polls the job status, and then fetches the job results). This is often set by the API's maximum number of concurrent jobs on the account level. Refer to the API's documentation for this information.", 2259 examples=[3, "{{ config['max_concurrent_async_job_count'] }}"], 2260 title="Maximum Concurrent Asynchronous Jobs", 2261 ) 2262 metadata: Optional[Dict[str, Any]] = Field( 2263 None, 2264 description="For internal Airbyte use only - DO NOT modify manually. Used by consumers of declarative manifests for storing related metadata.", 2265 ) 2266 description: Optional[str] = Field( 2267 None, 2268 description="A description of the connector. It will be presented on the Source documentation page.", 2269 )
check: Union[CheckStream, CheckDynamicStream]
streams: Optional[List[Union[ConditionalStreams, DeclarativeStream, StateDelegatingStream]]]
dynamic_streams: List[DynamicDeclarativeStream]
schemas: Optional[Schemas]
spec: Optional[Spec]
concurrency_level: Optional[ConcurrencyLevel]
api_budget: Optional[HTTPAPIBudget]
class
DeclarativeSource2.Config:
class
DeclarativeSource(pydantic.v1.main.BaseModel):
2272class DeclarativeSource(BaseModel): 2273 class Config: 2274 extra = Extra.forbid 2275 2276 __root__: Union[DeclarativeSource1, DeclarativeSource2] = Field( 2277 ..., 2278 description="An API source that extracts data according to its declarative components.", 2279 title="DeclarativeSource", 2280 )
class
DeclarativeSource.Config:
class
SelectiveAuthenticator(pydantic.v1.main.BaseModel):
2283class SelectiveAuthenticator(BaseModel): 2284 class Config: 2285 extra = Extra.allow 2286 2287 type: Literal["SelectiveAuthenticator"] 2288 authenticator_selection_path: List[str] = Field( 2289 ..., 2290 description="Path of the field in config with selected authenticator name", 2291 examples=[["auth"], ["auth", "type"]], 2292 title="Authenticator Selection Path", 2293 ) 2294 authenticators: Dict[ 2295 str, 2296 Union[ 2297 ApiKeyAuthenticator, 2298 BasicHttpAuthenticator, 2299 BearerAuthenticator, 2300 OAuthAuthenticator, 2301 JwtAuthenticator, 2302 SessionTokenAuthenticator, 2303 LegacySessionTokenAuthenticator, 2304 CustomAuthenticator, 2305 NoAuth, 2306 ], 2307 ] = Field( 2308 ..., 2309 description="Authenticators to select from.", 2310 examples=[ 2311 { 2312 "authenticators": { 2313 "token": "#/definitions/ApiKeyAuthenticator", 2314 "oauth": "#/definitions/OAuthAuthenticator", 2315 "jwt": "#/definitions/JwtAuthenticator", 2316 } 2317 } 2318 ], 2319 title="Authenticators", 2320 ) 2321 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters")
authenticators: Dict[str, Union[ApiKeyAuthenticator, BasicHttpAuthenticator, BearerAuthenticator, OAuthAuthenticator, JwtAuthenticator, SessionTokenAuthenticator, LegacySessionTokenAuthenticator, CustomAuthenticator, NoAuth]]
class
SelectiveAuthenticator.Config:
class
ConditionalStreams(pydantic.v1.main.BaseModel):
2324class ConditionalStreams(BaseModel): 2325 type: Literal["ConditionalStreams"] 2326 condition: str = Field( 2327 ..., 2328 description="Condition that will be evaluated to determine if a set of streams should be available.", 2329 examples=["{{ config['is_sandbox'] }}"], 2330 title="Condition", 2331 ) 2332 streams: List[DeclarativeStream] = Field( 2333 ..., 2334 description="Streams that will be used during an operation based on the condition.", 2335 title="Streams", 2336 ) 2337 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters")
streams: List[DeclarativeStream]
class
FileUploader(pydantic.v1.main.BaseModel):
2340class FileUploader(BaseModel): 2341 type: Literal["FileUploader"] 2342 requester: Union[HttpRequester, CustomRequester] = Field( 2343 ..., 2344 description="Requester component that describes how to prepare HTTP requests to send to the source API.", 2345 ) 2346 download_target_extractor: Union[DpathExtractor, CustomRecordExtractor] = Field( 2347 ..., 2348 description="Responsible for fetching the url where the file is located. This is applied on each records and not on the HTTP response", 2349 ) 2350 file_extractor: Optional[Union[DpathExtractor, CustomRecordExtractor]] = Field( 2351 None, 2352 description="Responsible for fetching the content of the file. If not defined, the assumption is that the whole response body is the file content", 2353 ) 2354 filename_extractor: Optional[str] = Field( 2355 None, 2356 description="Defines the name to store the file. Stream name is automatically added to the file path. File unique ID can be used to avoid overwriting files. Random UUID will be used if the extractor is not provided.", 2357 examples=[ 2358 "{{ record.id }}/{{ record.file_name }}/", 2359 "{{ record.id }}_{{ record.file_name }}/", 2360 ], 2361 ) 2362 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters")
requester: Union[HttpRequester, CustomRequester]
download_target_extractor: Union[DpathExtractor, CustomRecordExtractor]
file_extractor: Union[DpathExtractor, CustomRecordExtractor, NoneType]
class
DeclarativeStream(pydantic.v1.main.BaseModel):
2365class DeclarativeStream(BaseModel): 2366 class Config: 2367 extra = Extra.allow 2368 2369 type: Literal["DeclarativeStream"] 2370 name: Optional[str] = Field("", description="The stream name.", example=["Users"], title="Name") 2371 retriever: Union[SimpleRetriever, AsyncRetriever, CustomRetriever] = Field( 2372 ..., 2373 description="Component used to coordinate how records are extracted across stream slices and request pages.", 2374 title="Retriever", 2375 ) 2376 incremental_sync: Optional[ 2377 Union[DatetimeBasedCursor, IncrementingCountCursor, CustomIncrementalSync] 2378 ] = Field( 2379 None, 2380 description="Component used to fetch data incrementally based on a time field in the data.", 2381 title="Incremental Sync", 2382 ) 2383 primary_key: Optional[PrimaryKey] = Field("", title="Primary Key") 2384 schema_loader: Optional[ 2385 Union[ 2386 InlineSchemaLoader, 2387 DynamicSchemaLoader, 2388 JsonFileSchemaLoader, 2389 List[ 2390 Union[ 2391 InlineSchemaLoader, 2392 DynamicSchemaLoader, 2393 JsonFileSchemaLoader, 2394 CustomSchemaLoader, 2395 ] 2396 ], 2397 CustomSchemaLoader, 2398 ] 2399 ] = Field( 2400 None, 2401 description="One or many schema loaders can be used to retrieve the schema for the current stream. When multiple schema loaders are defined, schema properties will be merged together. Schema loaders defined first taking precedence in the event of a conflict.", 2402 title="Schema Loader", 2403 ) 2404 transformations: Optional[ 2405 List[ 2406 Union[ 2407 AddFields, 2408 RemoveFields, 2409 KeysToLower, 2410 KeysToSnakeCase, 2411 FlattenFields, 2412 DpathFlattenFields, 2413 KeysReplace, 2414 CustomTransformation, 2415 ] 2416 ] 2417 ] = Field( 2418 None, 2419 description="A list of transformations to be applied to each output record.", 2420 title="Transformations", 2421 ) 2422 state_migrations: Optional[ 2423 List[Union[LegacyToPerPartitionStateMigration, CustomStateMigration]] 2424 ] = Field( 2425 [], 2426 description="Array of state migrations to be applied on the input state", 2427 title="State Migrations", 2428 ) 2429 file_uploader: Optional[FileUploader] = Field( 2430 None, 2431 description="(experimental) Describes how to fetch a file", 2432 title="File Uploader", 2433 ) 2434 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters")
retriever: Union[SimpleRetriever, AsyncRetriever, CustomRetriever]
incremental_sync: Union[DatetimeBasedCursor, IncrementingCountCursor, CustomIncrementalSync, NoneType]
primary_key: Optional[PrimaryKey]
schema_loader: Union[InlineSchemaLoader, DynamicSchemaLoader, JsonFileSchemaLoader, List[Union[InlineSchemaLoader, DynamicSchemaLoader, JsonFileSchemaLoader, CustomSchemaLoader]], CustomSchemaLoader, NoneType]
transformations: Optional[List[Union[AddFields, RemoveFields, KeysToLower, KeysToSnakeCase, FlattenFields, DpathFlattenFields, KeysReplace, CustomTransformation]]]
state_migrations: Optional[List[Union[LegacyToPerPartitionStateMigration, CustomStateMigration]]]
file_uploader: Optional[FileUploader]
class
DeclarativeStream.Config:
class
SessionTokenAuthenticator(pydantic.v1.main.BaseModel):
2437class SessionTokenAuthenticator(BaseModel): 2438 type: Literal["SessionTokenAuthenticator"] 2439 login_requester: HttpRequester = Field( 2440 ..., 2441 description="Description of the request to perform to obtain a session token to perform data requests. The response body is expected to be a JSON object with a session token property.", 2442 examples=[ 2443 { 2444 "type": "HttpRequester", 2445 "url_base": "https://my_api.com", 2446 "path": "/login", 2447 "authenticator": { 2448 "type": "BasicHttpAuthenticator", 2449 "username": "{{ config.username }}", 2450 "password": "{{ config.password }}", 2451 }, 2452 } 2453 ], 2454 title="Login Requester", 2455 ) 2456 session_token_path: List[str] = Field( 2457 ..., 2458 description="The path in the response body returned from the login requester to the session token.", 2459 examples=[["access_token"], ["result", "token"]], 2460 title="Session Token Path", 2461 ) 2462 expiration_duration: Optional[str] = Field( 2463 None, 2464 description="The duration in ISO 8601 duration notation after which the session token expires, starting from the time it was obtained. Omitting it will result in the session token being refreshed for every request.", 2465 examples=["PT1H", "P1D"], 2466 title="Expiration Duration", 2467 ) 2468 request_authentication: Union[ 2469 SessionTokenRequestApiKeyAuthenticator, SessionTokenRequestBearerAuthenticator 2470 ] = Field( 2471 ..., 2472 description="Authentication method to use for requests sent to the API, specifying how to inject the session token.", 2473 title="Data Request Authentication", 2474 ) 2475 decoder: Optional[Union[JsonDecoder, XmlDecoder]] = Field( 2476 None, description="Component used to decode the response.", title="Decoder" 2477 ) 2478 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters")
login_requester: HttpRequester
request_authentication: Union[SessionTokenRequestApiKeyAuthenticator, SessionTokenRequestBearerAuthenticator]
decoder: Union[JsonDecoder, XmlDecoder, NoneType]
class
HttpRequester(airbyte_cdk.sources.declarative.models.base_model_with_deprecations.BaseModelWithDeprecations):
2481class HttpRequester(BaseModelWithDeprecations): 2482 type: Literal["HttpRequester"] 2483 url_base: Optional[str] = Field( 2484 None, 2485 deprecated=True, 2486 deprecation_message="Use `url` field instead.", 2487 description="Deprecated, use the `url` instead. Base URL of the API source. Do not put sensitive information (e.g. API tokens) into this field - Use the Authenticator component for this.", 2488 examples=[ 2489 "https://connect.squareup.com/v2", 2490 "{{ config['base_url'] or 'https://app.posthog.com'}}/api", 2491 "https://connect.squareup.com/v2/quotes/{{ stream_partition['id'] }}/quote_line_groups", 2492 "https://example.com/api/v1/resource/{{ next_page_token['id'] }}", 2493 ], 2494 title="API Base URL", 2495 ) 2496 url: Optional[str] = Field( 2497 None, 2498 description="The URL of the source API endpoint. Do not put sensitive information (e.g. API tokens) into this field - Use the Authenticator component for this.", 2499 examples=[ 2500 "https://connect.squareup.com/v2", 2501 "{{ config['url'] or 'https://app.posthog.com'}}/api", 2502 "https://connect.squareup.com/v2/quotes/{{ stream_partition['id'] }}/quote_line_groups", 2503 "https://example.com/api/v1/resource/{{ next_page_token['id'] }}", 2504 ], 2505 title="API Endpoint URL", 2506 ) 2507 path: Optional[str] = Field( 2508 None, 2509 deprecated=True, 2510 deprecation_message="Use `url` field instead.", 2511 description="Deprecated, use the `url` instead. Path the specific API endpoint that this stream represents. Do not put sensitive information (e.g. API tokens) into this field - Use the Authenticator component for this.", 2512 examples=[ 2513 "/products", 2514 "/quotes/{{ stream_partition['id'] }}/quote_line_groups", 2515 "/trades/{{ config['symbol_id'] }}/history", 2516 ], 2517 title="URL Path", 2518 ) 2519 http_method: Optional[HttpMethod] = Field( 2520 HttpMethod.GET, 2521 description="The HTTP method used to fetch data from the source (can be GET or POST).", 2522 examples=["GET", "POST"], 2523 title="HTTP Method", 2524 ) 2525 authenticator: Optional[ 2526 Union[ 2527 ApiKeyAuthenticator, 2528 BasicHttpAuthenticator, 2529 BearerAuthenticator, 2530 OAuthAuthenticator, 2531 JwtAuthenticator, 2532 SessionTokenAuthenticator, 2533 SelectiveAuthenticator, 2534 CustomAuthenticator, 2535 NoAuth, 2536 LegacySessionTokenAuthenticator, 2537 ] 2538 ] = Field( 2539 None, 2540 description="Authentication method to use for requests sent to the API.", 2541 title="Authenticator", 2542 ) 2543 fetch_properties_from_endpoint: Optional[PropertiesFromEndpoint] = Field( 2544 None, 2545 description="Allows for retrieving a dynamic set of properties from an API endpoint which can be injected into outbound request using the stream_partition.extra_fields.", 2546 title="Fetch Properties from Endpoint", 2547 ) 2548 request_parameters: Optional[Union[Dict[str, Union[str, QueryProperties]], str]] = Field( 2549 None, 2550 description="Specifies the query parameters that should be set on an outgoing HTTP request given the inputs.", 2551 examples=[ 2552 {"unit": "day"}, 2553 { 2554 "query": 'last_event_time BETWEEN TIMESTAMP "{{ stream_interval.start_time }}" AND TIMESTAMP "{{ stream_interval.end_time }}"' 2555 }, 2556 {"searchIn": "{{ ','.join(config.get('search_in', [])) }}"}, 2557 {"sort_by[asc]": "updated_at"}, 2558 ], 2559 title="Query Parameters", 2560 ) 2561 request_headers: Optional[Union[Dict[str, str], str]] = Field( 2562 None, 2563 description="Return any non-auth headers. Authentication headers will overwrite any overlapping headers returned from this method.", 2564 examples=[{"Output-Format": "JSON"}, {"Version": "{{ config['version'] }}"}], 2565 title="Request Headers", 2566 ) 2567 request_body_data: Optional[Union[Dict[str, str], str]] = Field( 2568 None, 2569 deprecated=True, 2570 deprecation_message="Use `request_body` field instead.", 2571 description="Specifies how to populate the body of the request with a non-JSON payload. Plain text will be sent as is, whereas objects will be converted to a urlencoded form.", 2572 examples=[ 2573 '[{"clause": {"type": "timestamp", "operator": 10, "parameters":\n [{"value": {{ stream_interval[\'start_time\'] | int * 1000 }} }]\n }, "orderBy": 1, "columnName": "Timestamp"}]/\n' 2574 ], 2575 title="Request Body Payload (Non-JSON)", 2576 ) 2577 request_body_json: Optional[Union[Dict[str, Any], str]] = Field( 2578 None, 2579 deprecated=True, 2580 deprecation_message="Use `request_body` field instead.", 2581 description="Specifies how to populate the body of the request with a JSON payload. Can contain nested objects.", 2582 examples=[ 2583 {"sort_order": "ASC", "sort_field": "CREATED_AT"}, 2584 {"key": "{{ config['value'] }}"}, 2585 {"sort": {"field": "updated_at", "order": "ascending"}}, 2586 ], 2587 title="Request Body JSON Payload", 2588 ) 2589 request_body: Optional[ 2590 Union[ 2591 RequestBodyPlainText, 2592 RequestBodyUrlEncodedForm, 2593 RequestBodyJsonObject, 2594 RequestBodyGraphQL, 2595 ] 2596 ] = Field( 2597 None, 2598 description="Specifies how to populate the body of the request with a payload. Can contain nested objects.", 2599 examples=[ 2600 { 2601 "type": "RequestBodyJsonObject", 2602 "value": {"sort_order": "ASC", "sort_field": "CREATED_AT"}, 2603 }, 2604 { 2605 "type": "RequestBodyJsonObject", 2606 "value": {"key": "{{ config['value'] }}"}, 2607 }, 2608 { 2609 "type": "RequestBodyJsonObject", 2610 "value": {"sort": {"field": "updated_at", "order": "ascending"}}, 2611 }, 2612 {"type": "RequestBodyPlainText", "value": "plain_text_body"}, 2613 { 2614 "type": "RequestBodyUrlEncodedForm", 2615 "value": {"param1": "value1", "param2": "{{ config['param2_value'] }}"}, 2616 }, 2617 { 2618 "type": "RequestBodyGraphQL", 2619 "value": { 2620 "query": { 2621 "param1": "value1", 2622 "param2": "{{ config['param2_value'] }}", 2623 } 2624 }, 2625 }, 2626 ], 2627 title="Request Body", 2628 ) 2629 error_handler: Optional[ 2630 Union[DefaultErrorHandler, CompositeErrorHandler, CustomErrorHandler] 2631 ] = Field( 2632 None, 2633 description="Error handler component that defines how to handle errors.", 2634 title="Error Handler", 2635 ) 2636 use_cache: Optional[bool] = Field( 2637 False, 2638 description="Enables stream requests caching. This field is automatically set by the CDK.", 2639 title="Use Cache", 2640 ) 2641 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters")
Pydantic BaseModel that warns when deprecated fields are accessed. The deprecation message is stored in the field's extra attributes. This class is used to create models that can have deprecated fields and show warnings when those fields are accessed or initialized.
The _deprecation_logs attribute is stored in the model itself.
The collected deprecation warnings are further propagated to the Airbyte log messages,
during the component creation process, in model_to_component._collect_model_deprecations().
The component implementation is not responsible for handling the deprecation warnings, since the deprecation warnings are already handled in the model itself.
http_method: Optional[HttpMethod]
authenticator: Union[ApiKeyAuthenticator, BasicHttpAuthenticator, BearerAuthenticator, OAuthAuthenticator, JwtAuthenticator, SessionTokenAuthenticator, SelectiveAuthenticator, CustomAuthenticator, NoAuth, LegacySessionTokenAuthenticator, NoneType]
fetch_properties_from_endpoint: Optional[PropertiesFromEndpoint]
request_parameters: Union[Dict[str, Union[str, QueryProperties]], str, NoneType]
request_body: Union[RequestBodyPlainText, RequestBodyUrlEncodedForm, RequestBodyJsonObject, RequestBodyGraphQL, NoneType]
error_handler: Union[DefaultErrorHandler, CompositeErrorHandler, CustomErrorHandler, NoneType]
class
DynamicSchemaLoader(pydantic.v1.main.BaseModel):
2644class DynamicSchemaLoader(BaseModel): 2645 type: Literal["DynamicSchemaLoader"] 2646 retriever: Union[SimpleRetriever, AsyncRetriever, CustomRetriever] = Field( 2647 ..., 2648 description="Component used to coordinate how records are extracted across stream slices and request pages.", 2649 title="Retriever", 2650 ) 2651 schema_filter: Optional[Union[RecordFilter, CustomRecordFilter]] = Field( 2652 None, 2653 description="Responsible for filtering fields to be added to json schema.", 2654 title="Schema Filter", 2655 ) 2656 schema_transformations: Optional[ 2657 List[ 2658 Union[ 2659 AddFields, 2660 RemoveFields, 2661 KeysToLower, 2662 KeysToSnakeCase, 2663 FlattenFields, 2664 DpathFlattenFields, 2665 KeysReplace, 2666 CustomTransformation, 2667 ] 2668 ] 2669 ] = Field( 2670 None, 2671 description="A list of transformations to be applied to the schema.", 2672 title="Schema Transformations", 2673 ) 2674 schema_type_identifier: SchemaTypeIdentifier 2675 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters")
retriever: Union[SimpleRetriever, AsyncRetriever, CustomRetriever]
schema_filter: Union[RecordFilter, CustomRecordFilter, NoneType]
schema_transformations: Optional[List[Union[AddFields, RemoveFields, KeysToLower, KeysToSnakeCase, FlattenFields, DpathFlattenFields, KeysReplace, CustomTransformation]]]
schema_type_identifier: SchemaTypeIdentifier
class
ParentStreamConfig(pydantic.v1.main.BaseModel):
2678class ParentStreamConfig(BaseModel): 2679 type: Literal["ParentStreamConfig"] 2680 lazy_read_pointer: Optional[List[str]] = Field( 2681 [], 2682 description="If set, this will enable lazy reading, using the initial read of parent records to extract child records.", 2683 title="Lazy Read Pointer", 2684 ) 2685 parent_key: str = Field( 2686 ..., 2687 description="The primary key of records from the parent stream that will be used during the retrieval of records for the current substream. This parent identifier field is typically a characteristic of the child records being extracted from the source API.", 2688 examples=["id", "{{ config['parent_record_id'] }}"], 2689 title="Parent Key", 2690 ) 2691 stream: Union[DeclarativeStream, StateDelegatingStream] = Field( 2692 ..., description="Reference to the parent stream.", title="Parent Stream" 2693 ) 2694 partition_field: str = Field( 2695 ..., 2696 description="While iterating over parent records during a sync, the parent_key value can be referenced by using this field.", 2697 examples=["parent_id", "{{ config['parent_partition_field'] }}"], 2698 title="Current Parent Key Value Identifier", 2699 ) 2700 request_option: Optional[RequestOption] = Field( 2701 None, 2702 description="A request option describing where the parent key value should be injected into and under what field name if applicable.", 2703 title="Request Option", 2704 ) 2705 incremental_dependency: Optional[bool] = Field( 2706 False, 2707 description="Indicates whether the parent stream should be read incrementally based on updates in the child stream.", 2708 title="Incremental Dependency", 2709 ) 2710 extra_fields: Optional[List[List[str]]] = Field( 2711 None, 2712 description="Array of field paths to include as additional fields in the stream slice. Each path is an array of strings representing keys to access fields in the respective parent record. Accessible via `stream_slice.extra_fields`. Missing fields are set to `None`.", 2713 title="Extra Fields", 2714 ) 2715 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters")
stream: Union[DeclarativeStream, StateDelegatingStream]
request_option: Optional[RequestOption]
class
PropertiesFromEndpoint(pydantic.v1.main.BaseModel):
2718class PropertiesFromEndpoint(BaseModel): 2719 type: Literal["PropertiesFromEndpoint"] 2720 property_field_path: List[str] = Field( 2721 ..., 2722 description="Describes the path to the field that should be extracted", 2723 examples=[["name"]], 2724 ) 2725 retriever: Union[SimpleRetriever, CustomRetriever] = Field( 2726 ..., 2727 description="Requester component that describes how to fetch the properties to query from a remote API endpoint.", 2728 ) 2729 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters")
retriever: Union[SimpleRetriever, CustomRetriever]
class
QueryProperties(pydantic.v1.main.BaseModel):
2732class QueryProperties(BaseModel): 2733 type: Literal["QueryProperties"] 2734 property_list: Union[List[str], PropertiesFromEndpoint] = Field( 2735 ..., 2736 description="The set of properties that will be queried for in the outbound request. This can either be statically defined or dynamic based on an API endpoint", 2737 title="Property List", 2738 ) 2739 always_include_properties: Optional[List[str]] = Field( 2740 None, 2741 description="The list of properties that should be included in every set of properties when multiple chunks of properties are being requested.", 2742 title="Always Include Properties", 2743 ) 2744 property_chunking: Optional[PropertyChunking] = Field( 2745 None, 2746 description="Defines how query properties will be grouped into smaller sets for APIs with limitations on the number of properties fetched per API request.", 2747 title="Property Chunking", 2748 ) 2749 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters")
property_list: Union[List[str], PropertiesFromEndpoint]
property_chunking: Optional[PropertyChunking]
class
StateDelegatingStream(pydantic.v1.main.BaseModel):
2752class StateDelegatingStream(BaseModel): 2753 type: Literal["StateDelegatingStream"] 2754 name: str = Field(..., description="The stream name.", example=["Users"], title="Name") 2755 full_refresh_stream: DeclarativeStream = Field( 2756 ..., 2757 description="Component used to coordinate how records are extracted across stream slices and request pages when the state is empty or not provided.", 2758 title="Full Refresh Stream", 2759 ) 2760 incremental_stream: DeclarativeStream = Field( 2761 ..., 2762 description="Component used to coordinate how records are extracted across stream slices and request pages when the state provided.", 2763 title="Incremental Stream", 2764 ) 2765 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters")
full_refresh_stream: DeclarativeStream
incremental_stream: DeclarativeStream
class
SimpleRetriever(pydantic.v1.main.BaseModel):
2768class SimpleRetriever(BaseModel): 2769 type: Literal["SimpleRetriever"] 2770 requester: Union[HttpRequester, CustomRequester] = Field( 2771 ..., 2772 description="Requester component that describes how to prepare HTTP requests to send to the source API.", 2773 ) 2774 decoder: Optional[ 2775 Union[ 2776 JsonDecoder, 2777 XmlDecoder, 2778 CsvDecoder, 2779 JsonlDecoder, 2780 GzipDecoder, 2781 IterableDecoder, 2782 ZipfileDecoder, 2783 CustomDecoder, 2784 ] 2785 ] = Field( 2786 None, 2787 description="Component decoding the response so records can be extracted.", 2788 title="HTTP Response Format", 2789 ) 2790 record_selector: RecordSelector = Field( 2791 ..., 2792 description="Component that describes how to extract records from a HTTP response.", 2793 ) 2794 paginator: Optional[Union[DefaultPaginator, NoPagination]] = Field( 2795 None, 2796 description="Paginator component that describes how to navigate through the API's pages.", 2797 ) 2798 ignore_stream_slicer_parameters_on_paginated_requests: Optional[bool] = Field( 2799 False, 2800 description="If true, the partition router and incremental request options will be ignored when paginating requests. Request options set directly on the requester will not be ignored.", 2801 ) 2802 partition_router: Optional[ 2803 Union[ 2804 ListPartitionRouter, 2805 SubstreamPartitionRouter, 2806 GroupingPartitionRouter, 2807 CustomPartitionRouter, 2808 List[ 2809 Union[ 2810 ListPartitionRouter, 2811 SubstreamPartitionRouter, 2812 GroupingPartitionRouter, 2813 CustomPartitionRouter, 2814 ] 2815 ], 2816 ] 2817 ] = Field( 2818 [], 2819 description="PartitionRouter component that describes how to partition the stream, enabling incremental syncs and checkpointing.", 2820 title="Partition Router", 2821 ) 2822 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters")
requester: Union[HttpRequester, CustomRequester]
decoder: Union[JsonDecoder, XmlDecoder, CsvDecoder, JsonlDecoder, GzipDecoder, IterableDecoder, ZipfileDecoder, CustomDecoder, NoneType]
record_selector: RecordSelector
paginator: Union[DefaultPaginator, NoPagination, NoneType]
partition_router: Union[ListPartitionRouter, SubstreamPartitionRouter, GroupingPartitionRouter, CustomPartitionRouter, List[Union[ListPartitionRouter, SubstreamPartitionRouter, GroupingPartitionRouter, CustomPartitionRouter]], NoneType]
class
AsyncRetriever(pydantic.v1.main.BaseModel):
2825class AsyncRetriever(BaseModel): 2826 type: Literal["AsyncRetriever"] 2827 record_selector: RecordSelector = Field( 2828 ..., 2829 description="Component that describes how to extract records from a HTTP response.", 2830 ) 2831 status_mapping: AsyncJobStatusMap = Field( 2832 ..., description="Async Job Status to Airbyte CDK Async Job Status mapping." 2833 ) 2834 status_extractor: Union[DpathExtractor, CustomRecordExtractor] = Field( 2835 ..., description="Responsible for fetching the actual status of the async job." 2836 ) 2837 download_target_extractor: Union[DpathExtractor, CustomRecordExtractor] = Field( 2838 ..., 2839 description="Responsible for fetching the final result `urls` provided by the completed / finished / ready async job.", 2840 ) 2841 download_extractor: Optional[ 2842 Union[DpathExtractor, CustomRecordExtractor, ResponseToFileExtractor] 2843 ] = Field(None, description="Responsible for fetching the records from provided urls.") 2844 creation_requester: Union[HttpRequester, CustomRequester] = Field( 2845 ..., 2846 description="Requester component that describes how to prepare HTTP requests to send to the source API to create the async server-side job.", 2847 ) 2848 polling_requester: Union[HttpRequester, CustomRequester] = Field( 2849 ..., 2850 description="Requester component that describes how to prepare HTTP requests to send to the source API to fetch the status of the running async job.", 2851 ) 2852 polling_job_timeout: Optional[Union[int, str]] = Field( 2853 None, 2854 description="The time in minutes after which the single Async Job should be considered as Timed Out.", 2855 ) 2856 download_target_requester: Optional[Union[HttpRequester, CustomRequester]] = Field( 2857 None, 2858 description="Requester component that describes how to prepare HTTP requests to send to the source API to extract the url from polling response by the completed async job.", 2859 ) 2860 download_requester: Union[HttpRequester, CustomRequester] = Field( 2861 ..., 2862 description="Requester component that describes how to prepare HTTP requests to send to the source API to download the data provided by the completed async job.", 2863 ) 2864 download_paginator: Optional[Union[DefaultPaginator, NoPagination]] = Field( 2865 None, 2866 description="Paginator component that describes how to navigate through the API's pages during download.", 2867 ) 2868 abort_requester: Optional[Union[HttpRequester, CustomRequester]] = Field( 2869 None, 2870 description="Requester component that describes how to prepare HTTP requests to send to the source API to abort a job once it is timed out from the source's perspective.", 2871 ) 2872 delete_requester: Optional[Union[HttpRequester, CustomRequester]] = Field( 2873 None, 2874 description="Requester component that describes how to prepare HTTP requests to send to the source API to delete a job once the records are extracted.", 2875 ) 2876 partition_router: Optional[ 2877 Union[ 2878 ListPartitionRouter, 2879 SubstreamPartitionRouter, 2880 GroupingPartitionRouter, 2881 CustomPartitionRouter, 2882 List[ 2883 Union[ 2884 ListPartitionRouter, 2885 SubstreamPartitionRouter, 2886 GroupingPartitionRouter, 2887 CustomPartitionRouter, 2888 ] 2889 ], 2890 ] 2891 ] = Field( 2892 [], 2893 description="PartitionRouter component that describes how to partition the stream, enabling incremental syncs and checkpointing.", 2894 title="Partition Router", 2895 ) 2896 decoder: Optional[ 2897 Union[ 2898 CsvDecoder, 2899 GzipDecoder, 2900 JsonDecoder, 2901 JsonlDecoder, 2902 IterableDecoder, 2903 XmlDecoder, 2904 ZipfileDecoder, 2905 CustomDecoder, 2906 ] 2907 ] = Field( 2908 None, 2909 description="Component decoding the response so records can be extracted.", 2910 title="HTTP Response Format", 2911 ) 2912 download_decoder: Optional[ 2913 Union[ 2914 CsvDecoder, 2915 GzipDecoder, 2916 JsonDecoder, 2917 JsonlDecoder, 2918 IterableDecoder, 2919 XmlDecoder, 2920 ZipfileDecoder, 2921 CustomDecoder, 2922 ] 2923 ] = Field( 2924 None, 2925 description="Component decoding the download response so records can be extracted.", 2926 title="Download HTTP Response Format", 2927 ) 2928 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters")
record_selector: RecordSelector
status_mapping: AsyncJobStatusMap
status_extractor: Union[DpathExtractor, CustomRecordExtractor]
download_target_extractor: Union[DpathExtractor, CustomRecordExtractor]
download_extractor: Union[DpathExtractor, CustomRecordExtractor, ResponseToFileExtractor, NoneType]
creation_requester: Union[HttpRequester, CustomRequester]
polling_requester: Union[HttpRequester, CustomRequester]
download_target_requester: Union[HttpRequester, CustomRequester, NoneType]
download_requester: Union[HttpRequester, CustomRequester]
download_paginator: Union[DefaultPaginator, NoPagination, NoneType]
abort_requester: Union[HttpRequester, CustomRequester, NoneType]
delete_requester: Union[HttpRequester, CustomRequester, NoneType]
partition_router: Union[ListPartitionRouter, SubstreamPartitionRouter, GroupingPartitionRouter, CustomPartitionRouter, List[Union[ListPartitionRouter, SubstreamPartitionRouter, GroupingPartitionRouter, CustomPartitionRouter]], NoneType]
decoder: Union[JsonDecoder, XmlDecoder, CsvDecoder, JsonlDecoder, GzipDecoder, IterableDecoder, ZipfileDecoder, CustomDecoder, NoneType]
download_decoder: Union[JsonDecoder, XmlDecoder, CsvDecoder, JsonlDecoder, GzipDecoder, IterableDecoder, ZipfileDecoder, CustomDecoder, NoneType]
class
SubstreamPartitionRouter(pydantic.v1.main.BaseModel):
2931class SubstreamPartitionRouter(BaseModel): 2932 type: Literal["SubstreamPartitionRouter"] 2933 parent_stream_configs: List[ParentStreamConfig] = Field( 2934 ..., 2935 description="Specifies which parent streams are being iterated over and how parent records should be used to partition the child stream data set.", 2936 title="Parent Stream Configs", 2937 ) 2938 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters")
parent_stream_configs: List[ParentStreamConfig]
class
GroupingPartitionRouter(pydantic.v1.main.BaseModel):
2941class GroupingPartitionRouter(BaseModel): 2942 type: Literal["GroupingPartitionRouter"] 2943 group_size: int = Field( 2944 ..., 2945 description="The number of partitions to include in each group. This determines how many partition values are batched together in a single slice.", 2946 examples=[10, 50], 2947 title="Group Size", 2948 ) 2949 underlying_partition_router: Union[ 2950 ListPartitionRouter, SubstreamPartitionRouter, CustomPartitionRouter 2951 ] = Field( 2952 ..., 2953 description="The partition router whose output will be grouped. This can be any valid partition router component.", 2954 title="Underlying Partition Router", 2955 ) 2956 deduplicate: Optional[bool] = Field( 2957 True, 2958 description="If true, ensures that partitions are unique within each group by removing duplicates based on the partition key.", 2959 title="Deduplicate Partitions", 2960 ) 2961 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters")
underlying_partition_router: Union[ListPartitionRouter, SubstreamPartitionRouter, CustomPartitionRouter]
class
HttpComponentsResolver(pydantic.v1.main.BaseModel):
2964class HttpComponentsResolver(BaseModel): 2965 type: Literal["HttpComponentsResolver"] 2966 retriever: Union[SimpleRetriever, AsyncRetriever, CustomRetriever] = Field( 2967 ..., 2968 description="Component used to coordinate how records are extracted across stream slices and request pages.", 2969 title="Retriever", 2970 ) 2971 components_mapping: List[ComponentMappingDefinition] 2972 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters")
retriever: Union[SimpleRetriever, AsyncRetriever, CustomRetriever]
components_mapping: List[ComponentMappingDefinition]
class
DynamicDeclarativeStream(pydantic.v1.main.BaseModel):
2975class DynamicDeclarativeStream(BaseModel): 2976 type: Literal["DynamicDeclarativeStream"] 2977 name: Optional[str] = Field( 2978 "", description="The dynamic stream name.", example=["Tables"], title="Name" 2979 ) 2980 stream_template: Union[DeclarativeStream, StateDelegatingStream] = Field( 2981 ..., description="Reference to the stream template.", title="Stream Template" 2982 ) 2983 components_resolver: Union[ 2984 HttpComponentsResolver, ConfigComponentsResolver, ParametrizedComponentsResolver 2985 ] = Field( 2986 ..., 2987 description="Component resolve and populates stream templates with components values.", 2988 title="Components Resolver", 2989 )
stream_template: Union[DeclarativeStream, StateDelegatingStream]
components_resolver: Union[HttpComponentsResolver, ConfigComponentsResolver, ParametrizedComponentsResolver]