airbyte_cdk.sources.declarative.models.declarative_component_schema
1# Copyright (c) 2025 Airbyte, Inc., all rights reserved. 2 3# generated by datamodel-codegen: 4# filename: declarative_component_schema.yaml 5 6from __future__ import annotations 7 8from enum import Enum 9from typing import Any, Dict, List, Literal, Optional, Union 10 11from pydantic.v1 import BaseModel, Extra, Field 12 13 14class AuthFlowType(Enum): 15 oauth2_0 = "oauth2.0" 16 oauth1_0 = "oauth1.0" 17 18 19class BasicHttpAuthenticator(BaseModel): 20 type: Literal["BasicHttpAuthenticator"] 21 username: str = Field( 22 ..., 23 description="The username that will be combined with the password, base64 encoded and used to make requests. Fill it in the user inputs.", 24 examples=["{{ config['username'] }}", "{{ config['api_key'] }}"], 25 title="Username", 26 ) 27 password: Optional[str] = Field( 28 "", 29 description="The password that will be combined with the username, base64 encoded and used to make requests. Fill it in the user inputs.", 30 examples=["{{ config['password'] }}", ""], 31 title="Password", 32 ) 33 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters") 34 35 36class BearerAuthenticator(BaseModel): 37 type: Literal["BearerAuthenticator"] 38 api_token: str = Field( 39 ..., 40 description="Token to inject as request header for authenticating with the API.", 41 examples=["{{ config['api_key'] }}", "{{ config['token'] }}"], 42 title="Bearer Token", 43 ) 44 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters") 45 46 47class DynamicStreamCheckConfig(BaseModel): 48 type: Literal["DynamicStreamCheckConfig"] 49 dynamic_stream_name: str = Field( 50 ..., description="The dynamic stream name.", title="Dynamic Stream Name" 51 ) 52 stream_count: Optional[int] = Field( 53 0, 54 description="The number of streams to attempt reading from during a check operation. If `stream_count` exceeds the total number of available streams, the minimum of the two values will be used.", 55 title="Stream Count", 56 ) 57 58 59class CheckDynamicStream(BaseModel): 60 type: Literal["CheckDynamicStream"] 61 stream_count: int = Field( 62 ..., 63 description="Numbers of the streams to try reading from when running a check operation.", 64 title="Stream Count", 65 ) 66 use_check_availability: Optional[bool] = Field( 67 True, 68 description="Enables stream check availability. This field is automatically set by the CDK.", 69 title="Use Check Availability", 70 ) 71 72 73class ConcurrencyLevel(BaseModel): 74 type: Optional[Literal["ConcurrencyLevel"]] = None 75 default_concurrency: Union[int, str] = Field( 76 ..., 77 description="The amount of concurrency that will applied during a sync. This value can be hardcoded or user-defined in the config if different users have varying volume thresholds in the target API.", 78 examples=[10, "{{ config['num_workers'] or 10 }}"], 79 title="Default Concurrency", 80 ) 81 max_concurrency: Optional[int] = Field( 82 None, 83 description="The maximum level of concurrency that will be used during a sync. This becomes a required field when the default_concurrency derives from the config, because it serves as a safeguard against a user-defined threshold that is too high.", 84 examples=[20, 100], 85 title="Max Concurrency", 86 ) 87 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters") 88 89 90class ConstantBackoffStrategy(BaseModel): 91 type: Literal["ConstantBackoffStrategy"] 92 backoff_time_in_seconds: Union[float, str] = Field( 93 ..., 94 description="Backoff time in seconds.", 95 examples=[30, 30.5, "{{ config['backoff_time'] }}"], 96 title="Backoff Time", 97 ) 98 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters") 99 100 101class CursorPagination(BaseModel): 102 type: Literal["CursorPagination"] 103 cursor_value: str = Field( 104 ..., 105 description="Value of the cursor defining the next page to fetch.", 106 examples=[ 107 "{{ headers.link.next.cursor }}", 108 "{{ last_record['key'] }}", 109 "{{ response['nextPage'] }}", 110 ], 111 title="Cursor Value", 112 ) 113 page_size: Optional[int] = Field( 114 None, 115 description="The number of records to include in each pages.", 116 examples=[100], 117 title="Page Size", 118 ) 119 stop_condition: Optional[str] = Field( 120 None, 121 description="Template string evaluating when to stop paginating.", 122 examples=[ 123 "{{ response.data.has_more is false }}", 124 "{{ 'next' not in headers['link'] }}", 125 ], 126 title="Stop Condition", 127 ) 128 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters") 129 130 131class CustomAuthenticator(BaseModel): 132 class Config: 133 extra = Extra.allow 134 135 type: Literal["CustomAuthenticator"] 136 class_name: str = Field( 137 ..., 138 description="Fully-qualified name of the class that will be implementing the custom authentication strategy. Has to be a sub class of DeclarativeAuthenticator. The format is `source_<name>.<package>.<class_name>`.", 139 examples=["source_railz.components.ShortLivedTokenAuthenticator"], 140 title="Class Name", 141 ) 142 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters") 143 144 145class CustomBackoffStrategy(BaseModel): 146 class Config: 147 extra = Extra.allow 148 149 type: Literal["CustomBackoffStrategy"] 150 class_name: str = Field( 151 ..., 152 description="Fully-qualified name of the class that will be implementing the custom backoff strategy. The format is `source_<name>.<package>.<class_name>`.", 153 examples=["source_railz.components.MyCustomBackoffStrategy"], 154 title="Class Name", 155 ) 156 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters") 157 158 159class CustomErrorHandler(BaseModel): 160 class Config: 161 extra = Extra.allow 162 163 type: Literal["CustomErrorHandler"] 164 class_name: str = Field( 165 ..., 166 description="Fully-qualified name of the class that will be implementing the custom error handler. The format is `source_<name>.<package>.<class_name>`.", 167 examples=["source_railz.components.MyCustomErrorHandler"], 168 title="Class Name", 169 ) 170 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters") 171 172 173class CustomIncrementalSync(BaseModel): 174 class Config: 175 extra = Extra.allow 176 177 type: Literal["CustomIncrementalSync"] 178 class_name: str = Field( 179 ..., 180 description="Fully-qualified name of the class that will be implementing the custom incremental sync. The format is `source_<name>.<package>.<class_name>`.", 181 examples=["source_railz.components.MyCustomIncrementalSync"], 182 title="Class Name", 183 ) 184 cursor_field: str = Field( 185 ..., 186 description="The location of the value on a record that will be used as a bookmark during sync.", 187 ) 188 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters") 189 190 191class CustomPaginationStrategy(BaseModel): 192 class Config: 193 extra = Extra.allow 194 195 type: Literal["CustomPaginationStrategy"] 196 class_name: str = Field( 197 ..., 198 description="Fully-qualified name of the class that will be implementing the custom pagination strategy. The format is `source_<name>.<package>.<class_name>`.", 199 examples=["source_railz.components.MyCustomPaginationStrategy"], 200 title="Class Name", 201 ) 202 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters") 203 204 205class CustomRecordExtractor(BaseModel): 206 class Config: 207 extra = Extra.allow 208 209 type: Literal["CustomRecordExtractor"] 210 class_name: str = Field( 211 ..., 212 description="Fully-qualified name of the class that will be implementing the custom record extraction strategy. The format is `source_<name>.<package>.<class_name>`.", 213 examples=["source_railz.components.MyCustomRecordExtractor"], 214 title="Class Name", 215 ) 216 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters") 217 218 219class CustomRecordFilter(BaseModel): 220 class Config: 221 extra = Extra.allow 222 223 type: Literal["CustomRecordFilter"] 224 class_name: str = Field( 225 ..., 226 description="Fully-qualified name of the class that will be implementing the custom record filter strategy. The format is `source_<name>.<package>.<class_name>`.", 227 examples=["source_railz.components.MyCustomCustomRecordFilter"], 228 title="Class Name", 229 ) 230 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters") 231 232 233class CustomRequester(BaseModel): 234 class Config: 235 extra = Extra.allow 236 237 type: Literal["CustomRequester"] 238 class_name: str = Field( 239 ..., 240 description="Fully-qualified name of the class that will be implementing the custom requester strategy. The format is `source_<name>.<package>.<class_name>`.", 241 examples=["source_railz.components.MyCustomRecordExtractor"], 242 title="Class Name", 243 ) 244 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters") 245 246 247class CustomRetriever(BaseModel): 248 class Config: 249 extra = Extra.allow 250 251 type: Literal["CustomRetriever"] 252 class_name: str = Field( 253 ..., 254 description="Fully-qualified name of the class that will be implementing the custom retriever strategy. The format is `source_<name>.<package>.<class_name>`.", 255 examples=["source_railz.components.MyCustomRetriever"], 256 title="Class Name", 257 ) 258 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters") 259 260 261class CustomPartitionRouter(BaseModel): 262 class Config: 263 extra = Extra.allow 264 265 type: Literal["CustomPartitionRouter"] 266 class_name: str = Field( 267 ..., 268 description="Fully-qualified name of the class that will be implementing the custom partition router. The format is `source_<name>.<package>.<class_name>`.", 269 examples=["source_railz.components.MyCustomPartitionRouter"], 270 title="Class Name", 271 ) 272 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters") 273 274 275class CustomSchemaLoader(BaseModel): 276 class Config: 277 extra = Extra.allow 278 279 type: Literal["CustomSchemaLoader"] 280 class_name: str = Field( 281 ..., 282 description="Fully-qualified name of the class that will be implementing the custom schema loader. The format is `source_<name>.<package>.<class_name>`.", 283 examples=["source_railz.components.MyCustomSchemaLoader"], 284 title="Class Name", 285 ) 286 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters") 287 288 289class CustomSchemaNormalization(BaseModel): 290 class Config: 291 extra = Extra.allow 292 293 type: Literal["CustomSchemaNormalization"] 294 class_name: str = Field( 295 ..., 296 description="Fully-qualified name of the class that will be implementing the custom normalization. The format is `source_<name>.<package>.<class_name>`.", 297 examples=[ 298 "source_amazon_seller_partner.components.LedgerDetailedViewReportsTypeTransformer" 299 ], 300 title="Class Name", 301 ) 302 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters") 303 304 305class CustomStateMigration(BaseModel): 306 class Config: 307 extra = Extra.allow 308 309 type: Literal["CustomStateMigration"] 310 class_name: str = Field( 311 ..., 312 description="Fully-qualified name of the class that will be implementing the custom state migration. The format is `source_<name>.<package>.<class_name>`.", 313 examples=["source_railz.components.MyCustomStateMigration"], 314 title="Class Name", 315 ) 316 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters") 317 318 319class CustomTransformation(BaseModel): 320 class Config: 321 extra = Extra.allow 322 323 type: Literal["CustomTransformation"] 324 class_name: str = Field( 325 ..., 326 description="Fully-qualified name of the class that will be implementing the custom transformation. The format is `source_<name>.<package>.<class_name>`.", 327 examples=["source_railz.components.MyCustomTransformation"], 328 title="Class Name", 329 ) 330 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters") 331 332 333class LegacyToPerPartitionStateMigration(BaseModel): 334 class Config: 335 extra = Extra.allow 336 337 type: Optional[Literal["LegacyToPerPartitionStateMigration"]] = None 338 339 340class Clamping(BaseModel): 341 target: str = Field( 342 ..., 343 description="The period of time that datetime windows will be clamped by", 344 examples=["DAY", "WEEK", "MONTH", "{{ config['target'] }}"], 345 title="Target", 346 ) 347 target_details: Optional[Dict[str, Any]] = None 348 349 350class Algorithm(Enum): 351 HS256 = "HS256" 352 HS384 = "HS384" 353 HS512 = "HS512" 354 ES256 = "ES256" 355 ES256K = "ES256K" 356 ES384 = "ES384" 357 ES512 = "ES512" 358 RS256 = "RS256" 359 RS384 = "RS384" 360 RS512 = "RS512" 361 PS256 = "PS256" 362 PS384 = "PS384" 363 PS512 = "PS512" 364 EdDSA = "EdDSA" 365 366 367class JwtHeaders(BaseModel): 368 class Config: 369 extra = Extra.forbid 370 371 kid: Optional[str] = Field( 372 None, 373 description="Private key ID for user account.", 374 examples=["{{ config['kid'] }}"], 375 title="Key Identifier", 376 ) 377 typ: Optional[str] = Field( 378 "JWT", 379 description="The media type of the complete JWT.", 380 examples=["JWT"], 381 title="Type", 382 ) 383 cty: Optional[str] = Field( 384 None, 385 description="Content type of JWT header.", 386 examples=["JWT"], 387 title="Content Type", 388 ) 389 390 391class JwtPayload(BaseModel): 392 class Config: 393 extra = Extra.forbid 394 395 iss: Optional[str] = Field( 396 None, 397 description="The user/principal that issued the JWT. Commonly a value unique to the user.", 398 examples=["{{ config['iss'] }}"], 399 title="Issuer", 400 ) 401 sub: Optional[str] = Field( 402 None, 403 description="The subject of the JWT. Commonly defined by the API.", 404 title="Subject", 405 ) 406 aud: Optional[str] = Field( 407 None, 408 description="The recipient that the JWT is intended for. Commonly defined by the API.", 409 examples=["appstoreconnect-v1"], 410 title="Audience", 411 ) 412 413 414class JwtAuthenticator(BaseModel): 415 type: Literal["JwtAuthenticator"] 416 secret_key: str = Field( 417 ..., 418 description="Secret used to sign the JSON web token.", 419 examples=["{{ config['secret_key'] }}"], 420 ) 421 base64_encode_secret_key: Optional[bool] = Field( 422 False, 423 description='When set to true, the secret key will be base64 encoded prior to being encoded as part of the JWT. Only set to "true" when required by the API.', 424 ) 425 algorithm: Algorithm = Field( 426 ..., 427 description="Algorithm used to sign the JSON web token.", 428 examples=["ES256", "HS256", "RS256", "{{ config['algorithm'] }}"], 429 ) 430 token_duration: Optional[int] = Field( 431 1200, 432 description="The amount of time in seconds a JWT token can be valid after being issued.", 433 examples=[1200, 3600], 434 title="Token Duration", 435 ) 436 header_prefix: Optional[str] = Field( 437 None, 438 description="The prefix to be used within the Authentication header.", 439 examples=["Bearer", "Basic"], 440 title="Header Prefix", 441 ) 442 jwt_headers: Optional[JwtHeaders] = Field( 443 None, 444 description="JWT headers used when signing JSON web token.", 445 title="JWT Headers", 446 ) 447 additional_jwt_headers: Optional[Dict[str, Any]] = Field( 448 None, 449 description="Additional headers to be included with the JWT headers object.", 450 title="Additional JWT Headers", 451 ) 452 jwt_payload: Optional[JwtPayload] = Field( 453 None, 454 description="JWT Payload used when signing JSON web token.", 455 title="JWT Payload", 456 ) 457 additional_jwt_payload: Optional[Dict[str, Any]] = Field( 458 None, 459 description="Additional properties to be added to the JWT payload.", 460 title="Additional JWT Payload Properties", 461 ) 462 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters") 463 464 465class RefreshTokenUpdater(BaseModel): 466 refresh_token_name: Optional[str] = Field( 467 "refresh_token", 468 description="The name of the property which contains the updated refresh token in the response from the token refresh endpoint.", 469 examples=["refresh_token"], 470 title="Refresh Token Property Name", 471 ) 472 access_token_config_path: Optional[List[str]] = Field( 473 ["credentials", "access_token"], 474 description="Config path to the access token. Make sure the field actually exists in the config.", 475 examples=[["credentials", "access_token"], ["access_token"]], 476 title="Config Path To Access Token", 477 ) 478 refresh_token_config_path: Optional[List[str]] = Field( 479 ["credentials", "refresh_token"], 480 description="Config path to the access token. Make sure the field actually exists in the config.", 481 examples=[["credentials", "refresh_token"], ["refresh_token"]], 482 title="Config Path To Refresh Token", 483 ) 484 token_expiry_date_config_path: Optional[List[str]] = Field( 485 ["credentials", "token_expiry_date"], 486 description="Config path to the expiry date. Make sure actually exists in the config.", 487 examples=[["credentials", "token_expiry_date"]], 488 title="Config Path To Expiry Date", 489 ) 490 refresh_token_error_status_codes: Optional[List[int]] = Field( 491 [], 492 description="Status Codes to Identify refresh token error in response (Refresh Token Error Key and Refresh Token Error Values should be also specified). Responses with one of the error status code and containing an error value will be flagged as a config error", 493 examples=[[400, 500]], 494 title="Refresh Token Error Status Codes", 495 ) 496 refresh_token_error_key: Optional[str] = Field( 497 "", 498 description="Key to Identify refresh token error in response (Refresh Token Error Status Codes and Refresh Token Error Values should be also specified).", 499 examples=["error"], 500 title="Refresh Token Error Key", 501 ) 502 refresh_token_error_values: Optional[List[str]] = Field( 503 [], 504 description='List of values to check for exception during token refresh process. Used to check if the error found in the response matches the key from the Refresh Token Error Key field (e.g. response={"error": "invalid_grant"}). Only responses with one of the error status code and containing an error value will be flagged as a config error', 505 examples=[["invalid_grant", "invalid_permissions"]], 506 title="Refresh Token Error Values", 507 ) 508 509 510class OAuthAuthenticator(BaseModel): 511 type: Literal["OAuthAuthenticator"] 512 client_id_name: Optional[str] = Field( 513 "client_id", 514 description="The name of the property to use to refresh the `access_token`.", 515 examples=["custom_app_id"], 516 title="Client ID Property Name", 517 ) 518 client_id: Optional[str] = Field( 519 None, 520 description="The OAuth client ID. Fill it in the user inputs.", 521 examples=["{{ config['client_id }}", "{{ config['credentials']['client_id }}"], 522 title="Client ID", 523 ) 524 client_secret_name: Optional[str] = Field( 525 "client_secret", 526 description="The name of the property to use to refresh the `access_token`.", 527 examples=["custom_app_secret"], 528 title="Client Secret Property Name", 529 ) 530 client_secret: Optional[str] = Field( 531 None, 532 description="The OAuth client secret. Fill it in the user inputs.", 533 examples=[ 534 "{{ config['client_secret }}", 535 "{{ config['credentials']['client_secret }}", 536 ], 537 title="Client Secret", 538 ) 539 refresh_token_name: Optional[str] = Field( 540 "refresh_token", 541 description="The name of the property to use to refresh the `access_token`.", 542 examples=["custom_app_refresh_value"], 543 title="Refresh Token Property Name", 544 ) 545 refresh_token: Optional[str] = Field( 546 None, 547 description="Credential artifact used to get a new access token.", 548 examples=[ 549 "{{ config['refresh_token'] }}", 550 "{{ config['credentials]['refresh_token'] }}", 551 ], 552 title="Refresh Token", 553 ) 554 token_refresh_endpoint: Optional[str] = Field( 555 None, 556 description="The full URL to call to obtain a new access token.", 557 examples=["https://connect.squareup.com/oauth2/token"], 558 title="Token Refresh Endpoint", 559 ) 560 access_token_name: Optional[str] = Field( 561 "access_token", 562 description="The name of the property which contains the access token in the response from the token refresh endpoint.", 563 examples=["access_token"], 564 title="Access Token Property Name", 565 ) 566 access_token_value: Optional[str] = Field( 567 None, 568 description="The value of the access_token to bypass the token refreshing using `refresh_token`.", 569 examples=["secret_access_token_value"], 570 title="Access Token Value", 571 ) 572 expires_in_name: Optional[str] = Field( 573 "expires_in", 574 description="The name of the property which contains the expiry date in the response from the token refresh endpoint.", 575 examples=["expires_in"], 576 title="Token Expiry Property Name", 577 ) 578 grant_type_name: Optional[str] = Field( 579 "grant_type", 580 description="The name of the property to use to refresh the `access_token`.", 581 examples=["custom_grant_type"], 582 title="Grant Type Property Name", 583 ) 584 grant_type: Optional[str] = Field( 585 "refresh_token", 586 description="Specifies the OAuth2 grant type. If set to refresh_token, the refresh_token needs to be provided as well. For client_credentials, only client id and secret are required. Other grant types are not officially supported.", 587 examples=["refresh_token", "client_credentials"], 588 title="Grant Type", 589 ) 590 refresh_request_body: Optional[Dict[str, Any]] = Field( 591 None, 592 description="Body of the request sent to get a new access token.", 593 examples=[ 594 { 595 "applicationId": "{{ config['application_id'] }}", 596 "applicationSecret": "{{ config['application_secret'] }}", 597 "token": "{{ config['token'] }}", 598 } 599 ], 600 title="Refresh Request Body", 601 ) 602 refresh_request_headers: Optional[Dict[str, Any]] = Field( 603 None, 604 description="Headers of the request sent to get a new access token.", 605 examples=[ 606 { 607 "Authorization": "<AUTH_TOKEN>", 608 "Content-Type": "application/x-www-form-urlencoded", 609 } 610 ], 611 title="Refresh Request Headers", 612 ) 613 scopes: Optional[List[str]] = Field( 614 None, 615 description="List of scopes that should be granted to the access token.", 616 examples=[["crm.list.read", "crm.objects.contacts.read", "crm.schema.contacts.read"]], 617 title="Scopes", 618 ) 619 token_expiry_date: Optional[str] = Field( 620 None, 621 description="The access token expiry date.", 622 examples=["2023-04-06T07:12:10.421833+00:00", 1680842386], 623 title="Token Expiry Date", 624 ) 625 token_expiry_date_format: Optional[str] = Field( 626 None, 627 description="The format of the time to expiration datetime. Provide it if the time is returned as a date-time string instead of seconds.", 628 examples=["%Y-%m-%d %H:%M:%S.%f+00:00"], 629 title="Token Expiry Date Format", 630 ) 631 refresh_token_updater: Optional[RefreshTokenUpdater] = Field( 632 None, 633 description="When the token updater is defined, new refresh tokens, access tokens and the access token expiry date are written back from the authentication response to the config object. This is important if the refresh token can only used once.", 634 title="Token Updater", 635 ) 636 profile_assertion: Optional[JwtAuthenticator] = Field( 637 None, 638 description="The authenticator being used to authenticate the client authenticator.", 639 title="Profile Assertion", 640 ) 641 use_profile_assertion: Optional[bool] = Field( 642 False, 643 description="Enable using profile assertion as a flow for OAuth authorization.", 644 title="Use Profile Assertion", 645 ) 646 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters") 647 648 649class Rate(BaseModel): 650 class Config: 651 extra = Extra.allow 652 653 limit: Union[int, str] = Field( 654 ..., 655 description="The maximum number of calls allowed within the interval.", 656 title="Limit", 657 ) 658 interval: str = Field( 659 ..., 660 description="The time interval for the rate limit.", 661 examples=["PT1H", "P1D"], 662 title="Interval", 663 ) 664 665 666class HttpRequestRegexMatcher(BaseModel): 667 class Config: 668 extra = Extra.allow 669 670 method: Optional[str] = Field( 671 None, description="The HTTP method to match (e.g., GET, POST).", title="Method" 672 ) 673 url_base: Optional[str] = Field( 674 None, 675 description='The base URL (scheme and host, e.g. "https://api.example.com") to match.', 676 title="URL Base", 677 ) 678 url_path_pattern: Optional[str] = Field( 679 None, 680 description="A regular expression pattern to match the URL path.", 681 title="URL Path Pattern", 682 ) 683 params: Optional[Dict[str, Any]] = Field( 684 None, description="The query parameters to match.", title="Parameters" 685 ) 686 headers: Optional[Dict[str, Any]] = Field( 687 None, description="The headers to match.", title="Headers" 688 ) 689 690 691class DpathExtractor(BaseModel): 692 type: Literal["DpathExtractor"] 693 field_path: List[str] = Field( 694 ..., 695 description='List of potentially nested fields describing the full path of the field to extract. Use "*" to extract all values from an array. See more info in the [docs](https://docs.airbyte.com/connector-development/config-based/understanding-the-yaml-file/record-selector).', 696 examples=[ 697 ["data"], 698 ["data", "records"], 699 ["data", "{{ parameters.name }}"], 700 ["data", "*", "record"], 701 ], 702 title="Field Path", 703 ) 704 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters") 705 706 707class ResponseToFileExtractor(BaseModel): 708 type: Literal["ResponseToFileExtractor"] 709 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters") 710 711 712class ExponentialBackoffStrategy(BaseModel): 713 type: Literal["ExponentialBackoffStrategy"] 714 factor: Optional[Union[float, str]] = Field( 715 5, 716 description="Multiplicative constant applied on each retry.", 717 examples=[5, 5.5, "10"], 718 title="Factor", 719 ) 720 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters") 721 722 723class GroupByKeyMergeStrategy(BaseModel): 724 type: Literal["GroupByKeyMergeStrategy"] 725 key: Union[str, List[str]] = Field( 726 ..., 727 description="The name of the field on the record whose value will be used to group properties that were retrieved through multiple API requests.", 728 examples=["id", ["parent_id", "end_date"]], 729 title="Key", 730 ) 731 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters") 732 733 734class SessionTokenRequestBearerAuthenticator(BaseModel): 735 type: Literal["Bearer"] 736 737 738class HttpMethod(Enum): 739 GET = "GET" 740 POST = "POST" 741 742 743class Action(Enum): 744 SUCCESS = "SUCCESS" 745 FAIL = "FAIL" 746 RETRY = "RETRY" 747 IGNORE = "IGNORE" 748 RATE_LIMITED = "RATE_LIMITED" 749 750 751class FailureType(Enum): 752 system_error = "system_error" 753 config_error = "config_error" 754 transient_error = "transient_error" 755 756 757class HttpResponseFilter(BaseModel): 758 type: Literal["HttpResponseFilter"] 759 action: Optional[Action] = Field( 760 None, 761 description="Action to execute if a response matches the filter.", 762 examples=["SUCCESS", "FAIL", "RETRY", "IGNORE", "RATE_LIMITED"], 763 title="Action", 764 ) 765 failure_type: Optional[FailureType] = Field( 766 None, 767 description="Failure type of traced exception if a response matches the filter.", 768 examples=["system_error", "config_error", "transient_error"], 769 title="Failure Type", 770 ) 771 error_message: Optional[str] = Field( 772 None, 773 description="Error Message to display if the response matches the filter.", 774 title="Error Message", 775 ) 776 error_message_contains: Optional[str] = Field( 777 None, 778 description="Match the response if its error message contains the substring.", 779 example=["This API operation is not enabled for this site"], 780 title="Error Message Substring", 781 ) 782 http_codes: Optional[List[int]] = Field( 783 None, 784 description="Match the response if its HTTP code is included in this list.", 785 examples=[[420, 429], [500]], 786 title="HTTP Codes", 787 unique_items=True, 788 ) 789 predicate: Optional[str] = Field( 790 None, 791 description="Match the response if the predicate evaluates to true.", 792 examples=[ 793 "{{ 'Too much requests' in response }}", 794 "{{ 'error_code' in response and response['error_code'] == 'ComplexityException' }}", 795 ], 796 title="Predicate", 797 ) 798 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters") 799 800 801class ComplexFieldType(BaseModel): 802 field_type: str 803 items: Optional[Union[str, ComplexFieldType]] = None 804 805 806class TypesMap(BaseModel): 807 target_type: Union[str, List[str], ComplexFieldType] 808 current_type: Union[str, List[str]] 809 condition: Optional[str] = None 810 811 812class SchemaTypeIdentifier(BaseModel): 813 type: Optional[Literal["SchemaTypeIdentifier"]] = None 814 schema_pointer: Optional[List[str]] = Field( 815 [], 816 description="List of nested fields defining the schema field path to extract. Defaults to [].", 817 title="Schema Path", 818 ) 819 key_pointer: List[str] = Field( 820 ..., 821 description="List of potentially nested fields describing the full path of the field key to extract.", 822 title="Key Path", 823 ) 824 type_pointer: Optional[List[str]] = Field( 825 None, 826 description="List of potentially nested fields describing the full path of the field type to extract.", 827 title="Type Path", 828 ) 829 types_mapping: Optional[List[TypesMap]] = None 830 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters") 831 832 833class InlineSchemaLoader(BaseModel): 834 type: Literal["InlineSchemaLoader"] 835 schema_: Optional[Dict[str, Any]] = Field( 836 None, 837 alias="schema", 838 description='Describes a streams\' schema. Refer to the <a href="https://docs.airbyte.com/understanding-airbyte/supported-data-types/">Data Types documentation</a> for more details on which types are valid.', 839 title="Schema", 840 ) 841 842 843class JsonFileSchemaLoader(BaseModel): 844 type: Literal["JsonFileSchemaLoader"] 845 file_path: Optional[str] = Field( 846 None, 847 description="Path to the JSON file defining the schema. The path is relative to the connector module's root.", 848 example=["./schemas/users.json"], 849 title="File Path", 850 ) 851 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters") 852 853 854class JsonDecoder(BaseModel): 855 type: Literal["JsonDecoder"] 856 857 858class JsonlDecoder(BaseModel): 859 type: Literal["JsonlDecoder"] 860 861 862class KeysToLower(BaseModel): 863 type: Literal["KeysToLower"] 864 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters") 865 866 867class KeysToSnakeCase(BaseModel): 868 type: Literal["KeysToSnakeCase"] 869 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters") 870 871 872class FlattenFields(BaseModel): 873 type: Literal["FlattenFields"] 874 flatten_lists: Optional[bool] = Field( 875 True, 876 description="Whether to flatten lists or leave it as is. Default is True.", 877 title="Flatten Lists", 878 ) 879 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters") 880 881 882class KeyTransformation(BaseModel): 883 prefix: Optional[Union[str, None]] = Field( 884 None, 885 description="Prefix to add for object keys. If not provided original keys remain unchanged.", 886 examples=[ 887 "flattened_", 888 ], 889 title="Key Prefix", 890 ) 891 suffix: Optional[Union[str, None]] = Field( 892 None, 893 description="Suffix to add for object keys. If not provided original keys remain unchanged.", 894 examples=[ 895 "_flattened", 896 ], 897 title="Key Suffix", 898 ) 899 900 901class DpathFlattenFields(BaseModel): 902 type: Literal["DpathFlattenFields"] 903 field_path: List[str] = Field( 904 ..., 905 description="A path to field that needs to be flattened.", 906 examples=[["data"], ["data", "*", "field"]], 907 title="Field Path", 908 ) 909 delete_origin_value: Optional[bool] = Field( 910 None, 911 description="Whether to delete the origin value or keep it. Default is False.", 912 title="Delete Origin Value", 913 ) 914 replace_record: Optional[bool] = Field( 915 None, 916 description="Whether to replace the origin record or not. Default is False.", 917 title="Replace Origin Record", 918 ) 919 key_transformation: Optional[Union[KeyTransformation, None]] = Field( 920 None, 921 description="Transformation for object keys. If not provided, original key will be used.", 922 title="Key transformation", 923 ) 924 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters") 925 926 927class KeysReplace(BaseModel): 928 type: Literal["KeysReplace"] 929 old: str = Field( 930 ..., 931 description="Old value to replace.", 932 examples=[ 933 " ", 934 "{{ record.id }}", 935 "{{ config['id'] }}", 936 "{{ stream_slice['id'] }}", 937 ], 938 title="Old value", 939 ) 940 new: str = Field( 941 ..., 942 description="New value to set.", 943 examples=[ 944 "_", 945 "{{ record.id }}", 946 "{{ config['id'] }}", 947 "{{ stream_slice['id'] }}", 948 ], 949 title="New value", 950 ) 951 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters") 952 953 954class IterableDecoder(BaseModel): 955 type: Literal["IterableDecoder"] 956 957 958class XmlDecoder(BaseModel): 959 type: Literal["XmlDecoder"] 960 961 962class CustomDecoder(BaseModel): 963 class Config: 964 extra = Extra.allow 965 966 type: Literal["CustomDecoder"] 967 class_name: str = Field( 968 ..., 969 description="Fully-qualified name of the class that will be implementing the custom decoding. Has to be a sub class of Decoder. The format is `source_<name>.<package>.<class_name>`.", 970 examples=["source_amazon_ads.components.GzipJsonlDecoder"], 971 title="Class Name", 972 ) 973 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters") 974 975 976class MinMaxDatetime(BaseModel): 977 type: Literal["MinMaxDatetime"] 978 datetime: str = Field( 979 ..., 980 description="Datetime value.", 981 examples=["2021-01-01", "2021-01-01T00:00:00Z", "{{ config['start_time'] }}"], 982 title="Datetime", 983 ) 984 datetime_format: Optional[str] = Field( 985 "", 986 description='Format of the datetime value. Defaults to "%Y-%m-%dT%H:%M:%S.%f%z" if left empty. Use placeholders starting with "%" to describe the format the API is using. The following placeholders are available:\n * **%s**: Epoch unix timestamp - `1686218963`\n * **%s_as_float**: Epoch unix timestamp in seconds as float with microsecond precision - `1686218963.123456`\n * **%ms**: Epoch unix timestamp - `1686218963123`\n * **%a**: Weekday (abbreviated) - `Sun`\n * **%A**: Weekday (full) - `Sunday`\n * **%w**: Weekday (decimal) - `0` (Sunday), `6` (Saturday)\n * **%d**: Day of the month (zero-padded) - `01`, `02`, ..., `31`\n * **%b**: Month (abbreviated) - `Jan`\n * **%B**: Month (full) - `January`\n * **%m**: Month (zero-padded) - `01`, `02`, ..., `12`\n * **%y**: Year (without century, zero-padded) - `00`, `01`, ..., `99`\n * **%Y**: Year (with century) - `0001`, `0002`, ..., `9999`\n * **%H**: Hour (24-hour, zero-padded) - `00`, `01`, ..., `23`\n * **%I**: Hour (12-hour, zero-padded) - `01`, `02`, ..., `12`\n * **%p**: AM/PM indicator\n * **%M**: Minute (zero-padded) - `00`, `01`, ..., `59`\n * **%S**: Second (zero-padded) - `00`, `01`, ..., `59`\n * **%f**: Microsecond (zero-padded to 6 digits) - `000000`, `000001`, ..., `999999`\n * **%_ms**: Millisecond (zero-padded to 3 digits) - `000`, `001`, ..., `999`\n * **%z**: UTC offset - `(empty)`, `+0000`, `-04:00`\n * **%Z**: Time zone name - `(empty)`, `UTC`, `GMT`\n * **%j**: Day of the year (zero-padded) - `001`, `002`, ..., `366`\n * **%U**: Week number of the year (Sunday as first day) - `00`, `01`, ..., `53`\n * **%W**: Week number of the year (Monday as first day) - `00`, `01`, ..., `53`\n * **%c**: Date and time representation - `Tue Aug 16 21:30:00 1988`\n * **%x**: Date representation - `08/16/1988`\n * **%X**: Time representation - `21:30:00`\n * **%%**: Literal \'%\' character\n\n Some placeholders depend on the locale of the underlying system - in most cases this locale is configured as en/US. For more information see the [Python documentation](https://docs.python.org/3/library/datetime.html#strftime-and-strptime-format-codes).\n', 987 examples=["%Y-%m-%dT%H:%M:%S.%f%z", "%Y-%m-%d", "%s"], 988 title="Datetime Format", 989 ) 990 max_datetime: Optional[str] = Field( 991 None, 992 description="Ceiling applied on the datetime value. Must be formatted with the datetime_format field.", 993 examples=["2021-01-01T00:00:00Z", "2021-01-01"], 994 title="Max Datetime", 995 ) 996 min_datetime: Optional[str] = Field( 997 None, 998 description="Floor applied on the datetime value. Must be formatted with the datetime_format field.", 999 examples=["2010-01-01T00:00:00Z", "2010-01-01"], 1000 title="Min Datetime", 1001 ) 1002 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters") 1003 1004 1005class NoAuth(BaseModel): 1006 type: Literal["NoAuth"] 1007 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters") 1008 1009 1010class NoPagination(BaseModel): 1011 type: Literal["NoPagination"] 1012 1013 1014class State(BaseModel): 1015 class Config: 1016 extra = Extra.allow 1017 1018 min: int 1019 max: int 1020 1021 1022class OauthConnectorInputSpecification(BaseModel): 1023 class Config: 1024 extra = Extra.allow 1025 1026 consent_url: str = Field( 1027 ..., 1028 description="The DeclarativeOAuth Specific string URL string template to initiate the authentication.\nThe placeholders are replaced during the processing to provide neccessary values.", 1029 examples=[ 1030 "https://domain.host.com/marketing_api/auth?{{client_id_key}}={{client_id_value}}&{{redirect_uri_key}}={{{{redirect_uri_value}} | urlEncoder}}&{{state_key}}={{state_value}}", 1031 "https://endpoint.host.com/oauth2/authorize?{{client_id_key}}={{client_id_value}}&{{redirect_uri_key}}={{{{redirect_uri_value}} | urlEncoder}}&{{scope_key}}={{{{scope_value}} | urlEncoder}}&{{state_key}}={{state_value}}&subdomain={{subdomain}}", 1032 ], 1033 title="Consent URL", 1034 ) 1035 scope: Optional[str] = Field( 1036 None, 1037 description="The DeclarativeOAuth Specific string of the scopes needed to be grant for authenticated user.", 1038 examples=["user:read user:read_orders workspaces:read"], 1039 title="Scopes", 1040 ) 1041 access_token_url: str = Field( 1042 ..., 1043 description="The DeclarativeOAuth Specific URL templated string to obtain the `access_token`, `refresh_token` etc.\nThe placeholders are replaced during the processing to provide neccessary values.", 1044 examples=[ 1045 "https://auth.host.com/oauth2/token?{{client_id_key}}={{client_id_value}}&{{client_secret_key}}={{client_secret_value}}&{{auth_code_key}}={{auth_code_value}}&{{redirect_uri_key}}={{{{redirect_uri_value}} | urlEncoder}}" 1046 ], 1047 title="Access Token URL", 1048 ) 1049 access_token_headers: Optional[Dict[str, Any]] = Field( 1050 None, 1051 description="The DeclarativeOAuth Specific optional headers to inject while exchanging the `auth_code` to `access_token` during `completeOAuthFlow` step.", 1052 examples=[ 1053 { 1054 "Authorization": "Basic {{ {{ client_id_value }}:{{ client_secret_value }} | base64Encoder }}" 1055 } 1056 ], 1057 title="Access Token Headers", 1058 ) 1059 access_token_params: Optional[Dict[str, Any]] = Field( 1060 None, 1061 description="The DeclarativeOAuth Specific optional query parameters to inject while exchanging the `auth_code` to `access_token` during `completeOAuthFlow` step.\nWhen this property is provided, the query params will be encoded as `Json` and included in the outgoing API request.", 1062 examples=[ 1063 { 1064 "{{ auth_code_key }}": "{{ auth_code_value }}", 1065 "{{ client_id_key }}": "{{ client_id_value }}", 1066 "{{ client_secret_key }}": "{{ client_secret_value }}", 1067 } 1068 ], 1069 title="Access Token Query Params (Json Encoded)", 1070 ) 1071 extract_output: Optional[List[str]] = Field( 1072 None, 1073 description="The DeclarativeOAuth Specific list of strings to indicate which keys should be extracted and returned back to the input config.", 1074 examples=[["access_token", "refresh_token", "other_field"]], 1075 title="Extract Output", 1076 ) 1077 state: Optional[State] = Field( 1078 None, 1079 description="The DeclarativeOAuth Specific object to provide the criteria of how the `state` query param should be constructed,\nincluding length and complexity.", 1080 examples=[{"min": 7, "max": 128}], 1081 title="Configurable State Query Param", 1082 ) 1083 client_id_key: Optional[str] = Field( 1084 None, 1085 description="The DeclarativeOAuth Specific optional override to provide the custom `client_id` key name, if required by data-provider.", 1086 examples=["my_custom_client_id_key_name"], 1087 title="Client ID Key Override", 1088 ) 1089 client_secret_key: Optional[str] = Field( 1090 None, 1091 description="The DeclarativeOAuth Specific optional override to provide the custom `client_secret` key name, if required by data-provider.", 1092 examples=["my_custom_client_secret_key_name"], 1093 title="Client Secret Key Override", 1094 ) 1095 scope_key: Optional[str] = Field( 1096 None, 1097 description="The DeclarativeOAuth Specific optional override to provide the custom `scope` key name, if required by data-provider.", 1098 examples=["my_custom_scope_key_key_name"], 1099 title="Scopes Key Override", 1100 ) 1101 state_key: Optional[str] = Field( 1102 None, 1103 description="The DeclarativeOAuth Specific optional override to provide the custom `state` key name, if required by data-provider.", 1104 examples=["my_custom_state_key_key_name"], 1105 title="State Key Override", 1106 ) 1107 auth_code_key: Optional[str] = Field( 1108 None, 1109 description="The DeclarativeOAuth Specific optional override to provide the custom `code` key name to something like `auth_code` or `custom_auth_code`, if required by data-provider.", 1110 examples=["my_custom_auth_code_key_name"], 1111 title="Auth Code Key Override", 1112 ) 1113 redirect_uri_key: Optional[str] = Field( 1114 None, 1115 description="The DeclarativeOAuth Specific optional override to provide the custom `redirect_uri` key name to something like `callback_uri`, if required by data-provider.", 1116 examples=["my_custom_redirect_uri_key_name"], 1117 title="Redirect URI Key Override", 1118 ) 1119 1120 1121class OAuthConfigSpecification(BaseModel): 1122 class Config: 1123 extra = Extra.allow 1124 1125 oauth_user_input_from_connector_config_specification: Optional[Dict[str, Any]] = Field( 1126 None, 1127 description="OAuth specific blob. This is a Json Schema used to validate Json configurations used as input to OAuth.\nMust be a valid non-nested JSON that refers to properties from ConnectorSpecification.connectionSpecification\nusing special annotation 'path_in_connector_config'.\nThese are input values the user is entering through the UI to authenticate to the connector, that might also shared\nas inputs for syncing data via the connector.\nExamples:\nif no connector values is shared during oauth flow, oauth_user_input_from_connector_config_specification=[]\nif connector values such as 'app_id' inside the top level are used to generate the API url for the oauth flow,\n oauth_user_input_from_connector_config_specification={\n app_id: {\n type: string\n path_in_connector_config: ['app_id']\n }\n }\nif connector values such as 'info.app_id' nested inside another object are used to generate the API url for the oauth flow,\n oauth_user_input_from_connector_config_specification={\n app_id: {\n type: string\n path_in_connector_config: ['info', 'app_id']\n }\n }", 1128 examples=[ 1129 {"app_id": {"type": "string", "path_in_connector_config": ["app_id"]}}, 1130 { 1131 "app_id": { 1132 "type": "string", 1133 "path_in_connector_config": ["info", "app_id"], 1134 } 1135 }, 1136 ], 1137 title="OAuth user input", 1138 ) 1139 oauth_connector_input_specification: Optional[OauthConnectorInputSpecification] = Field( 1140 None, 1141 description='The DeclarativeOAuth specific blob.\nPertains to the fields defined by the connector relating to the OAuth flow.\n\nInterpolation capabilities:\n- The variables placeholders are declared as `{{my_var}}`.\n- The nested resolution variables like `{{ {{my_nested_var}} }}` is allowed as well.\n\n- The allowed interpolation context is:\n + base64Encoder - encode to `base64`, {{ {{my_var_a}}:{{my_var_b}} | base64Encoder }}\n + base64Decorer - decode from `base64` encoded string, {{ {{my_string_variable_or_string_value}} | base64Decoder }}\n + urlEncoder - encode the input string to URL-like format, {{ https://test.host.com/endpoint | urlEncoder}}\n + urlDecorer - decode the input url-encoded string into text format, {{ urlDecoder:https%3A%2F%2Fairbyte.io | urlDecoder}}\n + codeChallengeS256 - get the `codeChallenge` encoded value to provide additional data-provider specific authorisation values, {{ {{state_value}} | codeChallengeS256 }}\n\nExamples:\n - The TikTok Marketing DeclarativeOAuth spec:\n {\n "oauth_connector_input_specification": {\n "type": "object",\n "additionalProperties": false,\n "properties": {\n "consent_url": "https://ads.tiktok.com/marketing_api/auth?{{client_id_key}}={{client_id_value}}&{{redirect_uri_key}}={{ {{redirect_uri_value}} | urlEncoder}}&{{state_key}}={{state_value}}",\n "access_token_url": "https://business-api.tiktok.com/open_api/v1.3/oauth2/access_token/",\n "access_token_params": {\n "{{ auth_code_key }}": "{{ auth_code_value }}",\n "{{ client_id_key }}": "{{ client_id_value }}",\n "{{ client_secret_key }}": "{{ client_secret_value }}"\n },\n "access_token_headers": {\n "Content-Type": "application/json",\n "Accept": "application/json"\n },\n "extract_output": ["data.access_token"],\n "client_id_key": "app_id",\n "client_secret_key": "secret",\n "auth_code_key": "auth_code"\n }\n }\n }', 1142 title="DeclarativeOAuth Connector Specification", 1143 ) 1144 complete_oauth_output_specification: Optional[Dict[str, Any]] = Field( 1145 None, 1146 description="OAuth specific blob. This is a Json Schema used to validate Json configurations produced by the OAuth flows as they are\nreturned by the distant OAuth APIs.\nMust be a valid JSON describing the fields to merge back to `ConnectorSpecification.connectionSpecification`.\nFor each field, a special annotation `path_in_connector_config` can be specified to determine where to merge it,\nExamples:\n complete_oauth_output_specification={\n refresh_token: {\n type: string,\n path_in_connector_config: ['credentials', 'refresh_token']\n }\n }", 1147 examples=[ 1148 { 1149 "refresh_token": { 1150 "type": "string,", 1151 "path_in_connector_config": ["credentials", "refresh_token"], 1152 } 1153 } 1154 ], 1155 title="OAuth output specification", 1156 ) 1157 complete_oauth_server_input_specification: Optional[Dict[str, Any]] = Field( 1158 None, 1159 description="OAuth specific blob. This is a Json Schema used to validate Json configurations persisted as Airbyte Server configurations.\nMust be a valid non-nested JSON describing additional fields configured by the Airbyte Instance or Workspace Admins to be used by the\nserver when completing an OAuth flow (typically exchanging an auth code for refresh token).\nExamples:\n complete_oauth_server_input_specification={\n client_id: {\n type: string\n },\n client_secret: {\n type: string\n }\n }", 1160 examples=[{"client_id": {"type": "string"}, "client_secret": {"type": "string"}}], 1161 title="OAuth input specification", 1162 ) 1163 complete_oauth_server_output_specification: Optional[Dict[str, Any]] = Field( 1164 None, 1165 description="OAuth specific blob. This is a Json Schema used to validate Json configurations persisted as Airbyte Server configurations that\nalso need to be merged back into the connector configuration at runtime.\nThis is a subset configuration of `complete_oauth_server_input_specification` that filters fields out to retain only the ones that\nare necessary for the connector to function with OAuth. (some fields could be used during oauth flows but not needed afterwards, therefore\nthey would be listed in the `complete_oauth_server_input_specification` but not `complete_oauth_server_output_specification`)\nMust be a valid non-nested JSON describing additional fields configured by the Airbyte Instance or Workspace Admins to be used by the\nconnector when using OAuth flow APIs.\nThese fields are to be merged back to `ConnectorSpecification.connectionSpecification`.\nFor each field, a special annotation `path_in_connector_config` can be specified to determine where to merge it,\nExamples:\n complete_oauth_server_output_specification={\n client_id: {\n type: string,\n path_in_connector_config: ['credentials', 'client_id']\n },\n client_secret: {\n type: string,\n path_in_connector_config: ['credentials', 'client_secret']\n }\n }", 1166 examples=[ 1167 { 1168 "client_id": { 1169 "type": "string,", 1170 "path_in_connector_config": ["credentials", "client_id"], 1171 }, 1172 "client_secret": { 1173 "type": "string,", 1174 "path_in_connector_config": ["credentials", "client_secret"], 1175 }, 1176 } 1177 ], 1178 title="OAuth server output specification", 1179 ) 1180 1181 1182class OffsetIncrement(BaseModel): 1183 type: Literal["OffsetIncrement"] 1184 page_size: Optional[Union[int, str]] = Field( 1185 None, 1186 description="The number of records to include in each pages.", 1187 examples=[100, "{{ config['page_size'] }}"], 1188 title="Limit", 1189 ) 1190 inject_on_first_request: Optional[bool] = Field( 1191 False, 1192 description="Using the `offset` with value `0` during the first request", 1193 title="Inject Offset", 1194 ) 1195 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters") 1196 1197 1198class PageIncrement(BaseModel): 1199 type: Literal["PageIncrement"] 1200 page_size: Optional[Union[int, str]] = Field( 1201 None, 1202 description="The number of records to include in each pages.", 1203 examples=[100, "100", "{{ config['page_size'] }}"], 1204 title="Page Size", 1205 ) 1206 start_from_page: Optional[int] = Field( 1207 0, 1208 description="Index of the first page to request.", 1209 examples=[0, 1], 1210 title="Start From Page", 1211 ) 1212 inject_on_first_request: Optional[bool] = Field( 1213 False, 1214 description="Using the `page number` with value defined by `start_from_page` during the first request", 1215 title="Inject Page Number", 1216 ) 1217 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters") 1218 1219 1220class PrimaryKey(BaseModel): 1221 __root__: Union[str, List[str], List[List[str]]] = Field( 1222 ..., 1223 description="The stream field to be used to distinguish unique records. Can either be a single field, an array of fields representing a composite key, or an array of arrays representing a composite key where the fields are nested fields.", 1224 examples=["id", ["code", "type"]], 1225 title="Primary Key", 1226 ) 1227 1228 1229class PropertyLimitType(Enum): 1230 characters = "characters" 1231 property_count = "property_count" 1232 1233 1234class PropertyChunking(BaseModel): 1235 type: Literal["PropertyChunking"] 1236 property_limit_type: PropertyLimitType = Field( 1237 ..., 1238 description="The type used to determine the maximum number of properties per chunk", 1239 title="Property Limit Type", 1240 ) 1241 property_limit: Optional[int] = Field( 1242 None, 1243 description="The maximum amount of properties that can be retrieved per request according to the limit type.", 1244 title="Property Limit", 1245 ) 1246 record_merge_strategy: Optional[GroupByKeyMergeStrategy] = Field( 1247 None, 1248 description="Dictates how to records that require multiple requests to get all properties should be emitted to the destination", 1249 title="Record Merge Strategy", 1250 ) 1251 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters") 1252 1253 1254class RecordFilter(BaseModel): 1255 type: Literal["RecordFilter"] 1256 condition: Optional[str] = Field( 1257 "", 1258 description="The predicate to filter a record. Records will be removed if evaluated to False.", 1259 examples=[ 1260 "{{ record['created_at'] >= stream_interval['start_time'] }}", 1261 "{{ record.status in ['active', 'expired'] }}", 1262 ], 1263 ) 1264 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters") 1265 1266 1267class SchemaNormalization(Enum): 1268 None_ = "None" 1269 Default = "Default" 1270 1271 1272class RemoveFields(BaseModel): 1273 type: Literal["RemoveFields"] 1274 condition: Optional[str] = Field( 1275 "", 1276 description="The predicate to filter a property by a property value. Property will be removed if it is empty OR expression is evaluated to True.,", 1277 examples=[ 1278 "{{ property|string == '' }}", 1279 "{{ property is integer }}", 1280 "{{ property|length > 5 }}", 1281 "{{ property == 'some_string_to_match' }}", 1282 ], 1283 ) 1284 field_pointers: List[List[str]] = Field( 1285 ..., 1286 description="Array of paths defining the field to remove. Each item is an array whose field describe the path of a field to remove.", 1287 examples=[["tags"], [["content", "html"], ["content", "plain_text"]]], 1288 title="Field Paths", 1289 ) 1290 1291 1292class RequestPath(BaseModel): 1293 type: Literal["RequestPath"] 1294 1295 1296class InjectInto(Enum): 1297 request_parameter = "request_parameter" 1298 header = "header" 1299 body_data = "body_data" 1300 body_json = "body_json" 1301 1302 1303class RequestOption(BaseModel): 1304 type: Literal["RequestOption"] 1305 field_name: Optional[str] = Field( 1306 None, 1307 description="Configures which key should be used in the location that the descriptor is being injected into. We hope to eventually deprecate this field in favor of `field_path` for all request_options, but must currently maintain it for backwards compatibility in the Builder.", 1308 examples=["segment_id"], 1309 title="Field Name", 1310 ) 1311 field_path: Optional[List[str]] = Field( 1312 None, 1313 description="Configures a path to be used for nested structures in JSON body requests (e.g. GraphQL queries)", 1314 examples=[["data", "viewer", "id"]], 1315 title="Field Path", 1316 ) 1317 inject_into: InjectInto = Field( 1318 ..., 1319 description="Configures where the descriptor should be set on the HTTP requests. Note that request parameters that are already encoded in the URL path will not be duplicated.", 1320 examples=["request_parameter", "header", "body_data", "body_json"], 1321 title="Inject Into", 1322 ) 1323 1324 1325class Schemas(BaseModel): 1326 pass 1327 1328 class Config: 1329 extra = Extra.allow 1330 1331 1332class LegacySessionTokenAuthenticator(BaseModel): 1333 type: Literal["LegacySessionTokenAuthenticator"] 1334 header: str = Field( 1335 ..., 1336 description="The name of the session token header that will be injected in the request", 1337 examples=["X-Session"], 1338 title="Session Request Header", 1339 ) 1340 login_url: str = Field( 1341 ..., 1342 description="Path of the login URL (do not include the base URL)", 1343 examples=["session"], 1344 title="Login Path", 1345 ) 1346 session_token: Optional[str] = Field( 1347 None, 1348 description="Session token to use if using a pre-defined token. Not needed if authenticating with username + password pair", 1349 example=["{{ config['session_token'] }}"], 1350 title="Session Token", 1351 ) 1352 session_token_response_key: str = Field( 1353 ..., 1354 description="Name of the key of the session token to be extracted from the response", 1355 examples=["id"], 1356 title="Response Token Response Key", 1357 ) 1358 username: Optional[str] = Field( 1359 None, 1360 description="Username used to authenticate and obtain a session token", 1361 examples=[" {{ config['username'] }}"], 1362 title="Username", 1363 ) 1364 password: Optional[str] = Field( 1365 "", 1366 description="Password used to authenticate and obtain a session token", 1367 examples=["{{ config['password'] }}", ""], 1368 title="Password", 1369 ) 1370 validate_session_url: str = Field( 1371 ..., 1372 description="Path of the URL to use to validate that the session token is valid (do not include the base URL)", 1373 examples=["user/current"], 1374 title="Validate Session Path", 1375 ) 1376 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters") 1377 1378 1379class CsvDecoder(BaseModel): 1380 type: Literal["CsvDecoder"] 1381 encoding: Optional[str] = "utf-8" 1382 delimiter: Optional[str] = "," 1383 1384 1385class AsyncJobStatusMap(BaseModel): 1386 type: Optional[Literal["AsyncJobStatusMap"]] = None 1387 running: List[str] 1388 completed: List[str] 1389 failed: List[str] 1390 timeout: List[str] 1391 1392 1393class ValueType(Enum): 1394 string = "string" 1395 number = "number" 1396 integer = "integer" 1397 boolean = "boolean" 1398 1399 1400class WaitTimeFromHeader(BaseModel): 1401 type: Literal["WaitTimeFromHeader"] 1402 header: str = Field( 1403 ..., 1404 description="The name of the response header defining how long to wait before retrying.", 1405 examples=["Retry-After"], 1406 title="Response Header Name", 1407 ) 1408 regex: Optional[str] = Field( 1409 None, 1410 description="Optional regex to apply on the header to extract its value. The regex should define a capture group defining the wait time.", 1411 examples=["([-+]?\\d+)"], 1412 title="Extraction Regex", 1413 ) 1414 max_waiting_time_in_seconds: Optional[float] = Field( 1415 None, 1416 description="Given the value extracted from the header is greater than this value, stop the stream.", 1417 examples=[3600], 1418 title="Max Waiting Time in Seconds", 1419 ) 1420 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters") 1421 1422 1423class WaitUntilTimeFromHeader(BaseModel): 1424 type: Literal["WaitUntilTimeFromHeader"] 1425 header: str = Field( 1426 ..., 1427 description="The name of the response header defining how long to wait before retrying.", 1428 examples=["wait_time"], 1429 title="Response Header", 1430 ) 1431 min_wait: Optional[Union[float, str]] = Field( 1432 None, 1433 description="Minimum time to wait before retrying.", 1434 examples=[10, "60"], 1435 title="Minimum Wait Time", 1436 ) 1437 regex: Optional[str] = Field( 1438 None, 1439 description="Optional regex to apply on the header to extract its value. The regex should define a capture group defining the wait time.", 1440 examples=["([-+]?\\d+)"], 1441 title="Extraction Regex", 1442 ) 1443 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters") 1444 1445 1446class ComponentMappingDefinition(BaseModel): 1447 type: Literal["ComponentMappingDefinition"] 1448 field_path: List[str] = Field( 1449 ..., 1450 description="A list of potentially nested fields indicating the full path where value will be added or updated.", 1451 examples=[ 1452 ["data"], 1453 ["data", "records"], 1454 ["data", 1, "name"], 1455 ["data", "{{ components_values.name }}"], 1456 ["data", "*", "record"], 1457 ["*", "**", "name"], 1458 ], 1459 title="Field Path", 1460 ) 1461 value: str = Field( 1462 ..., 1463 description="The dynamic or static value to assign to the key. Interpolated values can be used to dynamically determine the value during runtime.", 1464 examples=[ 1465 "{{ components_values['updates'] }}", 1466 "{{ components_values['MetaData']['LastUpdatedTime'] }}", 1467 "{{ config['segment_id'] }}", 1468 "{{ stream_slice['parent_id'] }}", 1469 "{{ stream_slice['extra_fields']['name'] }}", 1470 ], 1471 title="Value", 1472 ) 1473 value_type: Optional[ValueType] = Field( 1474 None, 1475 description="The expected data type of the value. If omitted, the type will be inferred from the value provided.", 1476 title="Value Type", 1477 ) 1478 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters") 1479 1480 1481class StreamConfig(BaseModel): 1482 type: Literal["StreamConfig"] 1483 configs_pointer: List[str] = Field( 1484 ..., 1485 description="A list of potentially nested fields indicating the full path in source config file where streams configs located.", 1486 examples=[["data"], ["data", "streams"], ["data", "{{ parameters.name }}"]], 1487 title="Configs Pointer", 1488 ) 1489 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters") 1490 1491 1492class ConfigComponentsResolver(BaseModel): 1493 type: Literal["ConfigComponentsResolver"] 1494 stream_config: StreamConfig 1495 components_mapping: List[ComponentMappingDefinition] 1496 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters") 1497 1498 1499class AddedFieldDefinition(BaseModel): 1500 type: Literal["AddedFieldDefinition"] 1501 path: List[str] = Field( 1502 ..., 1503 description="List of strings defining the path where to add the value on the record.", 1504 examples=[["segment_id"], ["metadata", "segment_id"]], 1505 title="Path", 1506 ) 1507 value: str = Field( 1508 ..., 1509 description="Value of the new field. Use {{ record['existing_field'] }} syntax to refer to other fields in the record.", 1510 examples=[ 1511 "{{ record['updates'] }}", 1512 "{{ record['MetaData']['LastUpdatedTime'] }}", 1513 "{{ stream_partition['segment_id'] }}", 1514 ], 1515 title="Value", 1516 ) 1517 value_type: Optional[ValueType] = Field( 1518 None, 1519 description="Type of the value. If not specified, the type will be inferred from the value.", 1520 title="Value Type", 1521 ) 1522 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters") 1523 1524 1525class AddFields(BaseModel): 1526 type: Literal["AddFields"] 1527 fields: List[AddedFieldDefinition] = Field( 1528 ..., 1529 description="List of transformations (path and corresponding value) that will be added to the record.", 1530 title="Fields", 1531 ) 1532 condition: Optional[str] = Field( 1533 "", 1534 description="Fields will be added if expression is evaluated to True.", 1535 examples=[ 1536 "{{ property|string == '' }}", 1537 "{{ property is integer }}", 1538 "{{ property|length > 5 }}", 1539 "{{ property == 'some_string_to_match' }}", 1540 ], 1541 ) 1542 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters") 1543 1544 1545class ApiKeyAuthenticator(BaseModel): 1546 type: Literal["ApiKeyAuthenticator"] 1547 api_token: Optional[str] = Field( 1548 None, 1549 description="The API key to inject in the request. Fill it in the user inputs.", 1550 examples=["{{ config['api_key'] }}", "Token token={{ config['api_key'] }}"], 1551 title="API Key", 1552 ) 1553 header: Optional[str] = Field( 1554 None, 1555 description="The name of the HTTP header that will be set to the API key. This setting is deprecated, use inject_into instead. Header and inject_into can not be defined at the same time.", 1556 examples=["Authorization", "Api-Token", "X-Auth-Token"], 1557 title="Header Name", 1558 ) 1559 inject_into: Optional[RequestOption] = Field( 1560 None, 1561 description="Configure how the API Key will be sent in requests to the source API. Either inject_into or header has to be defined.", 1562 examples=[ 1563 {"inject_into": "header", "field_name": "Authorization"}, 1564 {"inject_into": "request_parameter", "field_name": "authKey"}, 1565 ], 1566 title="Inject API Key Into Outgoing HTTP Request", 1567 ) 1568 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters") 1569 1570 1571class AuthFlow(BaseModel): 1572 auth_flow_type: Optional[AuthFlowType] = Field( 1573 None, description="The type of auth to use", title="Auth flow type" 1574 ) 1575 predicate_key: Optional[List[str]] = Field( 1576 None, 1577 description="JSON path to a field in the connectorSpecification that should exist for the advanced auth to be applicable.", 1578 examples=[["credentials", "auth_type"]], 1579 title="Predicate key", 1580 ) 1581 predicate_value: Optional[str] = Field( 1582 None, 1583 description="Value of the predicate_key fields for the advanced auth to be applicable.", 1584 examples=["Oauth"], 1585 title="Predicate value", 1586 ) 1587 oauth_config_specification: Optional[OAuthConfigSpecification] = None 1588 1589 1590class CheckStream(BaseModel): 1591 type: Literal["CheckStream"] 1592 stream_names: Optional[List[str]] = Field( 1593 None, 1594 description="Names of the streams to try reading from when running a check operation.", 1595 examples=[["users"], ["users", "contacts"]], 1596 title="Stream Names", 1597 ) 1598 dynamic_streams_check_configs: Optional[List[DynamicStreamCheckConfig]] = None 1599 1600 1601class IncrementingCountCursor(BaseModel): 1602 type: Literal["IncrementingCountCursor"] 1603 cursor_field: str = Field( 1604 ..., 1605 description="The location of the value on a record that will be used as a bookmark during sync. To ensure no data loss, the API must return records in ascending order based on the cursor field. Nested fields are not supported, so the field must be at the top level of the record. You can use a combination of Add Field and Remove Field transformations to move the nested field to the top.", 1606 examples=["created_at", "{{ config['record_cursor'] }}"], 1607 title="Cursor Field", 1608 ) 1609 start_value: Optional[Union[str, int]] = Field( 1610 None, 1611 description="The value that determines the earliest record that should be synced.", 1612 examples=[0, "{{ config['start_value'] }}"], 1613 title="Start Value", 1614 ) 1615 start_value_option: Optional[RequestOption] = Field( 1616 None, 1617 description="Optionally configures how the start value will be sent in requests to the source API.", 1618 title="Inject Start Value Into Outgoing HTTP Request", 1619 ) 1620 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters") 1621 1622 1623class DatetimeBasedCursor(BaseModel): 1624 type: Literal["DatetimeBasedCursor"] 1625 clamping: Optional[Clamping] = Field( 1626 None, 1627 description="This option is used to adjust the upper and lower boundaries of each datetime window to beginning and end of the provided target period (day, week, month)", 1628 title="Date Range Clamping", 1629 ) 1630 cursor_field: str = Field( 1631 ..., 1632 description="The location of the value on a record that will be used as a bookmark during sync. To ensure no data loss, the API must return records in ascending order based on the cursor field. Nested fields are not supported, so the field must be at the top level of the record. You can use a combination of Add Field and Remove Field transformations to move the nested field to the top.", 1633 examples=["created_at", "{{ config['record_cursor'] }}"], 1634 title="Cursor Field", 1635 ) 1636 datetime_format: str = Field( 1637 ..., 1638 description="The datetime format used to format the datetime values that are sent in outgoing requests to the API. Use placeholders starting with \"%\" to describe the format the API is using. The following placeholders are available:\n * **%s**: Epoch unix timestamp - `1686218963`\n * **%s_as_float**: Epoch unix timestamp in seconds as float with microsecond precision - `1686218963.123456`\n * **%ms**: Epoch unix timestamp (milliseconds) - `1686218963123`\n * **%a**: Weekday (abbreviated) - `Sun`\n * **%A**: Weekday (full) - `Sunday`\n * **%w**: Weekday (decimal) - `0` (Sunday), `6` (Saturday)\n * **%d**: Day of the month (zero-padded) - `01`, `02`, ..., `31`\n * **%b**: Month (abbreviated) - `Jan`\n * **%B**: Month (full) - `January`\n * **%m**: Month (zero-padded) - `01`, `02`, ..., `12`\n * **%y**: Year (without century, zero-padded) - `00`, `01`, ..., `99`\n * **%Y**: Year (with century) - `0001`, `0002`, ..., `9999`\n * **%H**: Hour (24-hour, zero-padded) - `00`, `01`, ..., `23`\n * **%I**: Hour (12-hour, zero-padded) - `01`, `02`, ..., `12`\n * **%p**: AM/PM indicator\n * **%M**: Minute (zero-padded) - `00`, `01`, ..., `59`\n * **%S**: Second (zero-padded) - `00`, `01`, ..., `59`\n * **%f**: Microsecond (zero-padded to 6 digits) - `000000`\n * **%_ms**: Millisecond (zero-padded to 3 digits) - `000`\n * **%z**: UTC offset - `(empty)`, `+0000`, `-04:00`\n * **%Z**: Time zone name - `(empty)`, `UTC`, `GMT`\n * **%j**: Day of the year (zero-padded) - `001`, `002`, ..., `366`\n * **%U**: Week number of the year (starting Sunday) - `00`, ..., `53`\n * **%W**: Week number of the year (starting Monday) - `00`, ..., `53`\n * **%c**: Date and time - `Tue Aug 16 21:30:00 1988`\n * **%x**: Date standard format - `08/16/1988`\n * **%X**: Time standard format - `21:30:00`\n * **%%**: Literal '%' character\n\n Some placeholders depend on the locale of the underlying system - in most cases this locale is configured as en/US. For more information see the [Python documentation](https://docs.python.org/3/library/datetime.html#strftime-and-strptime-format-codes).\n", 1639 examples=["%Y-%m-%dT%H:%M:%S.%f%z", "%Y-%m-%d", "%s", "%ms", "%s_as_float"], 1640 title="Outgoing Datetime Format", 1641 ) 1642 start_datetime: Union[str, MinMaxDatetime] = Field( 1643 ..., 1644 description="The datetime that determines the earliest record that should be synced.", 1645 examples=["2020-01-1T00:00:00Z", "{{ config['start_time'] }}"], 1646 title="Start Datetime", 1647 ) 1648 cursor_datetime_formats: Optional[List[str]] = Field( 1649 None, 1650 description="The possible formats for the cursor field, in order of preference. The first format that matches the cursor field value will be used to parse it. If not provided, the `datetime_format` will be used.", 1651 title="Cursor Datetime Formats", 1652 ) 1653 cursor_granularity: Optional[str] = Field( 1654 None, 1655 description="Smallest increment the datetime_format has (ISO 8601 duration) that is used to ensure the start of a slice does not overlap with the end of the previous one, e.g. for %Y-%m-%d the granularity should be P1D, for %Y-%m-%dT%H:%M:%SZ the granularity should be PT1S. Given this field is provided, `step` needs to be provided as well.", 1656 examples=["PT1S"], 1657 title="Cursor Granularity", 1658 ) 1659 end_datetime: Optional[Union[str, MinMaxDatetime]] = Field( 1660 None, 1661 description="The datetime that determines the last record that should be synced. If not provided, `{{ now_utc() }}` will be used.", 1662 examples=["2021-01-1T00:00:00Z", "{{ now_utc() }}", "{{ day_delta(-1) }}"], 1663 title="End Datetime", 1664 ) 1665 end_time_option: Optional[RequestOption] = Field( 1666 None, 1667 description="Optionally configures how the end datetime will be sent in requests to the source API.", 1668 title="Inject End Time Into Outgoing HTTP Request", 1669 ) 1670 is_data_feed: Optional[bool] = Field( 1671 None, 1672 description="A data feed API is an API that does not allow filtering and paginates the content from the most recent to the least recent. Given this, the CDK needs to know when to stop paginating and this field will generate a stop condition for pagination.", 1673 title="Whether the target API is formatted as a data feed", 1674 ) 1675 is_client_side_incremental: Optional[bool] = Field( 1676 None, 1677 description="If the target API endpoint does not take cursor values to filter records and returns all records anyway, the connector with this cursor will filter out records locally, and only emit new records from the last sync, hence incremental. This means that all records would be read from the API, but only new records will be emitted to the destination.", 1678 title="Whether the target API does not support filtering and returns all data (the cursor filters records in the client instead of the API side)", 1679 ) 1680 is_compare_strictly: Optional[bool] = Field( 1681 False, 1682 description="Set to True if the target API does not accept queries where the start time equal the end time.", 1683 title="Whether to skip requests if the start time equals the end time", 1684 ) 1685 global_substream_cursor: Optional[bool] = Field( 1686 False, 1687 description="This setting optimizes performance when the parent stream has thousands of partitions by storing the cursor as a single value rather than per partition. Notably, the substream state is updated only at the end of the sync, which helps prevent data loss in case of a sync failure. See more info in the [docs](https://docs.airbyte.com/connector-development/config-based/understanding-the-yaml-file/incremental-syncs).", 1688 title="Whether to store cursor as one value instead of per partition", 1689 ) 1690 lookback_window: Optional[str] = Field( 1691 None, 1692 description="Time interval before the start_datetime to read data for, e.g. P1M for looking back one month.", 1693 examples=["P1D", "P{{ config['lookback_days'] }}D"], 1694 title="Lookback Window", 1695 ) 1696 partition_field_end: Optional[str] = Field( 1697 None, 1698 description="Name of the partition start time field.", 1699 examples=["ending_time"], 1700 title="Partition Field End", 1701 ) 1702 partition_field_start: Optional[str] = Field( 1703 None, 1704 description="Name of the partition end time field.", 1705 examples=["starting_time"], 1706 title="Partition Field Start", 1707 ) 1708 start_time_option: Optional[RequestOption] = Field( 1709 None, 1710 description="Optionally configures how the start datetime will be sent in requests to the source API.", 1711 title="Inject Start Time Into Outgoing HTTP Request", 1712 ) 1713 step: Optional[str] = Field( 1714 None, 1715 description="The size of the time window (ISO8601 duration). Given this field is provided, `cursor_granularity` needs to be provided as well.", 1716 examples=["P1W", "{{ config['step_increment'] }}"], 1717 title="Step", 1718 ) 1719 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters") 1720 1721 1722class FixedWindowCallRatePolicy(BaseModel): 1723 class Config: 1724 extra = Extra.allow 1725 1726 type: Literal["FixedWindowCallRatePolicy"] 1727 period: str = Field( 1728 ..., description="The time interval for the rate limit window.", title="Period" 1729 ) 1730 call_limit: int = Field( 1731 ..., 1732 description="The maximum number of calls allowed within the period.", 1733 title="Call Limit", 1734 ) 1735 matchers: List[HttpRequestRegexMatcher] = Field( 1736 ..., 1737 description="List of matchers that define which requests this policy applies to.", 1738 title="Matchers", 1739 ) 1740 1741 1742class MovingWindowCallRatePolicy(BaseModel): 1743 class Config: 1744 extra = Extra.allow 1745 1746 type: Literal["MovingWindowCallRatePolicy"] 1747 rates: List[Rate] = Field( 1748 ..., 1749 description="List of rates that define the call limits for different time intervals.", 1750 title="Rates", 1751 ) 1752 matchers: List[HttpRequestRegexMatcher] = Field( 1753 ..., 1754 description="List of matchers that define which requests this policy applies to.", 1755 title="Matchers", 1756 ) 1757 1758 1759class UnlimitedCallRatePolicy(BaseModel): 1760 class Config: 1761 extra = Extra.allow 1762 1763 type: Literal["UnlimitedCallRatePolicy"] 1764 matchers: List[HttpRequestRegexMatcher] = Field( 1765 ..., 1766 description="List of matchers that define which requests this policy applies to.", 1767 title="Matchers", 1768 ) 1769 1770 1771class DefaultErrorHandler(BaseModel): 1772 type: Literal["DefaultErrorHandler"] 1773 backoff_strategies: Optional[ 1774 List[ 1775 Union[ 1776 ConstantBackoffStrategy, 1777 CustomBackoffStrategy, 1778 ExponentialBackoffStrategy, 1779 WaitTimeFromHeader, 1780 WaitUntilTimeFromHeader, 1781 ] 1782 ] 1783 ] = Field( 1784 None, 1785 description="List of backoff strategies to use to determine how long to wait before retrying a retryable request.", 1786 title="Backoff Strategies", 1787 ) 1788 max_retries: Optional[int] = Field( 1789 5, 1790 description="The maximum number of time to retry a retryable request before giving up and failing.", 1791 examples=[5, 0, 10], 1792 title="Max Retry Count", 1793 ) 1794 response_filters: Optional[List[HttpResponseFilter]] = Field( 1795 None, 1796 description="List of response filters to iterate on when deciding how to handle an error. When using an array of multiple filters, the filters will be applied sequentially and the response will be selected if it matches any of the filter's predicate.", 1797 title="Response Filters", 1798 ) 1799 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters") 1800 1801 1802class DefaultPaginator(BaseModel): 1803 type: Literal["DefaultPaginator"] 1804 pagination_strategy: Union[ 1805 CursorPagination, CustomPaginationStrategy, OffsetIncrement, PageIncrement 1806 ] = Field( 1807 ..., 1808 description="Strategy defining how records are paginated.", 1809 title="Pagination Strategy", 1810 ) 1811 page_size_option: Optional[RequestOption] = None 1812 page_token_option: Optional[Union[RequestOption, RequestPath]] = None 1813 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters") 1814 1815 1816class SessionTokenRequestApiKeyAuthenticator(BaseModel): 1817 type: Literal["ApiKey"] 1818 inject_into: RequestOption = Field( 1819 ..., 1820 description="Configure how the API Key will be sent in requests to the source API.", 1821 examples=[ 1822 {"inject_into": "header", "field_name": "Authorization"}, 1823 {"inject_into": "request_parameter", "field_name": "authKey"}, 1824 ], 1825 title="Inject API Key Into Outgoing HTTP Request", 1826 ) 1827 1828 1829class ListPartitionRouter(BaseModel): 1830 type: Literal["ListPartitionRouter"] 1831 cursor_field: str = Field( 1832 ..., 1833 description='While iterating over list values, the name of field used to reference a list value. The partition value can be accessed with string interpolation. e.g. "{{ stream_partition[\'my_key\'] }}" where "my_key" is the value of the cursor_field.', 1834 examples=["section", "{{ config['section_key'] }}"], 1835 title="Current Partition Value Identifier", 1836 ) 1837 values: Union[str, List[str]] = Field( 1838 ..., 1839 description="The list of attributes being iterated over and used as input for the requests made to the source API.", 1840 examples=[["section_a", "section_b", "section_c"], "{{ config['sections'] }}"], 1841 title="Partition Values", 1842 ) 1843 request_option: Optional[RequestOption] = Field( 1844 None, 1845 description="A request option describing where the list value should be injected into and under what field name if applicable.", 1846 title="Inject Partition Value Into Outgoing HTTP Request", 1847 ) 1848 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters") 1849 1850 1851class RecordSelector(BaseModel): 1852 type: Literal["RecordSelector"] 1853 extractor: Union[CustomRecordExtractor, DpathExtractor] 1854 record_filter: Optional[Union[CustomRecordFilter, RecordFilter]] = Field( 1855 None, 1856 description="Responsible for filtering records to be emitted by the Source.", 1857 title="Record Filter", 1858 ) 1859 schema_normalization: Optional[Union[SchemaNormalization, CustomSchemaNormalization]] = Field( 1860 SchemaNormalization.None_, 1861 description="Responsible for normalization according to the schema.", 1862 title="Schema Normalization", 1863 ) 1864 transform_before_filtering: Optional[bool] = Field( 1865 False, 1866 description="If true, transformation will be applied before record filtering.", 1867 ) 1868 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters") 1869 1870 1871class GzipDecoder(BaseModel): 1872 type: Literal["GzipDecoder"] 1873 decoder: Union[CsvDecoder, GzipDecoder, JsonDecoder, JsonlDecoder] 1874 1875 1876class Spec(BaseModel): 1877 type: Literal["Spec"] 1878 connection_specification: Dict[str, Any] = Field( 1879 ..., 1880 description="A connection specification describing how a the connector can be configured.", 1881 title="Connection Specification", 1882 ) 1883 documentation_url: Optional[str] = Field( 1884 None, 1885 description="URL of the connector's documentation page.", 1886 examples=["https://docs.airbyte.com/integrations/sources/dremio"], 1887 title="Documentation URL", 1888 ) 1889 advanced_auth: Optional[AuthFlow] = Field( 1890 None, 1891 description="Advanced specification for configuring the authentication flow.", 1892 title="Advanced Auth", 1893 ) 1894 1895 1896class CompositeErrorHandler(BaseModel): 1897 type: Literal["CompositeErrorHandler"] 1898 error_handlers: List[Union[CompositeErrorHandler, DefaultErrorHandler]] = Field( 1899 ..., 1900 description="List of error handlers to iterate on to determine how to handle a failed response.", 1901 title="Error Handlers", 1902 ) 1903 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters") 1904 1905 1906class HTTPAPIBudget(BaseModel): 1907 class Config: 1908 extra = Extra.allow 1909 1910 type: Literal["HTTPAPIBudget"] 1911 policies: List[ 1912 Union[ 1913 FixedWindowCallRatePolicy, 1914 MovingWindowCallRatePolicy, 1915 UnlimitedCallRatePolicy, 1916 ] 1917 ] = Field( 1918 ..., 1919 description="List of call rate policies that define how many calls are allowed.", 1920 title="Policies", 1921 ) 1922 ratelimit_reset_header: Optional[str] = Field( 1923 "ratelimit-reset", 1924 description="The HTTP response header name that indicates when the rate limit resets.", 1925 title="Rate Limit Reset Header", 1926 ) 1927 ratelimit_remaining_header: Optional[str] = Field( 1928 "ratelimit-remaining", 1929 description="The HTTP response header name that indicates the number of remaining allowed calls.", 1930 title="Rate Limit Remaining Header", 1931 ) 1932 status_codes_for_ratelimit_hit: Optional[List[int]] = Field( 1933 [429], 1934 description="List of HTTP status codes that indicate a rate limit has been hit.", 1935 title="Status Codes for Rate Limit Hit", 1936 ) 1937 1938 1939class ZipfileDecoder(BaseModel): 1940 class Config: 1941 extra = Extra.allow 1942 1943 type: Literal["ZipfileDecoder"] 1944 decoder: Union[CsvDecoder, GzipDecoder, JsonDecoder, JsonlDecoder] = Field( 1945 ..., 1946 description="Parser to parse the decompressed data from the zipfile(s).", 1947 title="Parser", 1948 ) 1949 1950 1951class DeclarativeSource1(BaseModel): 1952 class Config: 1953 extra = Extra.forbid 1954 1955 type: Literal["DeclarativeSource"] 1956 check: Union[CheckStream, CheckDynamicStream] 1957 streams: List[Union[DeclarativeStream, StateDelegatingStream]] 1958 dynamic_streams: Optional[List[DynamicDeclarativeStream]] = None 1959 version: str = Field( 1960 ..., 1961 description="The version of the Airbyte CDK used to build and test the source.", 1962 ) 1963 schemas: Optional[Schemas] = None 1964 definitions: Optional[Dict[str, Any]] = None 1965 spec: Optional[Spec] = None 1966 concurrency_level: Optional[ConcurrencyLevel] = None 1967 api_budget: Optional[HTTPAPIBudget] = None 1968 max_concurrent_async_job_count: Optional[Union[int, str]] = Field( 1969 None, 1970 description="Maximum number of concurrent asynchronous jobs to run. This property is only relevant for sources/streams that support asynchronous job execution through the AsyncRetriever (e.g. a report-based stream that initiates a job, polls the job status, and then fetches the job results). This is often set by the API's maximum number of concurrent jobs on the account level. Refer to the API's documentation for this information.", 1971 examples=[3, "{{ config['max_concurrent_async_job_count'] }}"], 1972 title="Maximum Concurrent Asynchronous Jobs", 1973 ) 1974 metadata: Optional[Dict[str, Any]] = Field( 1975 None, 1976 description="For internal Airbyte use only - DO NOT modify manually. Used by consumers of declarative manifests for storing related metadata.", 1977 ) 1978 description: Optional[str] = Field( 1979 None, 1980 description="A description of the connector. It will be presented on the Source documentation page.", 1981 ) 1982 1983 1984class DeclarativeSource2(BaseModel): 1985 class Config: 1986 extra = Extra.forbid 1987 1988 type: Literal["DeclarativeSource"] 1989 check: Union[CheckStream, CheckDynamicStream] 1990 streams: Optional[List[Union[DeclarativeStream, StateDelegatingStream]]] = None 1991 dynamic_streams: List[DynamicDeclarativeStream] 1992 version: str = Field( 1993 ..., 1994 description="The version of the Airbyte CDK used to build and test the source.", 1995 ) 1996 schemas: Optional[Schemas] = None 1997 definitions: Optional[Dict[str, Any]] = None 1998 spec: Optional[Spec] = None 1999 concurrency_level: Optional[ConcurrencyLevel] = None 2000 api_budget: Optional[HTTPAPIBudget] = None 2001 max_concurrent_async_job_count: Optional[Union[int, str]] = Field( 2002 None, 2003 description="Maximum number of concurrent asynchronous jobs to run. This property is only relevant for sources/streams that support asynchronous job execution through the AsyncRetriever (e.g. a report-based stream that initiates a job, polls the job status, and then fetches the job results). This is often set by the API's maximum number of concurrent jobs on the account level. Refer to the API's documentation for this information.", 2004 examples=[3, "{{ config['max_concurrent_async_job_count'] }}"], 2005 title="Maximum Concurrent Asynchronous Jobs", 2006 ) 2007 metadata: Optional[Dict[str, Any]] = Field( 2008 None, 2009 description="For internal Airbyte use only - DO NOT modify manually. Used by consumers of declarative manifests for storing related metadata.", 2010 ) 2011 description: Optional[str] = Field( 2012 None, 2013 description="A description of the connector. It will be presented on the Source documentation page.", 2014 ) 2015 2016 2017class DeclarativeSource(BaseModel): 2018 class Config: 2019 extra = Extra.forbid 2020 2021 __root__: Union[DeclarativeSource1, DeclarativeSource2] = Field( 2022 ..., 2023 description="An API source that extracts data according to its declarative components.", 2024 title="DeclarativeSource", 2025 ) 2026 2027 2028class SelectiveAuthenticator(BaseModel): 2029 class Config: 2030 extra = Extra.allow 2031 2032 type: Literal["SelectiveAuthenticator"] 2033 authenticator_selection_path: List[str] = Field( 2034 ..., 2035 description="Path of the field in config with selected authenticator name", 2036 examples=[["auth"], ["auth", "type"]], 2037 title="Authenticator Selection Path", 2038 ) 2039 authenticators: Dict[ 2040 str, 2041 Union[ 2042 ApiKeyAuthenticator, 2043 BasicHttpAuthenticator, 2044 BearerAuthenticator, 2045 CustomAuthenticator, 2046 OAuthAuthenticator, 2047 JwtAuthenticator, 2048 NoAuth, 2049 SessionTokenAuthenticator, 2050 LegacySessionTokenAuthenticator, 2051 ], 2052 ] = Field( 2053 ..., 2054 description="Authenticators to select from.", 2055 examples=[ 2056 { 2057 "authenticators": { 2058 "token": "#/definitions/ApiKeyAuthenticator", 2059 "oauth": "#/definitions/OAuthAuthenticator", 2060 "jwt": "#/definitions/JwtAuthenticator", 2061 } 2062 } 2063 ], 2064 title="Authenticators", 2065 ) 2066 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters") 2067 2068 2069class DeclarativeStream(BaseModel): 2070 class Config: 2071 extra = Extra.allow 2072 2073 type: Literal["DeclarativeStream"] 2074 retriever: Union[AsyncRetriever, CustomRetriever, SimpleRetriever] = Field( 2075 ..., 2076 description="Component used to coordinate how records are extracted across stream slices and request pages.", 2077 title="Retriever", 2078 ) 2079 incremental_sync: Optional[ 2080 Union[CustomIncrementalSync, DatetimeBasedCursor, IncrementingCountCursor] 2081 ] = Field( 2082 None, 2083 description="Component used to fetch data incrementally based on a time field in the data.", 2084 title="Incremental Sync", 2085 ) 2086 name: Optional[str] = Field("", description="The stream name.", example=["Users"], title="Name") 2087 primary_key: Optional[PrimaryKey] = Field( 2088 "", description="The primary key of the stream.", title="Primary Key" 2089 ) 2090 schema_loader: Optional[ 2091 Union[ 2092 DynamicSchemaLoader, 2093 InlineSchemaLoader, 2094 JsonFileSchemaLoader, 2095 CustomSchemaLoader, 2096 ] 2097 ] = Field( 2098 None, 2099 description="Component used to retrieve the schema for the current stream.", 2100 title="Schema Loader", 2101 ) 2102 transformations: Optional[ 2103 List[ 2104 Union[ 2105 AddFields, 2106 CustomTransformation, 2107 RemoveFields, 2108 KeysToLower, 2109 KeysToSnakeCase, 2110 FlattenFields, 2111 DpathFlattenFields, 2112 KeysReplace, 2113 ] 2114 ] 2115 ] = Field( 2116 None, 2117 description="A list of transformations to be applied to each output record.", 2118 title="Transformations", 2119 ) 2120 state_migrations: Optional[ 2121 List[Union[LegacyToPerPartitionStateMigration, CustomStateMigration]] 2122 ] = Field( 2123 [], 2124 description="Array of state migrations to be applied on the input state", 2125 title="State Migrations", 2126 ) 2127 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters") 2128 2129 2130class SessionTokenAuthenticator(BaseModel): 2131 type: Literal["SessionTokenAuthenticator"] 2132 login_requester: HttpRequester = Field( 2133 ..., 2134 description="Description of the request to perform to obtain a session token to perform data requests. The response body is expected to be a JSON object with a session token property.", 2135 examples=[ 2136 { 2137 "type": "HttpRequester", 2138 "url_base": "https://my_api.com", 2139 "path": "/login", 2140 "authenticator": { 2141 "type": "BasicHttpAuthenticator", 2142 "username": "{{ config.username }}", 2143 "password": "{{ config.password }}", 2144 }, 2145 } 2146 ], 2147 title="Login Requester", 2148 ) 2149 session_token_path: List[str] = Field( 2150 ..., 2151 description="The path in the response body returned from the login requester to the session token.", 2152 examples=[["access_token"], ["result", "token"]], 2153 title="Session Token Path", 2154 ) 2155 expiration_duration: Optional[str] = Field( 2156 None, 2157 description="The duration in ISO 8601 duration notation after which the session token expires, starting from the time it was obtained. Omitting it will result in the session token being refreshed for every request.", 2158 examples=["PT1H", "P1D"], 2159 title="Expiration Duration", 2160 ) 2161 request_authentication: Union[ 2162 SessionTokenRequestApiKeyAuthenticator, SessionTokenRequestBearerAuthenticator 2163 ] = Field( 2164 ..., 2165 description="Authentication method to use for requests sent to the API, specifying how to inject the session token.", 2166 title="Data Request Authentication", 2167 ) 2168 decoder: Optional[Union[JsonDecoder, XmlDecoder]] = Field( 2169 None, description="Component used to decode the response.", title="Decoder" 2170 ) 2171 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters") 2172 2173 2174class HttpRequester(BaseModel): 2175 type: Literal["HttpRequester"] 2176 url_base: str = Field( 2177 ..., 2178 description="Base URL of the API source. Do not put sensitive information (e.g. API tokens) into this field - Use the Authentication component for this.", 2179 examples=[ 2180 "https://connect.squareup.com/v2", 2181 "{{ config['base_url'] or 'https://app.posthog.com'}}/api", 2182 "https://connect.squareup.com/v2/quotes/{{ stream_partition['id'] }}/quote_line_groups", 2183 "https://example.com/api/v1/resource/{{ next_page_token['id'] }}", 2184 ], 2185 title="API Base URL", 2186 ) 2187 path: Optional[str] = Field( 2188 None, 2189 description="Path the specific API endpoint that this stream represents. Do not put sensitive information (e.g. API tokens) into this field - Use the Authentication component for this.", 2190 examples=[ 2191 "/products", 2192 "/quotes/{{ stream_partition['id'] }}/quote_line_groups", 2193 "/trades/{{ config['symbol_id'] }}/history", 2194 ], 2195 title="URL Path", 2196 ) 2197 authenticator: Optional[ 2198 Union[ 2199 ApiKeyAuthenticator, 2200 BasicHttpAuthenticator, 2201 BearerAuthenticator, 2202 CustomAuthenticator, 2203 OAuthAuthenticator, 2204 JwtAuthenticator, 2205 NoAuth, 2206 SessionTokenAuthenticator, 2207 LegacySessionTokenAuthenticator, 2208 SelectiveAuthenticator, 2209 ] 2210 ] = Field( 2211 None, 2212 description="Authentication method to use for requests sent to the API.", 2213 title="Authenticator", 2214 ) 2215 error_handler: Optional[ 2216 Union[DefaultErrorHandler, CustomErrorHandler, CompositeErrorHandler] 2217 ] = Field( 2218 None, 2219 description="Error handler component that defines how to handle errors.", 2220 title="Error Handler", 2221 ) 2222 http_method: Optional[HttpMethod] = Field( 2223 HttpMethod.GET, 2224 description="The HTTP method used to fetch data from the source (can be GET or POST).", 2225 examples=["GET", "POST"], 2226 title="HTTP Method", 2227 ) 2228 request_body_data: Optional[Union[str, Dict[str, str]]] = Field( 2229 None, 2230 description="Specifies how to populate the body of the request with a non-JSON payload. Plain text will be sent as is, whereas objects will be converted to a urlencoded form.", 2231 examples=[ 2232 '[{"clause": {"type": "timestamp", "operator": 10, "parameters":\n [{"value": {{ stream_interval[\'start_time\'] | int * 1000 }} }]\n }, "orderBy": 1, "columnName": "Timestamp"}]/\n' 2233 ], 2234 title="Request Body Payload (Non-JSON)", 2235 ) 2236 request_body_json: Optional[Union[str, Dict[str, Any]]] = Field( 2237 None, 2238 description="Specifies how to populate the body of the request with a JSON payload. Can contain nested objects.", 2239 examples=[ 2240 {"sort_order": "ASC", "sort_field": "CREATED_AT"}, 2241 {"key": "{{ config['value'] }}"}, 2242 {"sort": {"field": "updated_at", "order": "ascending"}}, 2243 ], 2244 title="Request Body JSON Payload", 2245 ) 2246 request_headers: Optional[Union[str, Dict[str, str]]] = Field( 2247 None, 2248 description="Return any non-auth headers. Authentication headers will overwrite any overlapping headers returned from this method.", 2249 examples=[{"Output-Format": "JSON"}, {"Version": "{{ config['version'] }}"}], 2250 title="Request Headers", 2251 ) 2252 request_parameters: Optional[Union[str, Dict[str, Union[str, Any]]]] = Field( 2253 None, 2254 description="Specifies the query parameters that should be set on an outgoing HTTP request given the inputs.", 2255 examples=[ 2256 {"unit": "day"}, 2257 { 2258 "query": 'last_event_time BETWEEN TIMESTAMP "{{ stream_interval.start_time }}" AND TIMESTAMP "{{ stream_interval.end_time }}"' 2259 }, 2260 {"searchIn": "{{ ','.join(config.get('search_in', [])) }}"}, 2261 {"sort_by[asc]": "updated_at"}, 2262 ], 2263 title="Query Parameters", 2264 ) 2265 use_cache: Optional[bool] = Field( 2266 False, 2267 description="Enables stream requests caching. This field is automatically set by the CDK.", 2268 title="Use Cache", 2269 ) 2270 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters") 2271 2272 2273class DynamicSchemaLoader(BaseModel): 2274 type: Literal["DynamicSchemaLoader"] 2275 retriever: Union[AsyncRetriever, CustomRetriever, SimpleRetriever] = Field( 2276 ..., 2277 description="Component used to coordinate how records are extracted across stream slices and request pages.", 2278 title="Retriever", 2279 ) 2280 schema_transformations: Optional[ 2281 List[ 2282 Union[ 2283 AddFields, 2284 CustomTransformation, 2285 RemoveFields, 2286 KeysToLower, 2287 KeysToSnakeCase, 2288 FlattenFields, 2289 DpathFlattenFields, 2290 KeysReplace, 2291 ] 2292 ] 2293 ] = Field( 2294 None, 2295 description="A list of transformations to be applied to the schema.", 2296 title="Schema Transformations", 2297 ) 2298 schema_type_identifier: SchemaTypeIdentifier 2299 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters") 2300 2301 2302class ParentStreamConfig(BaseModel): 2303 type: Literal["ParentStreamConfig"] 2304 lazy_read_pointer: Optional[List[str]] = Field( 2305 [], 2306 description="If set, this will enable lazy reading, using the initial read of parent records to extract child records.", 2307 title="Lazy Read Pointer", 2308 ) 2309 parent_key: str = Field( 2310 ..., 2311 description="The primary key of records from the parent stream that will be used during the retrieval of records for the current substream. This parent identifier field is typically a characteristic of the child records being extracted from the source API.", 2312 examples=["id", "{{ config['parent_record_id'] }}"], 2313 title="Parent Key", 2314 ) 2315 stream: Union[DeclarativeStream, StateDelegatingStream] = Field( 2316 ..., description="Reference to the parent stream.", title="Parent Stream" 2317 ) 2318 partition_field: str = Field( 2319 ..., 2320 description="While iterating over parent records during a sync, the parent_key value can be referenced by using this field.", 2321 examples=["parent_id", "{{ config['parent_partition_field'] }}"], 2322 title="Current Parent Key Value Identifier", 2323 ) 2324 request_option: Optional[RequestOption] = Field( 2325 None, 2326 description="A request option describing where the parent key value should be injected into and under what field name if applicable.", 2327 title="Request Option", 2328 ) 2329 incremental_dependency: Optional[bool] = Field( 2330 False, 2331 description="Indicates whether the parent stream should be read incrementally based on updates in the child stream.", 2332 title="Incremental Dependency", 2333 ) 2334 extra_fields: Optional[List[List[str]]] = Field( 2335 None, 2336 description="Array of field paths to include as additional fields in the stream slice. Each path is an array of strings representing keys to access fields in the respective parent record. Accessible via `stream_slice.extra_fields`. Missing fields are set to `None`.", 2337 title="Extra Fields", 2338 ) 2339 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters") 2340 2341 2342class PropertiesFromEndpoint(BaseModel): 2343 type: Literal["PropertiesFromEndpoint"] 2344 property_field_path: List[str] = Field( 2345 ..., 2346 description="Describes the path to the field that should be extracted", 2347 examples=[["name"]], 2348 ) 2349 retriever: Union[CustomRetriever, SimpleRetriever] = Field( 2350 ..., 2351 description="Requester component that describes how to fetch the properties to query from a remote API endpoint.", 2352 ) 2353 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters") 2354 2355 2356class QueryProperties(BaseModel): 2357 type: Literal["QueryProperties"] 2358 property_list: Union[List[str], PropertiesFromEndpoint] = Field( 2359 ..., 2360 description="The set of properties that will be queried for in the outbound request. This can either be statically defined or dynamic based on an API endpoint", 2361 title="Property List", 2362 ) 2363 always_include_properties: Optional[List[str]] = Field( 2364 None, 2365 description="The list of properties that should be included in every set of properties when multiple chunks of properties are being requested.", 2366 title="Always Include Properties", 2367 ) 2368 property_chunking: Optional[PropertyChunking] = Field( 2369 None, 2370 description="Defines how query properties will be grouped into smaller sets for APIs with limitations on the number of properties fetched per API request.", 2371 title="Property Chunking", 2372 ) 2373 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters") 2374 2375 2376class StateDelegatingStream(BaseModel): 2377 type: Literal["StateDelegatingStream"] 2378 name: str = Field(..., description="The stream name.", example=["Users"], title="Name") 2379 full_refresh_stream: DeclarativeStream = Field( 2380 ..., 2381 description="Component used to coordinate how records are extracted across stream slices and request pages when the state is empty or not provided.", 2382 title="Retriever", 2383 ) 2384 incremental_stream: DeclarativeStream = Field( 2385 ..., 2386 description="Component used to coordinate how records are extracted across stream slices and request pages when the state provided.", 2387 title="Retriever", 2388 ) 2389 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters") 2390 2391 2392class SimpleRetriever(BaseModel): 2393 type: Literal["SimpleRetriever"] 2394 record_selector: RecordSelector = Field( 2395 ..., 2396 description="Component that describes how to extract records from a HTTP response.", 2397 ) 2398 requester: Union[CustomRequester, HttpRequester] = Field( 2399 ..., 2400 description="Requester component that describes how to prepare HTTP requests to send to the source API.", 2401 ) 2402 paginator: Optional[Union[DefaultPaginator, NoPagination]] = Field( 2403 None, 2404 description="Paginator component that describes how to navigate through the API's pages.", 2405 ) 2406 ignore_stream_slicer_parameters_on_paginated_requests: Optional[bool] = Field( 2407 False, 2408 description="If true, the partition router and incremental request options will be ignored when paginating requests. Request options set directly on the requester will not be ignored.", 2409 ) 2410 partition_router: Optional[ 2411 Union[ 2412 CustomPartitionRouter, 2413 ListPartitionRouter, 2414 SubstreamPartitionRouter, 2415 GroupingPartitionRouter, 2416 List[ 2417 Union[ 2418 CustomPartitionRouter, 2419 ListPartitionRouter, 2420 SubstreamPartitionRouter, 2421 GroupingPartitionRouter, 2422 ] 2423 ], 2424 ] 2425 ] = Field( 2426 [], 2427 description="PartitionRouter component that describes how to partition the stream, enabling incremental syncs and checkpointing.", 2428 title="Partition Router", 2429 ) 2430 decoder: Optional[ 2431 Union[ 2432 CustomDecoder, 2433 CsvDecoder, 2434 GzipDecoder, 2435 JsonDecoder, 2436 JsonlDecoder, 2437 IterableDecoder, 2438 XmlDecoder, 2439 ZipfileDecoder, 2440 ] 2441 ] = Field( 2442 None, 2443 description="Component decoding the response so records can be extracted.", 2444 title="Decoder", 2445 ) 2446 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters") 2447 2448 2449class AsyncRetriever(BaseModel): 2450 type: Literal["AsyncRetriever"] 2451 record_selector: RecordSelector = Field( 2452 ..., 2453 description="Component that describes how to extract records from a HTTP response.", 2454 ) 2455 status_mapping: AsyncJobStatusMap = Field( 2456 ..., description="Async Job Status to Airbyte CDK Async Job Status mapping." 2457 ) 2458 status_extractor: Union[CustomRecordExtractor, DpathExtractor] = Field( 2459 ..., description="Responsible for fetching the actual status of the async job." 2460 ) 2461 download_target_extractor: Union[CustomRecordExtractor, DpathExtractor] = Field( 2462 ..., 2463 description="Responsible for fetching the final result `urls` provided by the completed / finished / ready async job.", 2464 ) 2465 download_extractor: Optional[ 2466 Union[CustomRecordExtractor, DpathExtractor, ResponseToFileExtractor] 2467 ] = Field(None, description="Responsible for fetching the records from provided urls.") 2468 creation_requester: Union[CustomRequester, HttpRequester] = Field( 2469 ..., 2470 description="Requester component that describes how to prepare HTTP requests to send to the source API to create the async server-side job.", 2471 ) 2472 polling_requester: Union[CustomRequester, HttpRequester] = Field( 2473 ..., 2474 description="Requester component that describes how to prepare HTTP requests to send to the source API to fetch the status of the running async job.", 2475 ) 2476 polling_job_timeout: Optional[Union[int, str]] = Field( 2477 None, 2478 description="The time in minutes after which the single Async Job should be considered as Timed Out.", 2479 ) 2480 download_target_requester: Optional[Union[CustomRequester, HttpRequester]] = Field( 2481 None, 2482 description="Requester component that describes how to prepare HTTP requests to send to the source API to extract the url from polling response by the completed async job.", 2483 ) 2484 download_requester: Union[CustomRequester, HttpRequester] = Field( 2485 ..., 2486 description="Requester component that describes how to prepare HTTP requests to send to the source API to download the data provided by the completed async job.", 2487 ) 2488 download_paginator: Optional[Union[DefaultPaginator, NoPagination]] = Field( 2489 None, 2490 description="Paginator component that describes how to navigate through the API's pages during download.", 2491 ) 2492 abort_requester: Optional[Union[CustomRequester, HttpRequester]] = Field( 2493 None, 2494 description="Requester component that describes how to prepare HTTP requests to send to the source API to abort a job once it is timed out from the source's perspective.", 2495 ) 2496 delete_requester: Optional[Union[CustomRequester, HttpRequester]] = Field( 2497 None, 2498 description="Requester component that describes how to prepare HTTP requests to send to the source API to delete a job once the records are extracted.", 2499 ) 2500 partition_router: Optional[ 2501 Union[ 2502 CustomPartitionRouter, 2503 ListPartitionRouter, 2504 SubstreamPartitionRouter, 2505 GroupingPartitionRouter, 2506 List[ 2507 Union[ 2508 CustomPartitionRouter, 2509 ListPartitionRouter, 2510 SubstreamPartitionRouter, 2511 GroupingPartitionRouter, 2512 ] 2513 ], 2514 ] 2515 ] = Field( 2516 [], 2517 description="PartitionRouter component that describes how to partition the stream, enabling incremental syncs and checkpointing.", 2518 title="Partition Router", 2519 ) 2520 decoder: Optional[ 2521 Union[ 2522 CustomDecoder, 2523 CsvDecoder, 2524 GzipDecoder, 2525 JsonDecoder, 2526 JsonlDecoder, 2527 IterableDecoder, 2528 XmlDecoder, 2529 ZipfileDecoder, 2530 ] 2531 ] = Field( 2532 None, 2533 description="Component decoding the response so records can be extracted.", 2534 title="Decoder", 2535 ) 2536 download_decoder: Optional[ 2537 Union[ 2538 CustomDecoder, 2539 CsvDecoder, 2540 GzipDecoder, 2541 JsonDecoder, 2542 JsonlDecoder, 2543 IterableDecoder, 2544 XmlDecoder, 2545 ZipfileDecoder, 2546 ] 2547 ] = Field( 2548 None, 2549 description="Component decoding the download response so records can be extracted.", 2550 title="Download Decoder", 2551 ) 2552 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters") 2553 2554 2555class SubstreamPartitionRouter(BaseModel): 2556 type: Literal["SubstreamPartitionRouter"] 2557 parent_stream_configs: List[ParentStreamConfig] = Field( 2558 ..., 2559 description="Specifies which parent streams are being iterated over and how parent records should be used to partition the child stream data set.", 2560 title="Parent Stream Configs", 2561 ) 2562 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters") 2563 2564 2565class GroupingPartitionRouter(BaseModel): 2566 type: Literal["GroupingPartitionRouter"] 2567 group_size: int = Field( 2568 ..., 2569 description="The number of partitions to include in each group. This determines how many partition values are batched together in a single slice.", 2570 examples=[10, 50], 2571 title="Group Size", 2572 ) 2573 underlying_partition_router: Union[ 2574 CustomPartitionRouter, ListPartitionRouter, SubstreamPartitionRouter 2575 ] = Field( 2576 ..., 2577 description="The partition router whose output will be grouped. This can be any valid partition router component.", 2578 title="Underlying Partition Router", 2579 ) 2580 deduplicate: Optional[bool] = Field( 2581 True, 2582 description="If true, ensures that partitions are unique within each group by removing duplicates based on the partition key.", 2583 title="Deduplicate Partitions", 2584 ) 2585 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters") 2586 2587 2588class HttpComponentsResolver(BaseModel): 2589 type: Literal["HttpComponentsResolver"] 2590 retriever: Union[AsyncRetriever, CustomRetriever, SimpleRetriever] = Field( 2591 ..., 2592 description="Component used to coordinate how records are extracted across stream slices and request pages.", 2593 title="Retriever", 2594 ) 2595 components_mapping: List[ComponentMappingDefinition] 2596 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters") 2597 2598 2599class DynamicDeclarativeStream(BaseModel): 2600 type: Literal["DynamicDeclarativeStream"] 2601 name: Optional[str] = Field( 2602 "", description="The dynamic stream name.", example=["Tables"], title="Name" 2603 ) 2604 stream_template: DeclarativeStream = Field( 2605 ..., description="Reference to the stream template.", title="Stream Template" 2606 ) 2607 components_resolver: Union[HttpComponentsResolver, ConfigComponentsResolver] = Field( 2608 ..., 2609 description="Component resolve and populates stream templates with components values.", 2610 title="Components Resolver", 2611 ) 2612 2613 2614ComplexFieldType.update_forward_refs() 2615GzipDecoder.update_forward_refs() 2616CompositeErrorHandler.update_forward_refs() 2617DeclarativeSource1.update_forward_refs() 2618DeclarativeSource2.update_forward_refs() 2619SelectiveAuthenticator.update_forward_refs() 2620DeclarativeStream.update_forward_refs() 2621SessionTokenAuthenticator.update_forward_refs() 2622DynamicSchemaLoader.update_forward_refs() 2623ParentStreamConfig.update_forward_refs() 2624PropertiesFromEndpoint.update_forward_refs() 2625SimpleRetriever.update_forward_refs() 2626AsyncRetriever.update_forward_refs()
class
AuthFlowType(enum.Enum):
An enumeration.
oauth2_0 =
<AuthFlowType.oauth2_0: 'oauth2.0'>
oauth1_0 =
<AuthFlowType.oauth1_0: 'oauth1.0'>
class
BasicHttpAuthenticator(pydantic.v1.main.BaseModel):
20class BasicHttpAuthenticator(BaseModel): 21 type: Literal["BasicHttpAuthenticator"] 22 username: str = Field( 23 ..., 24 description="The username that will be combined with the password, base64 encoded and used to make requests. Fill it in the user inputs.", 25 examples=["{{ config['username'] }}", "{{ config['api_key'] }}"], 26 title="Username", 27 ) 28 password: Optional[str] = Field( 29 "", 30 description="The password that will be combined with the username, base64 encoded and used to make requests. Fill it in the user inputs.", 31 examples=["{{ config['password'] }}", ""], 32 title="Password", 33 ) 34 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters")
class
BearerAuthenticator(pydantic.v1.main.BaseModel):
37class BearerAuthenticator(BaseModel): 38 type: Literal["BearerAuthenticator"] 39 api_token: str = Field( 40 ..., 41 description="Token to inject as request header for authenticating with the API.", 42 examples=["{{ config['api_key'] }}", "{{ config['token'] }}"], 43 title="Bearer Token", 44 ) 45 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters")
class
DynamicStreamCheckConfig(pydantic.v1.main.BaseModel):
48class DynamicStreamCheckConfig(BaseModel): 49 type: Literal["DynamicStreamCheckConfig"] 50 dynamic_stream_name: str = Field( 51 ..., description="The dynamic stream name.", title="Dynamic Stream Name" 52 ) 53 stream_count: Optional[int] = Field( 54 0, 55 description="The number of streams to attempt reading from during a check operation. If `stream_count` exceeds the total number of available streams, the minimum of the two values will be used.", 56 title="Stream Count", 57 )
class
CheckDynamicStream(pydantic.v1.main.BaseModel):
60class CheckDynamicStream(BaseModel): 61 type: Literal["CheckDynamicStream"] 62 stream_count: int = Field( 63 ..., 64 description="Numbers of the streams to try reading from when running a check operation.", 65 title="Stream Count", 66 ) 67 use_check_availability: Optional[bool] = Field( 68 True, 69 description="Enables stream check availability. This field is automatically set by the CDK.", 70 title="Use Check Availability", 71 )
class
ConcurrencyLevel(pydantic.v1.main.BaseModel):
74class ConcurrencyLevel(BaseModel): 75 type: Optional[Literal["ConcurrencyLevel"]] = None 76 default_concurrency: Union[int, str] = Field( 77 ..., 78 description="The amount of concurrency that will applied during a sync. This value can be hardcoded or user-defined in the config if different users have varying volume thresholds in the target API.", 79 examples=[10, "{{ config['num_workers'] or 10 }}"], 80 title="Default Concurrency", 81 ) 82 max_concurrency: Optional[int] = Field( 83 None, 84 description="The maximum level of concurrency that will be used during a sync. This becomes a required field when the default_concurrency derives from the config, because it serves as a safeguard against a user-defined threshold that is too high.", 85 examples=[20, 100], 86 title="Max Concurrency", 87 ) 88 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters")
class
ConstantBackoffStrategy(pydantic.v1.main.BaseModel):
91class ConstantBackoffStrategy(BaseModel): 92 type: Literal["ConstantBackoffStrategy"] 93 backoff_time_in_seconds: Union[float, str] = Field( 94 ..., 95 description="Backoff time in seconds.", 96 examples=[30, 30.5, "{{ config['backoff_time'] }}"], 97 title="Backoff Time", 98 ) 99 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters")
class
CursorPagination(pydantic.v1.main.BaseModel):
102class CursorPagination(BaseModel): 103 type: Literal["CursorPagination"] 104 cursor_value: str = Field( 105 ..., 106 description="Value of the cursor defining the next page to fetch.", 107 examples=[ 108 "{{ headers.link.next.cursor }}", 109 "{{ last_record['key'] }}", 110 "{{ response['nextPage'] }}", 111 ], 112 title="Cursor Value", 113 ) 114 page_size: Optional[int] = Field( 115 None, 116 description="The number of records to include in each pages.", 117 examples=[100], 118 title="Page Size", 119 ) 120 stop_condition: Optional[str] = Field( 121 None, 122 description="Template string evaluating when to stop paginating.", 123 examples=[ 124 "{{ response.data.has_more is false }}", 125 "{{ 'next' not in headers['link'] }}", 126 ], 127 title="Stop Condition", 128 ) 129 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters")
class
CustomAuthenticator(pydantic.v1.main.BaseModel):
132class CustomAuthenticator(BaseModel): 133 class Config: 134 extra = Extra.allow 135 136 type: Literal["CustomAuthenticator"] 137 class_name: str = Field( 138 ..., 139 description="Fully-qualified name of the class that will be implementing the custom authentication strategy. Has to be a sub class of DeclarativeAuthenticator. The format is `source_<name>.<package>.<class_name>`.", 140 examples=["source_railz.components.ShortLivedTokenAuthenticator"], 141 title="Class Name", 142 ) 143 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters")
class
CustomAuthenticator.Config:
class
CustomBackoffStrategy(pydantic.v1.main.BaseModel):
146class CustomBackoffStrategy(BaseModel): 147 class Config: 148 extra = Extra.allow 149 150 type: Literal["CustomBackoffStrategy"] 151 class_name: str = Field( 152 ..., 153 description="Fully-qualified name of the class that will be implementing the custom backoff strategy. The format is `source_<name>.<package>.<class_name>`.", 154 examples=["source_railz.components.MyCustomBackoffStrategy"], 155 title="Class Name", 156 ) 157 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters")
class
CustomBackoffStrategy.Config:
class
CustomErrorHandler(pydantic.v1.main.BaseModel):
160class CustomErrorHandler(BaseModel): 161 class Config: 162 extra = Extra.allow 163 164 type: Literal["CustomErrorHandler"] 165 class_name: str = Field( 166 ..., 167 description="Fully-qualified name of the class that will be implementing the custom error handler. The format is `source_<name>.<package>.<class_name>`.", 168 examples=["source_railz.components.MyCustomErrorHandler"], 169 title="Class Name", 170 ) 171 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters")
class
CustomErrorHandler.Config:
class
CustomIncrementalSync(pydantic.v1.main.BaseModel):
174class CustomIncrementalSync(BaseModel): 175 class Config: 176 extra = Extra.allow 177 178 type: Literal["CustomIncrementalSync"] 179 class_name: str = Field( 180 ..., 181 description="Fully-qualified name of the class that will be implementing the custom incremental sync. The format is `source_<name>.<package>.<class_name>`.", 182 examples=["source_railz.components.MyCustomIncrementalSync"], 183 title="Class Name", 184 ) 185 cursor_field: str = Field( 186 ..., 187 description="The location of the value on a record that will be used as a bookmark during sync.", 188 ) 189 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters")
class
CustomIncrementalSync.Config:
class
CustomPaginationStrategy(pydantic.v1.main.BaseModel):
192class CustomPaginationStrategy(BaseModel): 193 class Config: 194 extra = Extra.allow 195 196 type: Literal["CustomPaginationStrategy"] 197 class_name: str = Field( 198 ..., 199 description="Fully-qualified name of the class that will be implementing the custom pagination strategy. The format is `source_<name>.<package>.<class_name>`.", 200 examples=["source_railz.components.MyCustomPaginationStrategy"], 201 title="Class Name", 202 ) 203 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters")
class
CustomPaginationStrategy.Config:
class
CustomRecordExtractor(pydantic.v1.main.BaseModel):
206class CustomRecordExtractor(BaseModel): 207 class Config: 208 extra = Extra.allow 209 210 type: Literal["CustomRecordExtractor"] 211 class_name: str = Field( 212 ..., 213 description="Fully-qualified name of the class that will be implementing the custom record extraction strategy. The format is `source_<name>.<package>.<class_name>`.", 214 examples=["source_railz.components.MyCustomRecordExtractor"], 215 title="Class Name", 216 ) 217 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters")
class
CustomRecordExtractor.Config:
class
CustomRecordFilter(pydantic.v1.main.BaseModel):
220class CustomRecordFilter(BaseModel): 221 class Config: 222 extra = Extra.allow 223 224 type: Literal["CustomRecordFilter"] 225 class_name: str = Field( 226 ..., 227 description="Fully-qualified name of the class that will be implementing the custom record filter strategy. The format is `source_<name>.<package>.<class_name>`.", 228 examples=["source_railz.components.MyCustomCustomRecordFilter"], 229 title="Class Name", 230 ) 231 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters")
class
CustomRecordFilter.Config:
class
CustomRequester(pydantic.v1.main.BaseModel):
234class CustomRequester(BaseModel): 235 class Config: 236 extra = Extra.allow 237 238 type: Literal["CustomRequester"] 239 class_name: str = Field( 240 ..., 241 description="Fully-qualified name of the class that will be implementing the custom requester strategy. The format is `source_<name>.<package>.<class_name>`.", 242 examples=["source_railz.components.MyCustomRecordExtractor"], 243 title="Class Name", 244 ) 245 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters")
class
CustomRequester.Config:
class
CustomRetriever(pydantic.v1.main.BaseModel):
248class CustomRetriever(BaseModel): 249 class Config: 250 extra = Extra.allow 251 252 type: Literal["CustomRetriever"] 253 class_name: str = Field( 254 ..., 255 description="Fully-qualified name of the class that will be implementing the custom retriever strategy. The format is `source_<name>.<package>.<class_name>`.", 256 examples=["source_railz.components.MyCustomRetriever"], 257 title="Class Name", 258 ) 259 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters")
class
CustomRetriever.Config:
class
CustomPartitionRouter(pydantic.v1.main.BaseModel):
262class CustomPartitionRouter(BaseModel): 263 class Config: 264 extra = Extra.allow 265 266 type: Literal["CustomPartitionRouter"] 267 class_name: str = Field( 268 ..., 269 description="Fully-qualified name of the class that will be implementing the custom partition router. The format is `source_<name>.<package>.<class_name>`.", 270 examples=["source_railz.components.MyCustomPartitionRouter"], 271 title="Class Name", 272 ) 273 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters")
class
CustomPartitionRouter.Config:
class
CustomSchemaLoader(pydantic.v1.main.BaseModel):
276class CustomSchemaLoader(BaseModel): 277 class Config: 278 extra = Extra.allow 279 280 type: Literal["CustomSchemaLoader"] 281 class_name: str = Field( 282 ..., 283 description="Fully-qualified name of the class that will be implementing the custom schema loader. The format is `source_<name>.<package>.<class_name>`.", 284 examples=["source_railz.components.MyCustomSchemaLoader"], 285 title="Class Name", 286 ) 287 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters")
class
CustomSchemaLoader.Config:
class
CustomSchemaNormalization(pydantic.v1.main.BaseModel):
290class CustomSchemaNormalization(BaseModel): 291 class Config: 292 extra = Extra.allow 293 294 type: Literal["CustomSchemaNormalization"] 295 class_name: str = Field( 296 ..., 297 description="Fully-qualified name of the class that will be implementing the custom normalization. The format is `source_<name>.<package>.<class_name>`.", 298 examples=[ 299 "source_amazon_seller_partner.components.LedgerDetailedViewReportsTypeTransformer" 300 ], 301 title="Class Name", 302 ) 303 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters")
class
CustomSchemaNormalization.Config:
class
CustomStateMigration(pydantic.v1.main.BaseModel):
306class CustomStateMigration(BaseModel): 307 class Config: 308 extra = Extra.allow 309 310 type: Literal["CustomStateMigration"] 311 class_name: str = Field( 312 ..., 313 description="Fully-qualified name of the class that will be implementing the custom state migration. The format is `source_<name>.<package>.<class_name>`.", 314 examples=["source_railz.components.MyCustomStateMigration"], 315 title="Class Name", 316 ) 317 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters")
class
CustomStateMigration.Config:
class
CustomTransformation(pydantic.v1.main.BaseModel):
320class CustomTransformation(BaseModel): 321 class Config: 322 extra = Extra.allow 323 324 type: Literal["CustomTransformation"] 325 class_name: str = Field( 326 ..., 327 description="Fully-qualified name of the class that will be implementing the custom transformation. The format is `source_<name>.<package>.<class_name>`.", 328 examples=["source_railz.components.MyCustomTransformation"], 329 title="Class Name", 330 ) 331 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters")
class
CustomTransformation.Config:
class
LegacyToPerPartitionStateMigration(pydantic.v1.main.BaseModel):
class
LegacyToPerPartitionStateMigration.Config:
class
Clamping(pydantic.v1.main.BaseModel):
class
Algorithm(enum.Enum):
351class Algorithm(Enum): 352 HS256 = "HS256" 353 HS384 = "HS384" 354 HS512 = "HS512" 355 ES256 = "ES256" 356 ES256K = "ES256K" 357 ES384 = "ES384" 358 ES512 = "ES512" 359 RS256 = "RS256" 360 RS384 = "RS384" 361 RS512 = "RS512" 362 PS256 = "PS256" 363 PS384 = "PS384" 364 PS512 = "PS512" 365 EdDSA = "EdDSA"
An enumeration.
HS256 =
<Algorithm.HS256: 'HS256'>
HS384 =
<Algorithm.HS384: 'HS384'>
HS512 =
<Algorithm.HS512: 'HS512'>
ES256 =
<Algorithm.ES256: 'ES256'>
ES256K =
<Algorithm.ES256K: 'ES256K'>
ES384 =
<Algorithm.ES384: 'ES384'>
ES512 =
<Algorithm.ES512: 'ES512'>
RS256 =
<Algorithm.RS256: 'RS256'>
RS384 =
<Algorithm.RS384: 'RS384'>
RS512 =
<Algorithm.RS512: 'RS512'>
PS256 =
<Algorithm.PS256: 'PS256'>
PS384 =
<Algorithm.PS384: 'PS384'>
PS512 =
<Algorithm.PS512: 'PS512'>
EdDSA =
<Algorithm.EdDSA: 'EdDSA'>
class
JwtHeaders(pydantic.v1.main.BaseModel):
368class JwtHeaders(BaseModel): 369 class Config: 370 extra = Extra.forbid 371 372 kid: Optional[str] = Field( 373 None, 374 description="Private key ID for user account.", 375 examples=["{{ config['kid'] }}"], 376 title="Key Identifier", 377 ) 378 typ: Optional[str] = Field( 379 "JWT", 380 description="The media type of the complete JWT.", 381 examples=["JWT"], 382 title="Type", 383 ) 384 cty: Optional[str] = Field( 385 None, 386 description="Content type of JWT header.", 387 examples=["JWT"], 388 title="Content Type", 389 )
class
JwtHeaders.Config:
class
JwtPayload(pydantic.v1.main.BaseModel):
392class JwtPayload(BaseModel): 393 class Config: 394 extra = Extra.forbid 395 396 iss: Optional[str] = Field( 397 None, 398 description="The user/principal that issued the JWT. Commonly a value unique to the user.", 399 examples=["{{ config['iss'] }}"], 400 title="Issuer", 401 ) 402 sub: Optional[str] = Field( 403 None, 404 description="The subject of the JWT. Commonly defined by the API.", 405 title="Subject", 406 ) 407 aud: Optional[str] = Field( 408 None, 409 description="The recipient that the JWT is intended for. Commonly defined by the API.", 410 examples=["appstoreconnect-v1"], 411 title="Audience", 412 )
class
JwtPayload.Config:
class
JwtAuthenticator(pydantic.v1.main.BaseModel):
415class JwtAuthenticator(BaseModel): 416 type: Literal["JwtAuthenticator"] 417 secret_key: str = Field( 418 ..., 419 description="Secret used to sign the JSON web token.", 420 examples=["{{ config['secret_key'] }}"], 421 ) 422 base64_encode_secret_key: Optional[bool] = Field( 423 False, 424 description='When set to true, the secret key will be base64 encoded prior to being encoded as part of the JWT. Only set to "true" when required by the API.', 425 ) 426 algorithm: Algorithm = Field( 427 ..., 428 description="Algorithm used to sign the JSON web token.", 429 examples=["ES256", "HS256", "RS256", "{{ config['algorithm'] }}"], 430 ) 431 token_duration: Optional[int] = Field( 432 1200, 433 description="The amount of time in seconds a JWT token can be valid after being issued.", 434 examples=[1200, 3600], 435 title="Token Duration", 436 ) 437 header_prefix: Optional[str] = Field( 438 None, 439 description="The prefix to be used within the Authentication header.", 440 examples=["Bearer", "Basic"], 441 title="Header Prefix", 442 ) 443 jwt_headers: Optional[JwtHeaders] = Field( 444 None, 445 description="JWT headers used when signing JSON web token.", 446 title="JWT Headers", 447 ) 448 additional_jwt_headers: Optional[Dict[str, Any]] = Field( 449 None, 450 description="Additional headers to be included with the JWT headers object.", 451 title="Additional JWT Headers", 452 ) 453 jwt_payload: Optional[JwtPayload] = Field( 454 None, 455 description="JWT Payload used when signing JSON web token.", 456 title="JWT Payload", 457 ) 458 additional_jwt_payload: Optional[Dict[str, Any]] = Field( 459 None, 460 description="Additional properties to be added to the JWT payload.", 461 title="Additional JWT Payload Properties", 462 ) 463 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters")
algorithm: Algorithm
jwt_headers: Optional[JwtHeaders]
jwt_payload: Optional[JwtPayload]
class
RefreshTokenUpdater(pydantic.v1.main.BaseModel):
466class RefreshTokenUpdater(BaseModel): 467 refresh_token_name: Optional[str] = Field( 468 "refresh_token", 469 description="The name of the property which contains the updated refresh token in the response from the token refresh endpoint.", 470 examples=["refresh_token"], 471 title="Refresh Token Property Name", 472 ) 473 access_token_config_path: Optional[List[str]] = Field( 474 ["credentials", "access_token"], 475 description="Config path to the access token. Make sure the field actually exists in the config.", 476 examples=[["credentials", "access_token"], ["access_token"]], 477 title="Config Path To Access Token", 478 ) 479 refresh_token_config_path: Optional[List[str]] = Field( 480 ["credentials", "refresh_token"], 481 description="Config path to the access token. Make sure the field actually exists in the config.", 482 examples=[["credentials", "refresh_token"], ["refresh_token"]], 483 title="Config Path To Refresh Token", 484 ) 485 token_expiry_date_config_path: Optional[List[str]] = Field( 486 ["credentials", "token_expiry_date"], 487 description="Config path to the expiry date. Make sure actually exists in the config.", 488 examples=[["credentials", "token_expiry_date"]], 489 title="Config Path To Expiry Date", 490 ) 491 refresh_token_error_status_codes: Optional[List[int]] = Field( 492 [], 493 description="Status Codes to Identify refresh token error in response (Refresh Token Error Key and Refresh Token Error Values should be also specified). Responses with one of the error status code and containing an error value will be flagged as a config error", 494 examples=[[400, 500]], 495 title="Refresh Token Error Status Codes", 496 ) 497 refresh_token_error_key: Optional[str] = Field( 498 "", 499 description="Key to Identify refresh token error in response (Refresh Token Error Status Codes and Refresh Token Error Values should be also specified).", 500 examples=["error"], 501 title="Refresh Token Error Key", 502 ) 503 refresh_token_error_values: Optional[List[str]] = Field( 504 [], 505 description='List of values to check for exception during token refresh process. Used to check if the error found in the response matches the key from the Refresh Token Error Key field (e.g. response={"error": "invalid_grant"}). Only responses with one of the error status code and containing an error value will be flagged as a config error', 506 examples=[["invalid_grant", "invalid_permissions"]], 507 title="Refresh Token Error Values", 508 )
class
OAuthAuthenticator(pydantic.v1.main.BaseModel):
511class OAuthAuthenticator(BaseModel): 512 type: Literal["OAuthAuthenticator"] 513 client_id_name: Optional[str] = Field( 514 "client_id", 515 description="The name of the property to use to refresh the `access_token`.", 516 examples=["custom_app_id"], 517 title="Client ID Property Name", 518 ) 519 client_id: Optional[str] = Field( 520 None, 521 description="The OAuth client ID. Fill it in the user inputs.", 522 examples=["{{ config['client_id }}", "{{ config['credentials']['client_id }}"], 523 title="Client ID", 524 ) 525 client_secret_name: Optional[str] = Field( 526 "client_secret", 527 description="The name of the property to use to refresh the `access_token`.", 528 examples=["custom_app_secret"], 529 title="Client Secret Property Name", 530 ) 531 client_secret: Optional[str] = Field( 532 None, 533 description="The OAuth client secret. Fill it in the user inputs.", 534 examples=[ 535 "{{ config['client_secret }}", 536 "{{ config['credentials']['client_secret }}", 537 ], 538 title="Client Secret", 539 ) 540 refresh_token_name: Optional[str] = Field( 541 "refresh_token", 542 description="The name of the property to use to refresh the `access_token`.", 543 examples=["custom_app_refresh_value"], 544 title="Refresh Token Property Name", 545 ) 546 refresh_token: Optional[str] = Field( 547 None, 548 description="Credential artifact used to get a new access token.", 549 examples=[ 550 "{{ config['refresh_token'] }}", 551 "{{ config['credentials]['refresh_token'] }}", 552 ], 553 title="Refresh Token", 554 ) 555 token_refresh_endpoint: Optional[str] = Field( 556 None, 557 description="The full URL to call to obtain a new access token.", 558 examples=["https://connect.squareup.com/oauth2/token"], 559 title="Token Refresh Endpoint", 560 ) 561 access_token_name: Optional[str] = Field( 562 "access_token", 563 description="The name of the property which contains the access token in the response from the token refresh endpoint.", 564 examples=["access_token"], 565 title="Access Token Property Name", 566 ) 567 access_token_value: Optional[str] = Field( 568 None, 569 description="The value of the access_token to bypass the token refreshing using `refresh_token`.", 570 examples=["secret_access_token_value"], 571 title="Access Token Value", 572 ) 573 expires_in_name: Optional[str] = Field( 574 "expires_in", 575 description="The name of the property which contains the expiry date in the response from the token refresh endpoint.", 576 examples=["expires_in"], 577 title="Token Expiry Property Name", 578 ) 579 grant_type_name: Optional[str] = Field( 580 "grant_type", 581 description="The name of the property to use to refresh the `access_token`.", 582 examples=["custom_grant_type"], 583 title="Grant Type Property Name", 584 ) 585 grant_type: Optional[str] = Field( 586 "refresh_token", 587 description="Specifies the OAuth2 grant type. If set to refresh_token, the refresh_token needs to be provided as well. For client_credentials, only client id and secret are required. Other grant types are not officially supported.", 588 examples=["refresh_token", "client_credentials"], 589 title="Grant Type", 590 ) 591 refresh_request_body: Optional[Dict[str, Any]] = Field( 592 None, 593 description="Body of the request sent to get a new access token.", 594 examples=[ 595 { 596 "applicationId": "{{ config['application_id'] }}", 597 "applicationSecret": "{{ config['application_secret'] }}", 598 "token": "{{ config['token'] }}", 599 } 600 ], 601 title="Refresh Request Body", 602 ) 603 refresh_request_headers: Optional[Dict[str, Any]] = Field( 604 None, 605 description="Headers of the request sent to get a new access token.", 606 examples=[ 607 { 608 "Authorization": "<AUTH_TOKEN>", 609 "Content-Type": "application/x-www-form-urlencoded", 610 } 611 ], 612 title="Refresh Request Headers", 613 ) 614 scopes: Optional[List[str]] = Field( 615 None, 616 description="List of scopes that should be granted to the access token.", 617 examples=[["crm.list.read", "crm.objects.contacts.read", "crm.schema.contacts.read"]], 618 title="Scopes", 619 ) 620 token_expiry_date: Optional[str] = Field( 621 None, 622 description="The access token expiry date.", 623 examples=["2023-04-06T07:12:10.421833+00:00", 1680842386], 624 title="Token Expiry Date", 625 ) 626 token_expiry_date_format: Optional[str] = Field( 627 None, 628 description="The format of the time to expiration datetime. Provide it if the time is returned as a date-time string instead of seconds.", 629 examples=["%Y-%m-%d %H:%M:%S.%f+00:00"], 630 title="Token Expiry Date Format", 631 ) 632 refresh_token_updater: Optional[RefreshTokenUpdater] = Field( 633 None, 634 description="When the token updater is defined, new refresh tokens, access tokens and the access token expiry date are written back from the authentication response to the config object. This is important if the refresh token can only used once.", 635 title="Token Updater", 636 ) 637 profile_assertion: Optional[JwtAuthenticator] = Field( 638 None, 639 description="The authenticator being used to authenticate the client authenticator.", 640 title="Profile Assertion", 641 ) 642 use_profile_assertion: Optional[bool] = Field( 643 False, 644 description="Enable using profile assertion as a flow for OAuth authorization.", 645 title="Use Profile Assertion", 646 ) 647 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters")
refresh_token_updater: Optional[RefreshTokenUpdater]
profile_assertion: Optional[JwtAuthenticator]
class
Rate(pydantic.v1.main.BaseModel):
650class Rate(BaseModel): 651 class Config: 652 extra = Extra.allow 653 654 limit: Union[int, str] = Field( 655 ..., 656 description="The maximum number of calls allowed within the interval.", 657 title="Limit", 658 ) 659 interval: str = Field( 660 ..., 661 description="The time interval for the rate limit.", 662 examples=["PT1H", "P1D"], 663 title="Interval", 664 )
class
Rate.Config:
class
HttpRequestRegexMatcher(pydantic.v1.main.BaseModel):
667class HttpRequestRegexMatcher(BaseModel): 668 class Config: 669 extra = Extra.allow 670 671 method: Optional[str] = Field( 672 None, description="The HTTP method to match (e.g., GET, POST).", title="Method" 673 ) 674 url_base: Optional[str] = Field( 675 None, 676 description='The base URL (scheme and host, e.g. "https://api.example.com") to match.', 677 title="URL Base", 678 ) 679 url_path_pattern: Optional[str] = Field( 680 None, 681 description="A regular expression pattern to match the URL path.", 682 title="URL Path Pattern", 683 ) 684 params: Optional[Dict[str, Any]] = Field( 685 None, description="The query parameters to match.", title="Parameters" 686 ) 687 headers: Optional[Dict[str, Any]] = Field( 688 None, description="The headers to match.", title="Headers" 689 )
class
HttpRequestRegexMatcher.Config:
class
DpathExtractor(pydantic.v1.main.BaseModel):
692class DpathExtractor(BaseModel): 693 type: Literal["DpathExtractor"] 694 field_path: List[str] = Field( 695 ..., 696 description='List of potentially nested fields describing the full path of the field to extract. Use "*" to extract all values from an array. See more info in the [docs](https://docs.airbyte.com/connector-development/config-based/understanding-the-yaml-file/record-selector).', 697 examples=[ 698 ["data"], 699 ["data", "records"], 700 ["data", "{{ parameters.name }}"], 701 ["data", "*", "record"], 702 ], 703 title="Field Path", 704 ) 705 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters")
class
ResponseToFileExtractor(pydantic.v1.main.BaseModel):
class
ExponentialBackoffStrategy(pydantic.v1.main.BaseModel):
713class ExponentialBackoffStrategy(BaseModel): 714 type: Literal["ExponentialBackoffStrategy"] 715 factor: Optional[Union[float, str]] = Field( 716 5, 717 description="Multiplicative constant applied on each retry.", 718 examples=[5, 5.5, "10"], 719 title="Factor", 720 ) 721 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters")
class
GroupByKeyMergeStrategy(pydantic.v1.main.BaseModel):
724class GroupByKeyMergeStrategy(BaseModel): 725 type: Literal["GroupByKeyMergeStrategy"] 726 key: Union[str, List[str]] = Field( 727 ..., 728 description="The name of the field on the record whose value will be used to group properties that were retrieved through multiple API requests.", 729 examples=["id", ["parent_id", "end_date"]], 730 title="Key", 731 ) 732 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters")
class
SessionTokenRequestBearerAuthenticator(pydantic.v1.main.BaseModel):
class
HttpMethod(enum.Enum):
An enumeration.
GET =
<HttpMethod.GET: 'GET'>
POST =
<HttpMethod.POST: 'POST'>
class
Action(enum.Enum):
744class Action(Enum): 745 SUCCESS = "SUCCESS" 746 FAIL = "FAIL" 747 RETRY = "RETRY" 748 IGNORE = "IGNORE" 749 RATE_LIMITED = "RATE_LIMITED"
An enumeration.
SUCCESS =
<Action.SUCCESS: 'SUCCESS'>
FAIL =
<Action.FAIL: 'FAIL'>
RETRY =
<Action.RETRY: 'RETRY'>
IGNORE =
<Action.IGNORE: 'IGNORE'>
RATE_LIMITED =
<Action.RATE_LIMITED: 'RATE_LIMITED'>
class
FailureType(enum.Enum):
752class FailureType(Enum): 753 system_error = "system_error" 754 config_error = "config_error" 755 transient_error = "transient_error"
An enumeration.
system_error =
<FailureType.system_error: 'system_error'>
config_error =
<FailureType.config_error: 'config_error'>
transient_error =
<FailureType.transient_error: 'transient_error'>
class
HttpResponseFilter(pydantic.v1.main.BaseModel):
758class HttpResponseFilter(BaseModel): 759 type: Literal["HttpResponseFilter"] 760 action: Optional[Action] = Field( 761 None, 762 description="Action to execute if a response matches the filter.", 763 examples=["SUCCESS", "FAIL", "RETRY", "IGNORE", "RATE_LIMITED"], 764 title="Action", 765 ) 766 failure_type: Optional[FailureType] = Field( 767 None, 768 description="Failure type of traced exception if a response matches the filter.", 769 examples=["system_error", "config_error", "transient_error"], 770 title="Failure Type", 771 ) 772 error_message: Optional[str] = Field( 773 None, 774 description="Error Message to display if the response matches the filter.", 775 title="Error Message", 776 ) 777 error_message_contains: Optional[str] = Field( 778 None, 779 description="Match the response if its error message contains the substring.", 780 example=["This API operation is not enabled for this site"], 781 title="Error Message Substring", 782 ) 783 http_codes: Optional[List[int]] = Field( 784 None, 785 description="Match the response if its HTTP code is included in this list.", 786 examples=[[420, 429], [500]], 787 title="HTTP Codes", 788 unique_items=True, 789 ) 790 predicate: Optional[str] = Field( 791 None, 792 description="Match the response if the predicate evaluates to true.", 793 examples=[ 794 "{{ 'Too much requests' in response }}", 795 "{{ 'error_code' in response and response['error_code'] == 'ComplexityException' }}", 796 ], 797 title="Predicate", 798 ) 799 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters")
action: Optional[Action]
failure_type: Optional[FailureType]
class
ComplexFieldType(pydantic.v1.main.BaseModel):
802class ComplexFieldType(BaseModel): 803 field_type: str 804 items: Optional[Union[str, ComplexFieldType]] = None
items: Union[str, ComplexFieldType, NoneType]
class
TypesMap(pydantic.v1.main.BaseModel):
807class TypesMap(BaseModel): 808 target_type: Union[str, List[str], ComplexFieldType] 809 current_type: Union[str, List[str]] 810 condition: Optional[str] = None
target_type: Union[str, List[str], ComplexFieldType]
class
SchemaTypeIdentifier(pydantic.v1.main.BaseModel):
813class SchemaTypeIdentifier(BaseModel): 814 type: Optional[Literal["SchemaTypeIdentifier"]] = None 815 schema_pointer: Optional[List[str]] = Field( 816 [], 817 description="List of nested fields defining the schema field path to extract. Defaults to [].", 818 title="Schema Path", 819 ) 820 key_pointer: List[str] = Field( 821 ..., 822 description="List of potentially nested fields describing the full path of the field key to extract.", 823 title="Key Path", 824 ) 825 type_pointer: Optional[List[str]] = Field( 826 None, 827 description="List of potentially nested fields describing the full path of the field type to extract.", 828 title="Type Path", 829 ) 830 types_mapping: Optional[List[TypesMap]] = None 831 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters")
types_mapping: Optional[List[TypesMap]]
class
InlineSchemaLoader(pydantic.v1.main.BaseModel):
834class InlineSchemaLoader(BaseModel): 835 type: Literal["InlineSchemaLoader"] 836 schema_: Optional[Dict[str, Any]] = Field( 837 None, 838 alias="schema", 839 description='Describes a streams\' schema. Refer to the <a href="https://docs.airbyte.com/understanding-airbyte/supported-data-types/">Data Types documentation</a> for more details on which types are valid.', 840 title="Schema", 841 )
class
JsonFileSchemaLoader(pydantic.v1.main.BaseModel):
844class JsonFileSchemaLoader(BaseModel): 845 type: Literal["JsonFileSchemaLoader"] 846 file_path: Optional[str] = Field( 847 None, 848 description="Path to the JSON file defining the schema. The path is relative to the connector module's root.", 849 example=["./schemas/users.json"], 850 title="File Path", 851 ) 852 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters")
class
JsonDecoder(pydantic.v1.main.BaseModel):
class
JsonlDecoder(pydantic.v1.main.BaseModel):
class
KeysToLower(pydantic.v1.main.BaseModel):
class
KeysToSnakeCase(pydantic.v1.main.BaseModel):
class
FlattenFields(pydantic.v1.main.BaseModel):
873class FlattenFields(BaseModel): 874 type: Literal["FlattenFields"] 875 flatten_lists: Optional[bool] = Field( 876 True, 877 description="Whether to flatten lists or leave it as is. Default is True.", 878 title="Flatten Lists", 879 ) 880 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters")
class
KeyTransformation(pydantic.v1.main.BaseModel):
883class KeyTransformation(BaseModel): 884 prefix: Optional[Union[str, None]] = Field( 885 None, 886 description="Prefix to add for object keys. If not provided original keys remain unchanged.", 887 examples=[ 888 "flattened_", 889 ], 890 title="Key Prefix", 891 ) 892 suffix: Optional[Union[str, None]] = Field( 893 None, 894 description="Suffix to add for object keys. If not provided original keys remain unchanged.", 895 examples=[ 896 "_flattened", 897 ], 898 title="Key Suffix", 899 )
class
DpathFlattenFields(pydantic.v1.main.BaseModel):
902class DpathFlattenFields(BaseModel): 903 type: Literal["DpathFlattenFields"] 904 field_path: List[str] = Field( 905 ..., 906 description="A path to field that needs to be flattened.", 907 examples=[["data"], ["data", "*", "field"]], 908 title="Field Path", 909 ) 910 delete_origin_value: Optional[bool] = Field( 911 None, 912 description="Whether to delete the origin value or keep it. Default is False.", 913 title="Delete Origin Value", 914 ) 915 replace_record: Optional[bool] = Field( 916 None, 917 description="Whether to replace the origin record or not. Default is False.", 918 title="Replace Origin Record", 919 ) 920 key_transformation: Optional[Union[KeyTransformation, None]] = Field( 921 None, 922 description="Transformation for object keys. If not provided, original key will be used.", 923 title="Key transformation", 924 ) 925 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters")
key_transformation: Optional[KeyTransformation]
class
KeysReplace(pydantic.v1.main.BaseModel):
928class KeysReplace(BaseModel): 929 type: Literal["KeysReplace"] 930 old: str = Field( 931 ..., 932 description="Old value to replace.", 933 examples=[ 934 " ", 935 "{{ record.id }}", 936 "{{ config['id'] }}", 937 "{{ stream_slice['id'] }}", 938 ], 939 title="Old value", 940 ) 941 new: str = Field( 942 ..., 943 description="New value to set.", 944 examples=[ 945 "_", 946 "{{ record.id }}", 947 "{{ config['id'] }}", 948 "{{ stream_slice['id'] }}", 949 ], 950 title="New value", 951 ) 952 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters")
class
IterableDecoder(pydantic.v1.main.BaseModel):
class
XmlDecoder(pydantic.v1.main.BaseModel):
class
CustomDecoder(pydantic.v1.main.BaseModel):
963class CustomDecoder(BaseModel): 964 class Config: 965 extra = Extra.allow 966 967 type: Literal["CustomDecoder"] 968 class_name: str = Field( 969 ..., 970 description="Fully-qualified name of the class that will be implementing the custom decoding. Has to be a sub class of Decoder. The format is `source_<name>.<package>.<class_name>`.", 971 examples=["source_amazon_ads.components.GzipJsonlDecoder"], 972 title="Class Name", 973 ) 974 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters")
class
CustomDecoder.Config:
class
MinMaxDatetime(pydantic.v1.main.BaseModel):
977class MinMaxDatetime(BaseModel): 978 type: Literal["MinMaxDatetime"] 979 datetime: str = Field( 980 ..., 981 description="Datetime value.", 982 examples=["2021-01-01", "2021-01-01T00:00:00Z", "{{ config['start_time'] }}"], 983 title="Datetime", 984 ) 985 datetime_format: Optional[str] = Field( 986 "", 987 description='Format of the datetime value. Defaults to "%Y-%m-%dT%H:%M:%S.%f%z" if left empty. Use placeholders starting with "%" to describe the format the API is using. The following placeholders are available:\n * **%s**: Epoch unix timestamp - `1686218963`\n * **%s_as_float**: Epoch unix timestamp in seconds as float with microsecond precision - `1686218963.123456`\n * **%ms**: Epoch unix timestamp - `1686218963123`\n * **%a**: Weekday (abbreviated) - `Sun`\n * **%A**: Weekday (full) - `Sunday`\n * **%w**: Weekday (decimal) - `0` (Sunday), `6` (Saturday)\n * **%d**: Day of the month (zero-padded) - `01`, `02`, ..., `31`\n * **%b**: Month (abbreviated) - `Jan`\n * **%B**: Month (full) - `January`\n * **%m**: Month (zero-padded) - `01`, `02`, ..., `12`\n * **%y**: Year (without century, zero-padded) - `00`, `01`, ..., `99`\n * **%Y**: Year (with century) - `0001`, `0002`, ..., `9999`\n * **%H**: Hour (24-hour, zero-padded) - `00`, `01`, ..., `23`\n * **%I**: Hour (12-hour, zero-padded) - `01`, `02`, ..., `12`\n * **%p**: AM/PM indicator\n * **%M**: Minute (zero-padded) - `00`, `01`, ..., `59`\n * **%S**: Second (zero-padded) - `00`, `01`, ..., `59`\n * **%f**: Microsecond (zero-padded to 6 digits) - `000000`, `000001`, ..., `999999`\n * **%_ms**: Millisecond (zero-padded to 3 digits) - `000`, `001`, ..., `999`\n * **%z**: UTC offset - `(empty)`, `+0000`, `-04:00`\n * **%Z**: Time zone name - `(empty)`, `UTC`, `GMT`\n * **%j**: Day of the year (zero-padded) - `001`, `002`, ..., `366`\n * **%U**: Week number of the year (Sunday as first day) - `00`, `01`, ..., `53`\n * **%W**: Week number of the year (Monday as first day) - `00`, `01`, ..., `53`\n * **%c**: Date and time representation - `Tue Aug 16 21:30:00 1988`\n * **%x**: Date representation - `08/16/1988`\n * **%X**: Time representation - `21:30:00`\n * **%%**: Literal \'%\' character\n\n Some placeholders depend on the locale of the underlying system - in most cases this locale is configured as en/US. For more information see the [Python documentation](https://docs.python.org/3/library/datetime.html#strftime-and-strptime-format-codes).\n', 988 examples=["%Y-%m-%dT%H:%M:%S.%f%z", "%Y-%m-%d", "%s"], 989 title="Datetime Format", 990 ) 991 max_datetime: Optional[str] = Field( 992 None, 993 description="Ceiling applied on the datetime value. Must be formatted with the datetime_format field.", 994 examples=["2021-01-01T00:00:00Z", "2021-01-01"], 995 title="Max Datetime", 996 ) 997 min_datetime: Optional[str] = Field( 998 None, 999 description="Floor applied on the datetime value. Must be formatted with the datetime_format field.", 1000 examples=["2010-01-01T00:00:00Z", "2010-01-01"], 1001 title="Min Datetime", 1002 ) 1003 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters")
class
NoAuth(pydantic.v1.main.BaseModel):
class
NoPagination(pydantic.v1.main.BaseModel):
class
State(pydantic.v1.main.BaseModel):
class
State.Config:
class
OauthConnectorInputSpecification(pydantic.v1.main.BaseModel):
1023class OauthConnectorInputSpecification(BaseModel): 1024 class Config: 1025 extra = Extra.allow 1026 1027 consent_url: str = Field( 1028 ..., 1029 description="The DeclarativeOAuth Specific string URL string template to initiate the authentication.\nThe placeholders are replaced during the processing to provide neccessary values.", 1030 examples=[ 1031 "https://domain.host.com/marketing_api/auth?{{client_id_key}}={{client_id_value}}&{{redirect_uri_key}}={{{{redirect_uri_value}} | urlEncoder}}&{{state_key}}={{state_value}}", 1032 "https://endpoint.host.com/oauth2/authorize?{{client_id_key}}={{client_id_value}}&{{redirect_uri_key}}={{{{redirect_uri_value}} | urlEncoder}}&{{scope_key}}={{{{scope_value}} | urlEncoder}}&{{state_key}}={{state_value}}&subdomain={{subdomain}}", 1033 ], 1034 title="Consent URL", 1035 ) 1036 scope: Optional[str] = Field( 1037 None, 1038 description="The DeclarativeOAuth Specific string of the scopes needed to be grant for authenticated user.", 1039 examples=["user:read user:read_orders workspaces:read"], 1040 title="Scopes", 1041 ) 1042 access_token_url: str = Field( 1043 ..., 1044 description="The DeclarativeOAuth Specific URL templated string to obtain the `access_token`, `refresh_token` etc.\nThe placeholders are replaced during the processing to provide neccessary values.", 1045 examples=[ 1046 "https://auth.host.com/oauth2/token?{{client_id_key}}={{client_id_value}}&{{client_secret_key}}={{client_secret_value}}&{{auth_code_key}}={{auth_code_value}}&{{redirect_uri_key}}={{{{redirect_uri_value}} | urlEncoder}}" 1047 ], 1048 title="Access Token URL", 1049 ) 1050 access_token_headers: Optional[Dict[str, Any]] = Field( 1051 None, 1052 description="The DeclarativeOAuth Specific optional headers to inject while exchanging the `auth_code` to `access_token` during `completeOAuthFlow` step.", 1053 examples=[ 1054 { 1055 "Authorization": "Basic {{ {{ client_id_value }}:{{ client_secret_value }} | base64Encoder }}" 1056 } 1057 ], 1058 title="Access Token Headers", 1059 ) 1060 access_token_params: Optional[Dict[str, Any]] = Field( 1061 None, 1062 description="The DeclarativeOAuth Specific optional query parameters to inject while exchanging the `auth_code` to `access_token` during `completeOAuthFlow` step.\nWhen this property is provided, the query params will be encoded as `Json` and included in the outgoing API request.", 1063 examples=[ 1064 { 1065 "{{ auth_code_key }}": "{{ auth_code_value }}", 1066 "{{ client_id_key }}": "{{ client_id_value }}", 1067 "{{ client_secret_key }}": "{{ client_secret_value }}", 1068 } 1069 ], 1070 title="Access Token Query Params (Json Encoded)", 1071 ) 1072 extract_output: Optional[List[str]] = Field( 1073 None, 1074 description="The DeclarativeOAuth Specific list of strings to indicate which keys should be extracted and returned back to the input config.", 1075 examples=[["access_token", "refresh_token", "other_field"]], 1076 title="Extract Output", 1077 ) 1078 state: Optional[State] = Field( 1079 None, 1080 description="The DeclarativeOAuth Specific object to provide the criteria of how the `state` query param should be constructed,\nincluding length and complexity.", 1081 examples=[{"min": 7, "max": 128}], 1082 title="Configurable State Query Param", 1083 ) 1084 client_id_key: Optional[str] = Field( 1085 None, 1086 description="The DeclarativeOAuth Specific optional override to provide the custom `client_id` key name, if required by data-provider.", 1087 examples=["my_custom_client_id_key_name"], 1088 title="Client ID Key Override", 1089 ) 1090 client_secret_key: Optional[str] = Field( 1091 None, 1092 description="The DeclarativeOAuth Specific optional override to provide the custom `client_secret` key name, if required by data-provider.", 1093 examples=["my_custom_client_secret_key_name"], 1094 title="Client Secret Key Override", 1095 ) 1096 scope_key: Optional[str] = Field( 1097 None, 1098 description="The DeclarativeOAuth Specific optional override to provide the custom `scope` key name, if required by data-provider.", 1099 examples=["my_custom_scope_key_key_name"], 1100 title="Scopes Key Override", 1101 ) 1102 state_key: Optional[str] = Field( 1103 None, 1104 description="The DeclarativeOAuth Specific optional override to provide the custom `state` key name, if required by data-provider.", 1105 examples=["my_custom_state_key_key_name"], 1106 title="State Key Override", 1107 ) 1108 auth_code_key: Optional[str] = Field( 1109 None, 1110 description="The DeclarativeOAuth Specific optional override to provide the custom `code` key name to something like `auth_code` or `custom_auth_code`, if required by data-provider.", 1111 examples=["my_custom_auth_code_key_name"], 1112 title="Auth Code Key Override", 1113 ) 1114 redirect_uri_key: Optional[str] = Field( 1115 None, 1116 description="The DeclarativeOAuth Specific optional override to provide the custom `redirect_uri` key name to something like `callback_uri`, if required by data-provider.", 1117 examples=["my_custom_redirect_uri_key_name"], 1118 title="Redirect URI Key Override", 1119 )
state: Optional[State]
class
OauthConnectorInputSpecification.Config:
class
OAuthConfigSpecification(pydantic.v1.main.BaseModel):
1122class OAuthConfigSpecification(BaseModel): 1123 class Config: 1124 extra = Extra.allow 1125 1126 oauth_user_input_from_connector_config_specification: Optional[Dict[str, Any]] = Field( 1127 None, 1128 description="OAuth specific blob. This is a Json Schema used to validate Json configurations used as input to OAuth.\nMust be a valid non-nested JSON that refers to properties from ConnectorSpecification.connectionSpecification\nusing special annotation 'path_in_connector_config'.\nThese are input values the user is entering through the UI to authenticate to the connector, that might also shared\nas inputs for syncing data via the connector.\nExamples:\nif no connector values is shared during oauth flow, oauth_user_input_from_connector_config_specification=[]\nif connector values such as 'app_id' inside the top level are used to generate the API url for the oauth flow,\n oauth_user_input_from_connector_config_specification={\n app_id: {\n type: string\n path_in_connector_config: ['app_id']\n }\n }\nif connector values such as 'info.app_id' nested inside another object are used to generate the API url for the oauth flow,\n oauth_user_input_from_connector_config_specification={\n app_id: {\n type: string\n path_in_connector_config: ['info', 'app_id']\n }\n }", 1129 examples=[ 1130 {"app_id": {"type": "string", "path_in_connector_config": ["app_id"]}}, 1131 { 1132 "app_id": { 1133 "type": "string", 1134 "path_in_connector_config": ["info", "app_id"], 1135 } 1136 }, 1137 ], 1138 title="OAuth user input", 1139 ) 1140 oauth_connector_input_specification: Optional[OauthConnectorInputSpecification] = Field( 1141 None, 1142 description='The DeclarativeOAuth specific blob.\nPertains to the fields defined by the connector relating to the OAuth flow.\n\nInterpolation capabilities:\n- The variables placeholders are declared as `{{my_var}}`.\n- The nested resolution variables like `{{ {{my_nested_var}} }}` is allowed as well.\n\n- The allowed interpolation context is:\n + base64Encoder - encode to `base64`, {{ {{my_var_a}}:{{my_var_b}} | base64Encoder }}\n + base64Decorer - decode from `base64` encoded string, {{ {{my_string_variable_or_string_value}} | base64Decoder }}\n + urlEncoder - encode the input string to URL-like format, {{ https://test.host.com/endpoint | urlEncoder}}\n + urlDecorer - decode the input url-encoded string into text format, {{ urlDecoder:https%3A%2F%2Fairbyte.io | urlDecoder}}\n + codeChallengeS256 - get the `codeChallenge` encoded value to provide additional data-provider specific authorisation values, {{ {{state_value}} | codeChallengeS256 }}\n\nExamples:\n - The TikTok Marketing DeclarativeOAuth spec:\n {\n "oauth_connector_input_specification": {\n "type": "object",\n "additionalProperties": false,\n "properties": {\n "consent_url": "https://ads.tiktok.com/marketing_api/auth?{{client_id_key}}={{client_id_value}}&{{redirect_uri_key}}={{ {{redirect_uri_value}} | urlEncoder}}&{{state_key}}={{state_value}}",\n "access_token_url": "https://business-api.tiktok.com/open_api/v1.3/oauth2/access_token/",\n "access_token_params": {\n "{{ auth_code_key }}": "{{ auth_code_value }}",\n "{{ client_id_key }}": "{{ client_id_value }}",\n "{{ client_secret_key }}": "{{ client_secret_value }}"\n },\n "access_token_headers": {\n "Content-Type": "application/json",\n "Accept": "application/json"\n },\n "extract_output": ["data.access_token"],\n "client_id_key": "app_id",\n "client_secret_key": "secret",\n "auth_code_key": "auth_code"\n }\n }\n }', 1143 title="DeclarativeOAuth Connector Specification", 1144 ) 1145 complete_oauth_output_specification: Optional[Dict[str, Any]] = Field( 1146 None, 1147 description="OAuth specific blob. This is a Json Schema used to validate Json configurations produced by the OAuth flows as they are\nreturned by the distant OAuth APIs.\nMust be a valid JSON describing the fields to merge back to `ConnectorSpecification.connectionSpecification`.\nFor each field, a special annotation `path_in_connector_config` can be specified to determine where to merge it,\nExamples:\n complete_oauth_output_specification={\n refresh_token: {\n type: string,\n path_in_connector_config: ['credentials', 'refresh_token']\n }\n }", 1148 examples=[ 1149 { 1150 "refresh_token": { 1151 "type": "string,", 1152 "path_in_connector_config": ["credentials", "refresh_token"], 1153 } 1154 } 1155 ], 1156 title="OAuth output specification", 1157 ) 1158 complete_oauth_server_input_specification: Optional[Dict[str, Any]] = Field( 1159 None, 1160 description="OAuth specific blob. This is a Json Schema used to validate Json configurations persisted as Airbyte Server configurations.\nMust be a valid non-nested JSON describing additional fields configured by the Airbyte Instance or Workspace Admins to be used by the\nserver when completing an OAuth flow (typically exchanging an auth code for refresh token).\nExamples:\n complete_oauth_server_input_specification={\n client_id: {\n type: string\n },\n client_secret: {\n type: string\n }\n }", 1161 examples=[{"client_id": {"type": "string"}, "client_secret": {"type": "string"}}], 1162 title="OAuth input specification", 1163 ) 1164 complete_oauth_server_output_specification: Optional[Dict[str, Any]] = Field( 1165 None, 1166 description="OAuth specific blob. This is a Json Schema used to validate Json configurations persisted as Airbyte Server configurations that\nalso need to be merged back into the connector configuration at runtime.\nThis is a subset configuration of `complete_oauth_server_input_specification` that filters fields out to retain only the ones that\nare necessary for the connector to function with OAuth. (some fields could be used during oauth flows but not needed afterwards, therefore\nthey would be listed in the `complete_oauth_server_input_specification` but not `complete_oauth_server_output_specification`)\nMust be a valid non-nested JSON describing additional fields configured by the Airbyte Instance or Workspace Admins to be used by the\nconnector when using OAuth flow APIs.\nThese fields are to be merged back to `ConnectorSpecification.connectionSpecification`.\nFor each field, a special annotation `path_in_connector_config` can be specified to determine where to merge it,\nExamples:\n complete_oauth_server_output_specification={\n client_id: {\n type: string,\n path_in_connector_config: ['credentials', 'client_id']\n },\n client_secret: {\n type: string,\n path_in_connector_config: ['credentials', 'client_secret']\n }\n }", 1167 examples=[ 1168 { 1169 "client_id": { 1170 "type": "string,", 1171 "path_in_connector_config": ["credentials", "client_id"], 1172 }, 1173 "client_secret": { 1174 "type": "string,", 1175 "path_in_connector_config": ["credentials", "client_secret"], 1176 }, 1177 } 1178 ], 1179 title="OAuth server output specification", 1180 )
oauth_connector_input_specification: Optional[OauthConnectorInputSpecification]
class
OAuthConfigSpecification.Config:
class
OffsetIncrement(pydantic.v1.main.BaseModel):
1183class OffsetIncrement(BaseModel): 1184 type: Literal["OffsetIncrement"] 1185 page_size: Optional[Union[int, str]] = Field( 1186 None, 1187 description="The number of records to include in each pages.", 1188 examples=[100, "{{ config['page_size'] }}"], 1189 title="Limit", 1190 ) 1191 inject_on_first_request: Optional[bool] = Field( 1192 False, 1193 description="Using the `offset` with value `0` during the first request", 1194 title="Inject Offset", 1195 ) 1196 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters")
class
PageIncrement(pydantic.v1.main.BaseModel):
1199class PageIncrement(BaseModel): 1200 type: Literal["PageIncrement"] 1201 page_size: Optional[Union[int, str]] = Field( 1202 None, 1203 description="The number of records to include in each pages.", 1204 examples=[100, "100", "{{ config['page_size'] }}"], 1205 title="Page Size", 1206 ) 1207 start_from_page: Optional[int] = Field( 1208 0, 1209 description="Index of the first page to request.", 1210 examples=[0, 1], 1211 title="Start From Page", 1212 ) 1213 inject_on_first_request: Optional[bool] = Field( 1214 False, 1215 description="Using the `page number` with value defined by `start_from_page` during the first request", 1216 title="Inject Page Number", 1217 ) 1218 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters")
class
PrimaryKey(pydantic.v1.main.BaseModel):
1221class PrimaryKey(BaseModel): 1222 __root__: Union[str, List[str], List[List[str]]] = Field( 1223 ..., 1224 description="The stream field to be used to distinguish unique records. Can either be a single field, an array of fields representing a composite key, or an array of arrays representing a composite key where the fields are nested fields.", 1225 examples=["id", ["code", "type"]], 1226 title="Primary Key", 1227 )
class
PropertyLimitType(enum.Enum):
1230class PropertyLimitType(Enum): 1231 characters = "characters" 1232 property_count = "property_count"
An enumeration.
characters =
<PropertyLimitType.characters: 'characters'>
property_count =
<PropertyLimitType.property_count: 'property_count'>
class
PropertyChunking(pydantic.v1.main.BaseModel):
1235class PropertyChunking(BaseModel): 1236 type: Literal["PropertyChunking"] 1237 property_limit_type: PropertyLimitType = Field( 1238 ..., 1239 description="The type used to determine the maximum number of properties per chunk", 1240 title="Property Limit Type", 1241 ) 1242 property_limit: Optional[int] = Field( 1243 None, 1244 description="The maximum amount of properties that can be retrieved per request according to the limit type.", 1245 title="Property Limit", 1246 ) 1247 record_merge_strategy: Optional[GroupByKeyMergeStrategy] = Field( 1248 None, 1249 description="Dictates how to records that require multiple requests to get all properties should be emitted to the destination", 1250 title="Record Merge Strategy", 1251 ) 1252 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters")
property_limit_type: PropertyLimitType
record_merge_strategy: Optional[GroupByKeyMergeStrategy]
class
RecordFilter(pydantic.v1.main.BaseModel):
1255class RecordFilter(BaseModel): 1256 type: Literal["RecordFilter"] 1257 condition: Optional[str] = Field( 1258 "", 1259 description="The predicate to filter a record. Records will be removed if evaluated to False.", 1260 examples=[ 1261 "{{ record['created_at'] >= stream_interval['start_time'] }}", 1262 "{{ record.status in ['active', 'expired'] }}", 1263 ], 1264 ) 1265 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters")
class
SchemaNormalization(enum.Enum):
An enumeration.
None_ =
<SchemaNormalization.None_: 'None'>
Default =
<SchemaNormalization.Default: 'Default'>
class
RemoveFields(pydantic.v1.main.BaseModel):
1273class RemoveFields(BaseModel): 1274 type: Literal["RemoveFields"] 1275 condition: Optional[str] = Field( 1276 "", 1277 description="The predicate to filter a property by a property value. Property will be removed if it is empty OR expression is evaluated to True.,", 1278 examples=[ 1279 "{{ property|string == '' }}", 1280 "{{ property is integer }}", 1281 "{{ property|length > 5 }}", 1282 "{{ property == 'some_string_to_match' }}", 1283 ], 1284 ) 1285 field_pointers: List[List[str]] = Field( 1286 ..., 1287 description="Array of paths defining the field to remove. Each item is an array whose field describe the path of a field to remove.", 1288 examples=[["tags"], [["content", "html"], ["content", "plain_text"]]], 1289 title="Field Paths", 1290 )
class
RequestPath(pydantic.v1.main.BaseModel):
class
InjectInto(enum.Enum):
1297class InjectInto(Enum): 1298 request_parameter = "request_parameter" 1299 header = "header" 1300 body_data = "body_data" 1301 body_json = "body_json"
An enumeration.
request_parameter =
<InjectInto.request_parameter: 'request_parameter'>
header =
<InjectInto.header: 'header'>
body_data =
<InjectInto.body_data: 'body_data'>
body_json =
<InjectInto.body_json: 'body_json'>
class
RequestOption(pydantic.v1.main.BaseModel):
1304class RequestOption(BaseModel): 1305 type: Literal["RequestOption"] 1306 field_name: Optional[str] = Field( 1307 None, 1308 description="Configures which key should be used in the location that the descriptor is being injected into. We hope to eventually deprecate this field in favor of `field_path` for all request_options, but must currently maintain it for backwards compatibility in the Builder.", 1309 examples=["segment_id"], 1310 title="Field Name", 1311 ) 1312 field_path: Optional[List[str]] = Field( 1313 None, 1314 description="Configures a path to be used for nested structures in JSON body requests (e.g. GraphQL queries)", 1315 examples=[["data", "viewer", "id"]], 1316 title="Field Path", 1317 ) 1318 inject_into: InjectInto = Field( 1319 ..., 1320 description="Configures where the descriptor should be set on the HTTP requests. Note that request parameters that are already encoded in the URL path will not be duplicated.", 1321 examples=["request_parameter", "header", "body_data", "body_json"], 1322 title="Inject Into", 1323 )
inject_into: InjectInto
class
Schemas(pydantic.v1.main.BaseModel):
class
Schemas.Config:
class
LegacySessionTokenAuthenticator(pydantic.v1.main.BaseModel):
1333class LegacySessionTokenAuthenticator(BaseModel): 1334 type: Literal["LegacySessionTokenAuthenticator"] 1335 header: str = Field( 1336 ..., 1337 description="The name of the session token header that will be injected in the request", 1338 examples=["X-Session"], 1339 title="Session Request Header", 1340 ) 1341 login_url: str = Field( 1342 ..., 1343 description="Path of the login URL (do not include the base URL)", 1344 examples=["session"], 1345 title="Login Path", 1346 ) 1347 session_token: Optional[str] = Field( 1348 None, 1349 description="Session token to use if using a pre-defined token. Not needed if authenticating with username + password pair", 1350 example=["{{ config['session_token'] }}"], 1351 title="Session Token", 1352 ) 1353 session_token_response_key: str = Field( 1354 ..., 1355 description="Name of the key of the session token to be extracted from the response", 1356 examples=["id"], 1357 title="Response Token Response Key", 1358 ) 1359 username: Optional[str] = Field( 1360 None, 1361 description="Username used to authenticate and obtain a session token", 1362 examples=[" {{ config['username'] }}"], 1363 title="Username", 1364 ) 1365 password: Optional[str] = Field( 1366 "", 1367 description="Password used to authenticate and obtain a session token", 1368 examples=["{{ config['password'] }}", ""], 1369 title="Password", 1370 ) 1371 validate_session_url: str = Field( 1372 ..., 1373 description="Path of the URL to use to validate that the session token is valid (do not include the base URL)", 1374 examples=["user/current"], 1375 title="Validate Session Path", 1376 ) 1377 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters")
class
CsvDecoder(pydantic.v1.main.BaseModel):
class
AsyncJobStatusMap(pydantic.v1.main.BaseModel):
class
ValueType(enum.Enum):
1394class ValueType(Enum): 1395 string = "string" 1396 number = "number" 1397 integer = "integer" 1398 boolean = "boolean"
An enumeration.
string =
<ValueType.string: 'string'>
number =
<ValueType.number: 'number'>
integer =
<ValueType.integer: 'integer'>
boolean =
<ValueType.boolean: 'boolean'>
class
WaitTimeFromHeader(pydantic.v1.main.BaseModel):
1401class WaitTimeFromHeader(BaseModel): 1402 type: Literal["WaitTimeFromHeader"] 1403 header: str = Field( 1404 ..., 1405 description="The name of the response header defining how long to wait before retrying.", 1406 examples=["Retry-After"], 1407 title="Response Header Name", 1408 ) 1409 regex: Optional[str] = Field( 1410 None, 1411 description="Optional regex to apply on the header to extract its value. The regex should define a capture group defining the wait time.", 1412 examples=["([-+]?\\d+)"], 1413 title="Extraction Regex", 1414 ) 1415 max_waiting_time_in_seconds: Optional[float] = Field( 1416 None, 1417 description="Given the value extracted from the header is greater than this value, stop the stream.", 1418 examples=[3600], 1419 title="Max Waiting Time in Seconds", 1420 ) 1421 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters")
class
WaitUntilTimeFromHeader(pydantic.v1.main.BaseModel):
1424class WaitUntilTimeFromHeader(BaseModel): 1425 type: Literal["WaitUntilTimeFromHeader"] 1426 header: str = Field( 1427 ..., 1428 description="The name of the response header defining how long to wait before retrying.", 1429 examples=["wait_time"], 1430 title="Response Header", 1431 ) 1432 min_wait: Optional[Union[float, str]] = Field( 1433 None, 1434 description="Minimum time to wait before retrying.", 1435 examples=[10, "60"], 1436 title="Minimum Wait Time", 1437 ) 1438 regex: Optional[str] = Field( 1439 None, 1440 description="Optional regex to apply on the header to extract its value. The regex should define a capture group defining the wait time.", 1441 examples=["([-+]?\\d+)"], 1442 title="Extraction Regex", 1443 ) 1444 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters")
class
ComponentMappingDefinition(pydantic.v1.main.BaseModel):
1447class ComponentMappingDefinition(BaseModel): 1448 type: Literal["ComponentMappingDefinition"] 1449 field_path: List[str] = Field( 1450 ..., 1451 description="A list of potentially nested fields indicating the full path where value will be added or updated.", 1452 examples=[ 1453 ["data"], 1454 ["data", "records"], 1455 ["data", 1, "name"], 1456 ["data", "{{ components_values.name }}"], 1457 ["data", "*", "record"], 1458 ["*", "**", "name"], 1459 ], 1460 title="Field Path", 1461 ) 1462 value: str = Field( 1463 ..., 1464 description="The dynamic or static value to assign to the key. Interpolated values can be used to dynamically determine the value during runtime.", 1465 examples=[ 1466 "{{ components_values['updates'] }}", 1467 "{{ components_values['MetaData']['LastUpdatedTime'] }}", 1468 "{{ config['segment_id'] }}", 1469 "{{ stream_slice['parent_id'] }}", 1470 "{{ stream_slice['extra_fields']['name'] }}", 1471 ], 1472 title="Value", 1473 ) 1474 value_type: Optional[ValueType] = Field( 1475 None, 1476 description="The expected data type of the value. If omitted, the type will be inferred from the value provided.", 1477 title="Value Type", 1478 ) 1479 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters")
value_type: Optional[ValueType]
class
StreamConfig(pydantic.v1.main.BaseModel):
1482class StreamConfig(BaseModel): 1483 type: Literal["StreamConfig"] 1484 configs_pointer: List[str] = Field( 1485 ..., 1486 description="A list of potentially nested fields indicating the full path in source config file where streams configs located.", 1487 examples=[["data"], ["data", "streams"], ["data", "{{ parameters.name }}"]], 1488 title="Configs Pointer", 1489 ) 1490 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters")
class
ConfigComponentsResolver(pydantic.v1.main.BaseModel):
1493class ConfigComponentsResolver(BaseModel): 1494 type: Literal["ConfigComponentsResolver"] 1495 stream_config: StreamConfig 1496 components_mapping: List[ComponentMappingDefinition] 1497 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters")
stream_config: StreamConfig
components_mapping: List[ComponentMappingDefinition]
class
AddedFieldDefinition(pydantic.v1.main.BaseModel):
1500class AddedFieldDefinition(BaseModel): 1501 type: Literal["AddedFieldDefinition"] 1502 path: List[str] = Field( 1503 ..., 1504 description="List of strings defining the path where to add the value on the record.", 1505 examples=[["segment_id"], ["metadata", "segment_id"]], 1506 title="Path", 1507 ) 1508 value: str = Field( 1509 ..., 1510 description="Value of the new field. Use {{ record['existing_field'] }} syntax to refer to other fields in the record.", 1511 examples=[ 1512 "{{ record['updates'] }}", 1513 "{{ record['MetaData']['LastUpdatedTime'] }}", 1514 "{{ stream_partition['segment_id'] }}", 1515 ], 1516 title="Value", 1517 ) 1518 value_type: Optional[ValueType] = Field( 1519 None, 1520 description="Type of the value. If not specified, the type will be inferred from the value.", 1521 title="Value Type", 1522 ) 1523 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters")
value_type: Optional[ValueType]
class
AddFields(pydantic.v1.main.BaseModel):
1526class AddFields(BaseModel): 1527 type: Literal["AddFields"] 1528 fields: List[AddedFieldDefinition] = Field( 1529 ..., 1530 description="List of transformations (path and corresponding value) that will be added to the record.", 1531 title="Fields", 1532 ) 1533 condition: Optional[str] = Field( 1534 "", 1535 description="Fields will be added if expression is evaluated to True.", 1536 examples=[ 1537 "{{ property|string == '' }}", 1538 "{{ property is integer }}", 1539 "{{ property|length > 5 }}", 1540 "{{ property == 'some_string_to_match' }}", 1541 ], 1542 ) 1543 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters")
fields: List[AddedFieldDefinition]
class
ApiKeyAuthenticator(pydantic.v1.main.BaseModel):
1546class ApiKeyAuthenticator(BaseModel): 1547 type: Literal["ApiKeyAuthenticator"] 1548 api_token: Optional[str] = Field( 1549 None, 1550 description="The API key to inject in the request. Fill it in the user inputs.", 1551 examples=["{{ config['api_key'] }}", "Token token={{ config['api_key'] }}"], 1552 title="API Key", 1553 ) 1554 header: Optional[str] = Field( 1555 None, 1556 description="The name of the HTTP header that will be set to the API key. This setting is deprecated, use inject_into instead. Header and inject_into can not be defined at the same time.", 1557 examples=["Authorization", "Api-Token", "X-Auth-Token"], 1558 title="Header Name", 1559 ) 1560 inject_into: Optional[RequestOption] = Field( 1561 None, 1562 description="Configure how the API Key will be sent in requests to the source API. Either inject_into or header has to be defined.", 1563 examples=[ 1564 {"inject_into": "header", "field_name": "Authorization"}, 1565 {"inject_into": "request_parameter", "field_name": "authKey"}, 1566 ], 1567 title="Inject API Key Into Outgoing HTTP Request", 1568 ) 1569 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters")
inject_into: Optional[RequestOption]
class
AuthFlow(pydantic.v1.main.BaseModel):
1572class AuthFlow(BaseModel): 1573 auth_flow_type: Optional[AuthFlowType] = Field( 1574 None, description="The type of auth to use", title="Auth flow type" 1575 ) 1576 predicate_key: Optional[List[str]] = Field( 1577 None, 1578 description="JSON path to a field in the connectorSpecification that should exist for the advanced auth to be applicable.", 1579 examples=[["credentials", "auth_type"]], 1580 title="Predicate key", 1581 ) 1582 predicate_value: Optional[str] = Field( 1583 None, 1584 description="Value of the predicate_key fields for the advanced auth to be applicable.", 1585 examples=["Oauth"], 1586 title="Predicate value", 1587 ) 1588 oauth_config_specification: Optional[OAuthConfigSpecification] = None
auth_flow_type: Optional[AuthFlowType]
oauth_config_specification: Optional[OAuthConfigSpecification]
class
CheckStream(pydantic.v1.main.BaseModel):
1591class CheckStream(BaseModel): 1592 type: Literal["CheckStream"] 1593 stream_names: Optional[List[str]] = Field( 1594 None, 1595 description="Names of the streams to try reading from when running a check operation.", 1596 examples=[["users"], ["users", "contacts"]], 1597 title="Stream Names", 1598 ) 1599 dynamic_streams_check_configs: Optional[List[DynamicStreamCheckConfig]] = None
dynamic_streams_check_configs: Optional[List[DynamicStreamCheckConfig]]
class
IncrementingCountCursor(pydantic.v1.main.BaseModel):
1602class IncrementingCountCursor(BaseModel): 1603 type: Literal["IncrementingCountCursor"] 1604 cursor_field: str = Field( 1605 ..., 1606 description="The location of the value on a record that will be used as a bookmark during sync. To ensure no data loss, the API must return records in ascending order based on the cursor field. Nested fields are not supported, so the field must be at the top level of the record. You can use a combination of Add Field and Remove Field transformations to move the nested field to the top.", 1607 examples=["created_at", "{{ config['record_cursor'] }}"], 1608 title="Cursor Field", 1609 ) 1610 start_value: Optional[Union[str, int]] = Field( 1611 None, 1612 description="The value that determines the earliest record that should be synced.", 1613 examples=[0, "{{ config['start_value'] }}"], 1614 title="Start Value", 1615 ) 1616 start_value_option: Optional[RequestOption] = Field( 1617 None, 1618 description="Optionally configures how the start value will be sent in requests to the source API.", 1619 title="Inject Start Value Into Outgoing HTTP Request", 1620 ) 1621 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters")
start_value_option: Optional[RequestOption]
class
DatetimeBasedCursor(pydantic.v1.main.BaseModel):
1624class DatetimeBasedCursor(BaseModel): 1625 type: Literal["DatetimeBasedCursor"] 1626 clamping: Optional[Clamping] = Field( 1627 None, 1628 description="This option is used to adjust the upper and lower boundaries of each datetime window to beginning and end of the provided target period (day, week, month)", 1629 title="Date Range Clamping", 1630 ) 1631 cursor_field: str = Field( 1632 ..., 1633 description="The location of the value on a record that will be used as a bookmark during sync. To ensure no data loss, the API must return records in ascending order based on the cursor field. Nested fields are not supported, so the field must be at the top level of the record. You can use a combination of Add Field and Remove Field transformations to move the nested field to the top.", 1634 examples=["created_at", "{{ config['record_cursor'] }}"], 1635 title="Cursor Field", 1636 ) 1637 datetime_format: str = Field( 1638 ..., 1639 description="The datetime format used to format the datetime values that are sent in outgoing requests to the API. Use placeholders starting with \"%\" to describe the format the API is using. The following placeholders are available:\n * **%s**: Epoch unix timestamp - `1686218963`\n * **%s_as_float**: Epoch unix timestamp in seconds as float with microsecond precision - `1686218963.123456`\n * **%ms**: Epoch unix timestamp (milliseconds) - `1686218963123`\n * **%a**: Weekday (abbreviated) - `Sun`\n * **%A**: Weekday (full) - `Sunday`\n * **%w**: Weekday (decimal) - `0` (Sunday), `6` (Saturday)\n * **%d**: Day of the month (zero-padded) - `01`, `02`, ..., `31`\n * **%b**: Month (abbreviated) - `Jan`\n * **%B**: Month (full) - `January`\n * **%m**: Month (zero-padded) - `01`, `02`, ..., `12`\n * **%y**: Year (without century, zero-padded) - `00`, `01`, ..., `99`\n * **%Y**: Year (with century) - `0001`, `0002`, ..., `9999`\n * **%H**: Hour (24-hour, zero-padded) - `00`, `01`, ..., `23`\n * **%I**: Hour (12-hour, zero-padded) - `01`, `02`, ..., `12`\n * **%p**: AM/PM indicator\n * **%M**: Minute (zero-padded) - `00`, `01`, ..., `59`\n * **%S**: Second (zero-padded) - `00`, `01`, ..., `59`\n * **%f**: Microsecond (zero-padded to 6 digits) - `000000`\n * **%_ms**: Millisecond (zero-padded to 3 digits) - `000`\n * **%z**: UTC offset - `(empty)`, `+0000`, `-04:00`\n * **%Z**: Time zone name - `(empty)`, `UTC`, `GMT`\n * **%j**: Day of the year (zero-padded) - `001`, `002`, ..., `366`\n * **%U**: Week number of the year (starting Sunday) - `00`, ..., `53`\n * **%W**: Week number of the year (starting Monday) - `00`, ..., `53`\n * **%c**: Date and time - `Tue Aug 16 21:30:00 1988`\n * **%x**: Date standard format - `08/16/1988`\n * **%X**: Time standard format - `21:30:00`\n * **%%**: Literal '%' character\n\n Some placeholders depend on the locale of the underlying system - in most cases this locale is configured as en/US. For more information see the [Python documentation](https://docs.python.org/3/library/datetime.html#strftime-and-strptime-format-codes).\n", 1640 examples=["%Y-%m-%dT%H:%M:%S.%f%z", "%Y-%m-%d", "%s", "%ms", "%s_as_float"], 1641 title="Outgoing Datetime Format", 1642 ) 1643 start_datetime: Union[str, MinMaxDatetime] = Field( 1644 ..., 1645 description="The datetime that determines the earliest record that should be synced.", 1646 examples=["2020-01-1T00:00:00Z", "{{ config['start_time'] }}"], 1647 title="Start Datetime", 1648 ) 1649 cursor_datetime_formats: Optional[List[str]] = Field( 1650 None, 1651 description="The possible formats for the cursor field, in order of preference. The first format that matches the cursor field value will be used to parse it. If not provided, the `datetime_format` will be used.", 1652 title="Cursor Datetime Formats", 1653 ) 1654 cursor_granularity: Optional[str] = Field( 1655 None, 1656 description="Smallest increment the datetime_format has (ISO 8601 duration) that is used to ensure the start of a slice does not overlap with the end of the previous one, e.g. for %Y-%m-%d the granularity should be P1D, for %Y-%m-%dT%H:%M:%SZ the granularity should be PT1S. Given this field is provided, `step` needs to be provided as well.", 1657 examples=["PT1S"], 1658 title="Cursor Granularity", 1659 ) 1660 end_datetime: Optional[Union[str, MinMaxDatetime]] = Field( 1661 None, 1662 description="The datetime that determines the last record that should be synced. If not provided, `{{ now_utc() }}` will be used.", 1663 examples=["2021-01-1T00:00:00Z", "{{ now_utc() }}", "{{ day_delta(-1) }}"], 1664 title="End Datetime", 1665 ) 1666 end_time_option: Optional[RequestOption] = Field( 1667 None, 1668 description="Optionally configures how the end datetime will be sent in requests to the source API.", 1669 title="Inject End Time Into Outgoing HTTP Request", 1670 ) 1671 is_data_feed: Optional[bool] = Field( 1672 None, 1673 description="A data feed API is an API that does not allow filtering and paginates the content from the most recent to the least recent. Given this, the CDK needs to know when to stop paginating and this field will generate a stop condition for pagination.", 1674 title="Whether the target API is formatted as a data feed", 1675 ) 1676 is_client_side_incremental: Optional[bool] = Field( 1677 None, 1678 description="If the target API endpoint does not take cursor values to filter records and returns all records anyway, the connector with this cursor will filter out records locally, and only emit new records from the last sync, hence incremental. This means that all records would be read from the API, but only new records will be emitted to the destination.", 1679 title="Whether the target API does not support filtering and returns all data (the cursor filters records in the client instead of the API side)", 1680 ) 1681 is_compare_strictly: Optional[bool] = Field( 1682 False, 1683 description="Set to True if the target API does not accept queries where the start time equal the end time.", 1684 title="Whether to skip requests if the start time equals the end time", 1685 ) 1686 global_substream_cursor: Optional[bool] = Field( 1687 False, 1688 description="This setting optimizes performance when the parent stream has thousands of partitions by storing the cursor as a single value rather than per partition. Notably, the substream state is updated only at the end of the sync, which helps prevent data loss in case of a sync failure. See more info in the [docs](https://docs.airbyte.com/connector-development/config-based/understanding-the-yaml-file/incremental-syncs).", 1689 title="Whether to store cursor as one value instead of per partition", 1690 ) 1691 lookback_window: Optional[str] = Field( 1692 None, 1693 description="Time interval before the start_datetime to read data for, e.g. P1M for looking back one month.", 1694 examples=["P1D", "P{{ config['lookback_days'] }}D"], 1695 title="Lookback Window", 1696 ) 1697 partition_field_end: Optional[str] = Field( 1698 None, 1699 description="Name of the partition start time field.", 1700 examples=["ending_time"], 1701 title="Partition Field End", 1702 ) 1703 partition_field_start: Optional[str] = Field( 1704 None, 1705 description="Name of the partition end time field.", 1706 examples=["starting_time"], 1707 title="Partition Field Start", 1708 ) 1709 start_time_option: Optional[RequestOption] = Field( 1710 None, 1711 description="Optionally configures how the start datetime will be sent in requests to the source API.", 1712 title="Inject Start Time Into Outgoing HTTP Request", 1713 ) 1714 step: Optional[str] = Field( 1715 None, 1716 description="The size of the time window (ISO8601 duration). Given this field is provided, `cursor_granularity` needs to be provided as well.", 1717 examples=["P1W", "{{ config['step_increment'] }}"], 1718 title="Step", 1719 ) 1720 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters")
clamping: Optional[Clamping]
start_datetime: Union[str, MinMaxDatetime]
end_datetime: Union[str, MinMaxDatetime, NoneType]
end_time_option: Optional[RequestOption]
start_time_option: Optional[RequestOption]
class
FixedWindowCallRatePolicy(pydantic.v1.main.BaseModel):
1723class FixedWindowCallRatePolicy(BaseModel): 1724 class Config: 1725 extra = Extra.allow 1726 1727 type: Literal["FixedWindowCallRatePolicy"] 1728 period: str = Field( 1729 ..., description="The time interval for the rate limit window.", title="Period" 1730 ) 1731 call_limit: int = Field( 1732 ..., 1733 description="The maximum number of calls allowed within the period.", 1734 title="Call Limit", 1735 ) 1736 matchers: List[HttpRequestRegexMatcher] = Field( 1737 ..., 1738 description="List of matchers that define which requests this policy applies to.", 1739 title="Matchers", 1740 )
matchers: List[HttpRequestRegexMatcher]
class
FixedWindowCallRatePolicy.Config:
class
MovingWindowCallRatePolicy(pydantic.v1.main.BaseModel):
1743class MovingWindowCallRatePolicy(BaseModel): 1744 class Config: 1745 extra = Extra.allow 1746 1747 type: Literal["MovingWindowCallRatePolicy"] 1748 rates: List[Rate] = Field( 1749 ..., 1750 description="List of rates that define the call limits for different time intervals.", 1751 title="Rates", 1752 ) 1753 matchers: List[HttpRequestRegexMatcher] = Field( 1754 ..., 1755 description="List of matchers that define which requests this policy applies to.", 1756 title="Matchers", 1757 )
rates: List[Rate]
matchers: List[HttpRequestRegexMatcher]
class
MovingWindowCallRatePolicy.Config:
class
UnlimitedCallRatePolicy(pydantic.v1.main.BaseModel):
1760class UnlimitedCallRatePolicy(BaseModel): 1761 class Config: 1762 extra = Extra.allow 1763 1764 type: Literal["UnlimitedCallRatePolicy"] 1765 matchers: List[HttpRequestRegexMatcher] = Field( 1766 ..., 1767 description="List of matchers that define which requests this policy applies to.", 1768 title="Matchers", 1769 )
matchers: List[HttpRequestRegexMatcher]
class
UnlimitedCallRatePolicy.Config:
class
DefaultErrorHandler(pydantic.v1.main.BaseModel):
1772class DefaultErrorHandler(BaseModel): 1773 type: Literal["DefaultErrorHandler"] 1774 backoff_strategies: Optional[ 1775 List[ 1776 Union[ 1777 ConstantBackoffStrategy, 1778 CustomBackoffStrategy, 1779 ExponentialBackoffStrategy, 1780 WaitTimeFromHeader, 1781 WaitUntilTimeFromHeader, 1782 ] 1783 ] 1784 ] = Field( 1785 None, 1786 description="List of backoff strategies to use to determine how long to wait before retrying a retryable request.", 1787 title="Backoff Strategies", 1788 ) 1789 max_retries: Optional[int] = Field( 1790 5, 1791 description="The maximum number of time to retry a retryable request before giving up and failing.", 1792 examples=[5, 0, 10], 1793 title="Max Retry Count", 1794 ) 1795 response_filters: Optional[List[HttpResponseFilter]] = Field( 1796 None, 1797 description="List of response filters to iterate on when deciding how to handle an error. When using an array of multiple filters, the filters will be applied sequentially and the response will be selected if it matches any of the filter's predicate.", 1798 title="Response Filters", 1799 ) 1800 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters")
backoff_strategies: Optional[List[Union[ConstantBackoffStrategy, CustomBackoffStrategy, ExponentialBackoffStrategy, WaitTimeFromHeader, WaitUntilTimeFromHeader]]]
response_filters: Optional[List[HttpResponseFilter]]
class
DefaultPaginator(pydantic.v1.main.BaseModel):
1803class DefaultPaginator(BaseModel): 1804 type: Literal["DefaultPaginator"] 1805 pagination_strategy: Union[ 1806 CursorPagination, CustomPaginationStrategy, OffsetIncrement, PageIncrement 1807 ] = Field( 1808 ..., 1809 description="Strategy defining how records are paginated.", 1810 title="Pagination Strategy", 1811 ) 1812 page_size_option: Optional[RequestOption] = None 1813 page_token_option: Optional[Union[RequestOption, RequestPath]] = None 1814 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters")
pagination_strategy: Union[CursorPagination, CustomPaginationStrategy, OffsetIncrement, PageIncrement]
page_size_option: Optional[RequestOption]
page_token_option: Union[RequestOption, RequestPath, NoneType]
class
SessionTokenRequestApiKeyAuthenticator(pydantic.v1.main.BaseModel):
1817class SessionTokenRequestApiKeyAuthenticator(BaseModel): 1818 type: Literal["ApiKey"] 1819 inject_into: RequestOption = Field( 1820 ..., 1821 description="Configure how the API Key will be sent in requests to the source API.", 1822 examples=[ 1823 {"inject_into": "header", "field_name": "Authorization"}, 1824 {"inject_into": "request_parameter", "field_name": "authKey"}, 1825 ], 1826 title="Inject API Key Into Outgoing HTTP Request", 1827 )
inject_into: RequestOption
class
ListPartitionRouter(pydantic.v1.main.BaseModel):
1830class ListPartitionRouter(BaseModel): 1831 type: Literal["ListPartitionRouter"] 1832 cursor_field: str = Field( 1833 ..., 1834 description='While iterating over list values, the name of field used to reference a list value. The partition value can be accessed with string interpolation. e.g. "{{ stream_partition[\'my_key\'] }}" where "my_key" is the value of the cursor_field.', 1835 examples=["section", "{{ config['section_key'] }}"], 1836 title="Current Partition Value Identifier", 1837 ) 1838 values: Union[str, List[str]] = Field( 1839 ..., 1840 description="The list of attributes being iterated over and used as input for the requests made to the source API.", 1841 examples=[["section_a", "section_b", "section_c"], "{{ config['sections'] }}"], 1842 title="Partition Values", 1843 ) 1844 request_option: Optional[RequestOption] = Field( 1845 None, 1846 description="A request option describing where the list value should be injected into and under what field name if applicable.", 1847 title="Inject Partition Value Into Outgoing HTTP Request", 1848 ) 1849 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters")
request_option: Optional[RequestOption]
class
RecordSelector(pydantic.v1.main.BaseModel):
1852class RecordSelector(BaseModel): 1853 type: Literal["RecordSelector"] 1854 extractor: Union[CustomRecordExtractor, DpathExtractor] 1855 record_filter: Optional[Union[CustomRecordFilter, RecordFilter]] = Field( 1856 None, 1857 description="Responsible for filtering records to be emitted by the Source.", 1858 title="Record Filter", 1859 ) 1860 schema_normalization: Optional[Union[SchemaNormalization, CustomSchemaNormalization]] = Field( 1861 SchemaNormalization.None_, 1862 description="Responsible for normalization according to the schema.", 1863 title="Schema Normalization", 1864 ) 1865 transform_before_filtering: Optional[bool] = Field( 1866 False, 1867 description="If true, transformation will be applied before record filtering.", 1868 ) 1869 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters")
extractor: Union[CustomRecordExtractor, DpathExtractor]
record_filter: Union[CustomRecordFilter, RecordFilter, NoneType]
schema_normalization: Union[SchemaNormalization, CustomSchemaNormalization, NoneType]
class
GzipDecoder(pydantic.v1.main.BaseModel):
1872class GzipDecoder(BaseModel): 1873 type: Literal["GzipDecoder"] 1874 decoder: Union[CsvDecoder, GzipDecoder, JsonDecoder, JsonlDecoder]
decoder: Union[CsvDecoder, GzipDecoder, JsonDecoder, JsonlDecoder]
class
Spec(pydantic.v1.main.BaseModel):
1877class Spec(BaseModel): 1878 type: Literal["Spec"] 1879 connection_specification: Dict[str, Any] = Field( 1880 ..., 1881 description="A connection specification describing how a the connector can be configured.", 1882 title="Connection Specification", 1883 ) 1884 documentation_url: Optional[str] = Field( 1885 None, 1886 description="URL of the connector's documentation page.", 1887 examples=["https://docs.airbyte.com/integrations/sources/dremio"], 1888 title="Documentation URL", 1889 ) 1890 advanced_auth: Optional[AuthFlow] = Field( 1891 None, 1892 description="Advanced specification for configuring the authentication flow.", 1893 title="Advanced Auth", 1894 )
advanced_auth: Optional[AuthFlow]
class
CompositeErrorHandler(pydantic.v1.main.BaseModel):
1897class CompositeErrorHandler(BaseModel): 1898 type: Literal["CompositeErrorHandler"] 1899 error_handlers: List[Union[CompositeErrorHandler, DefaultErrorHandler]] = Field( 1900 ..., 1901 description="List of error handlers to iterate on to determine how to handle a failed response.", 1902 title="Error Handlers", 1903 ) 1904 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters")
error_handlers: List[Union[CompositeErrorHandler, DefaultErrorHandler]]
class
HTTPAPIBudget(pydantic.v1.main.BaseModel):
1907class HTTPAPIBudget(BaseModel): 1908 class Config: 1909 extra = Extra.allow 1910 1911 type: Literal["HTTPAPIBudget"] 1912 policies: List[ 1913 Union[ 1914 FixedWindowCallRatePolicy, 1915 MovingWindowCallRatePolicy, 1916 UnlimitedCallRatePolicy, 1917 ] 1918 ] = Field( 1919 ..., 1920 description="List of call rate policies that define how many calls are allowed.", 1921 title="Policies", 1922 ) 1923 ratelimit_reset_header: Optional[str] = Field( 1924 "ratelimit-reset", 1925 description="The HTTP response header name that indicates when the rate limit resets.", 1926 title="Rate Limit Reset Header", 1927 ) 1928 ratelimit_remaining_header: Optional[str] = Field( 1929 "ratelimit-remaining", 1930 description="The HTTP response header name that indicates the number of remaining allowed calls.", 1931 title="Rate Limit Remaining Header", 1932 ) 1933 status_codes_for_ratelimit_hit: Optional[List[int]] = Field( 1934 [429], 1935 description="List of HTTP status codes that indicate a rate limit has been hit.", 1936 title="Status Codes for Rate Limit Hit", 1937 )
policies: List[Union[FixedWindowCallRatePolicy, MovingWindowCallRatePolicy, UnlimitedCallRatePolicy]]
class
HTTPAPIBudget.Config:
class
ZipfileDecoder(pydantic.v1.main.BaseModel):
1940class ZipfileDecoder(BaseModel): 1941 class Config: 1942 extra = Extra.allow 1943 1944 type: Literal["ZipfileDecoder"] 1945 decoder: Union[CsvDecoder, GzipDecoder, JsonDecoder, JsonlDecoder] = Field( 1946 ..., 1947 description="Parser to parse the decompressed data from the zipfile(s).", 1948 title="Parser", 1949 )
decoder: Union[CsvDecoder, GzipDecoder, JsonDecoder, JsonlDecoder]
class
ZipfileDecoder.Config:
class
DeclarativeSource1(pydantic.v1.main.BaseModel):
1952class DeclarativeSource1(BaseModel): 1953 class Config: 1954 extra = Extra.forbid 1955 1956 type: Literal["DeclarativeSource"] 1957 check: Union[CheckStream, CheckDynamicStream] 1958 streams: List[Union[DeclarativeStream, StateDelegatingStream]] 1959 dynamic_streams: Optional[List[DynamicDeclarativeStream]] = None 1960 version: str = Field( 1961 ..., 1962 description="The version of the Airbyte CDK used to build and test the source.", 1963 ) 1964 schemas: Optional[Schemas] = None 1965 definitions: Optional[Dict[str, Any]] = None 1966 spec: Optional[Spec] = None 1967 concurrency_level: Optional[ConcurrencyLevel] = None 1968 api_budget: Optional[HTTPAPIBudget] = None 1969 max_concurrent_async_job_count: Optional[Union[int, str]] = Field( 1970 None, 1971 description="Maximum number of concurrent asynchronous jobs to run. This property is only relevant for sources/streams that support asynchronous job execution through the AsyncRetriever (e.g. a report-based stream that initiates a job, polls the job status, and then fetches the job results). This is often set by the API's maximum number of concurrent jobs on the account level. Refer to the API's documentation for this information.", 1972 examples=[3, "{{ config['max_concurrent_async_job_count'] }}"], 1973 title="Maximum Concurrent Asynchronous Jobs", 1974 ) 1975 metadata: Optional[Dict[str, Any]] = Field( 1976 None, 1977 description="For internal Airbyte use only - DO NOT modify manually. Used by consumers of declarative manifests for storing related metadata.", 1978 ) 1979 description: Optional[str] = Field( 1980 None, 1981 description="A description of the connector. It will be presented on the Source documentation page.", 1982 )
check: Union[CheckStream, CheckDynamicStream]
streams: List[Union[DeclarativeStream, StateDelegatingStream]]
dynamic_streams: Optional[List[DynamicDeclarativeStream]]
schemas: Optional[Schemas]
spec: Optional[Spec]
concurrency_level: Optional[ConcurrencyLevel]
api_budget: Optional[HTTPAPIBudget]
class
DeclarativeSource1.Config:
class
DeclarativeSource2(pydantic.v1.main.BaseModel):
1985class DeclarativeSource2(BaseModel): 1986 class Config: 1987 extra = Extra.forbid 1988 1989 type: Literal["DeclarativeSource"] 1990 check: Union[CheckStream, CheckDynamicStream] 1991 streams: Optional[List[Union[DeclarativeStream, StateDelegatingStream]]] = None 1992 dynamic_streams: List[DynamicDeclarativeStream] 1993 version: str = Field( 1994 ..., 1995 description="The version of the Airbyte CDK used to build and test the source.", 1996 ) 1997 schemas: Optional[Schemas] = None 1998 definitions: Optional[Dict[str, Any]] = None 1999 spec: Optional[Spec] = None 2000 concurrency_level: Optional[ConcurrencyLevel] = None 2001 api_budget: Optional[HTTPAPIBudget] = None 2002 max_concurrent_async_job_count: Optional[Union[int, str]] = Field( 2003 None, 2004 description="Maximum number of concurrent asynchronous jobs to run. This property is only relevant for sources/streams that support asynchronous job execution through the AsyncRetriever (e.g. a report-based stream that initiates a job, polls the job status, and then fetches the job results). This is often set by the API's maximum number of concurrent jobs on the account level. Refer to the API's documentation for this information.", 2005 examples=[3, "{{ config['max_concurrent_async_job_count'] }}"], 2006 title="Maximum Concurrent Asynchronous Jobs", 2007 ) 2008 metadata: Optional[Dict[str, Any]] = Field( 2009 None, 2010 description="For internal Airbyte use only - DO NOT modify manually. Used by consumers of declarative manifests for storing related metadata.", 2011 ) 2012 description: Optional[str] = Field( 2013 None, 2014 description="A description of the connector. It will be presented on the Source documentation page.", 2015 )
check: Union[CheckStream, CheckDynamicStream]
streams: Optional[List[Union[DeclarativeStream, StateDelegatingStream]]]
dynamic_streams: List[DynamicDeclarativeStream]
schemas: Optional[Schemas]
spec: Optional[Spec]
concurrency_level: Optional[ConcurrencyLevel]
api_budget: Optional[HTTPAPIBudget]
class
DeclarativeSource2.Config:
class
DeclarativeSource(pydantic.v1.main.BaseModel):
2018class DeclarativeSource(BaseModel): 2019 class Config: 2020 extra = Extra.forbid 2021 2022 __root__: Union[DeclarativeSource1, DeclarativeSource2] = Field( 2023 ..., 2024 description="An API source that extracts data according to its declarative components.", 2025 title="DeclarativeSource", 2026 )
class
DeclarativeSource.Config:
class
SelectiveAuthenticator(pydantic.v1.main.BaseModel):
2029class SelectiveAuthenticator(BaseModel): 2030 class Config: 2031 extra = Extra.allow 2032 2033 type: Literal["SelectiveAuthenticator"] 2034 authenticator_selection_path: List[str] = Field( 2035 ..., 2036 description="Path of the field in config with selected authenticator name", 2037 examples=[["auth"], ["auth", "type"]], 2038 title="Authenticator Selection Path", 2039 ) 2040 authenticators: Dict[ 2041 str, 2042 Union[ 2043 ApiKeyAuthenticator, 2044 BasicHttpAuthenticator, 2045 BearerAuthenticator, 2046 CustomAuthenticator, 2047 OAuthAuthenticator, 2048 JwtAuthenticator, 2049 NoAuth, 2050 SessionTokenAuthenticator, 2051 LegacySessionTokenAuthenticator, 2052 ], 2053 ] = Field( 2054 ..., 2055 description="Authenticators to select from.", 2056 examples=[ 2057 { 2058 "authenticators": { 2059 "token": "#/definitions/ApiKeyAuthenticator", 2060 "oauth": "#/definitions/OAuthAuthenticator", 2061 "jwt": "#/definitions/JwtAuthenticator", 2062 } 2063 } 2064 ], 2065 title="Authenticators", 2066 ) 2067 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters")
authenticators: Dict[str, Union[ApiKeyAuthenticator, BasicHttpAuthenticator, BearerAuthenticator, CustomAuthenticator, OAuthAuthenticator, JwtAuthenticator, NoAuth, SessionTokenAuthenticator, LegacySessionTokenAuthenticator]]
class
SelectiveAuthenticator.Config:
class
DeclarativeStream(pydantic.v1.main.BaseModel):
2070class DeclarativeStream(BaseModel): 2071 class Config: 2072 extra = Extra.allow 2073 2074 type: Literal["DeclarativeStream"] 2075 retriever: Union[AsyncRetriever, CustomRetriever, SimpleRetriever] = Field( 2076 ..., 2077 description="Component used to coordinate how records are extracted across stream slices and request pages.", 2078 title="Retriever", 2079 ) 2080 incremental_sync: Optional[ 2081 Union[CustomIncrementalSync, DatetimeBasedCursor, IncrementingCountCursor] 2082 ] = Field( 2083 None, 2084 description="Component used to fetch data incrementally based on a time field in the data.", 2085 title="Incremental Sync", 2086 ) 2087 name: Optional[str] = Field("", description="The stream name.", example=["Users"], title="Name") 2088 primary_key: Optional[PrimaryKey] = Field( 2089 "", description="The primary key of the stream.", title="Primary Key" 2090 ) 2091 schema_loader: Optional[ 2092 Union[ 2093 DynamicSchemaLoader, 2094 InlineSchemaLoader, 2095 JsonFileSchemaLoader, 2096 CustomSchemaLoader, 2097 ] 2098 ] = Field( 2099 None, 2100 description="Component used to retrieve the schema for the current stream.", 2101 title="Schema Loader", 2102 ) 2103 transformations: Optional[ 2104 List[ 2105 Union[ 2106 AddFields, 2107 CustomTransformation, 2108 RemoveFields, 2109 KeysToLower, 2110 KeysToSnakeCase, 2111 FlattenFields, 2112 DpathFlattenFields, 2113 KeysReplace, 2114 ] 2115 ] 2116 ] = Field( 2117 None, 2118 description="A list of transformations to be applied to each output record.", 2119 title="Transformations", 2120 ) 2121 state_migrations: Optional[ 2122 List[Union[LegacyToPerPartitionStateMigration, CustomStateMigration]] 2123 ] = Field( 2124 [], 2125 description="Array of state migrations to be applied on the input state", 2126 title="State Migrations", 2127 ) 2128 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters")
retriever: Union[AsyncRetriever, CustomRetriever, SimpleRetriever]
incremental_sync: Union[CustomIncrementalSync, DatetimeBasedCursor, IncrementingCountCursor, NoneType]
primary_key: Optional[PrimaryKey]
schema_loader: Union[DynamicSchemaLoader, InlineSchemaLoader, JsonFileSchemaLoader, CustomSchemaLoader, NoneType]
transformations: Optional[List[Union[AddFields, CustomTransformation, RemoveFields, KeysToLower, KeysToSnakeCase, FlattenFields, DpathFlattenFields, KeysReplace]]]
state_migrations: Optional[List[Union[LegacyToPerPartitionStateMigration, CustomStateMigration]]]
class
DeclarativeStream.Config:
class
SessionTokenAuthenticator(pydantic.v1.main.BaseModel):
2131class SessionTokenAuthenticator(BaseModel): 2132 type: Literal["SessionTokenAuthenticator"] 2133 login_requester: HttpRequester = Field( 2134 ..., 2135 description="Description of the request to perform to obtain a session token to perform data requests. The response body is expected to be a JSON object with a session token property.", 2136 examples=[ 2137 { 2138 "type": "HttpRequester", 2139 "url_base": "https://my_api.com", 2140 "path": "/login", 2141 "authenticator": { 2142 "type": "BasicHttpAuthenticator", 2143 "username": "{{ config.username }}", 2144 "password": "{{ config.password }}", 2145 }, 2146 } 2147 ], 2148 title="Login Requester", 2149 ) 2150 session_token_path: List[str] = Field( 2151 ..., 2152 description="The path in the response body returned from the login requester to the session token.", 2153 examples=[["access_token"], ["result", "token"]], 2154 title="Session Token Path", 2155 ) 2156 expiration_duration: Optional[str] = Field( 2157 None, 2158 description="The duration in ISO 8601 duration notation after which the session token expires, starting from the time it was obtained. Omitting it will result in the session token being refreshed for every request.", 2159 examples=["PT1H", "P1D"], 2160 title="Expiration Duration", 2161 ) 2162 request_authentication: Union[ 2163 SessionTokenRequestApiKeyAuthenticator, SessionTokenRequestBearerAuthenticator 2164 ] = Field( 2165 ..., 2166 description="Authentication method to use for requests sent to the API, specifying how to inject the session token.", 2167 title="Data Request Authentication", 2168 ) 2169 decoder: Optional[Union[JsonDecoder, XmlDecoder]] = Field( 2170 None, description="Component used to decode the response.", title="Decoder" 2171 ) 2172 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters")
login_requester: HttpRequester
request_authentication: Union[SessionTokenRequestApiKeyAuthenticator, SessionTokenRequestBearerAuthenticator]
decoder: Union[JsonDecoder, XmlDecoder, NoneType]
class
HttpRequester(pydantic.v1.main.BaseModel):
2175class HttpRequester(BaseModel): 2176 type: Literal["HttpRequester"] 2177 url_base: str = Field( 2178 ..., 2179 description="Base URL of the API source. Do not put sensitive information (e.g. API tokens) into this field - Use the Authentication component for this.", 2180 examples=[ 2181 "https://connect.squareup.com/v2", 2182 "{{ config['base_url'] or 'https://app.posthog.com'}}/api", 2183 "https://connect.squareup.com/v2/quotes/{{ stream_partition['id'] }}/quote_line_groups", 2184 "https://example.com/api/v1/resource/{{ next_page_token['id'] }}", 2185 ], 2186 title="API Base URL", 2187 ) 2188 path: Optional[str] = Field( 2189 None, 2190 description="Path the specific API endpoint that this stream represents. Do not put sensitive information (e.g. API tokens) into this field - Use the Authentication component for this.", 2191 examples=[ 2192 "/products", 2193 "/quotes/{{ stream_partition['id'] }}/quote_line_groups", 2194 "/trades/{{ config['symbol_id'] }}/history", 2195 ], 2196 title="URL Path", 2197 ) 2198 authenticator: Optional[ 2199 Union[ 2200 ApiKeyAuthenticator, 2201 BasicHttpAuthenticator, 2202 BearerAuthenticator, 2203 CustomAuthenticator, 2204 OAuthAuthenticator, 2205 JwtAuthenticator, 2206 NoAuth, 2207 SessionTokenAuthenticator, 2208 LegacySessionTokenAuthenticator, 2209 SelectiveAuthenticator, 2210 ] 2211 ] = Field( 2212 None, 2213 description="Authentication method to use for requests sent to the API.", 2214 title="Authenticator", 2215 ) 2216 error_handler: Optional[ 2217 Union[DefaultErrorHandler, CustomErrorHandler, CompositeErrorHandler] 2218 ] = Field( 2219 None, 2220 description="Error handler component that defines how to handle errors.", 2221 title="Error Handler", 2222 ) 2223 http_method: Optional[HttpMethod] = Field( 2224 HttpMethod.GET, 2225 description="The HTTP method used to fetch data from the source (can be GET or POST).", 2226 examples=["GET", "POST"], 2227 title="HTTP Method", 2228 ) 2229 request_body_data: Optional[Union[str, Dict[str, str]]] = Field( 2230 None, 2231 description="Specifies how to populate the body of the request with a non-JSON payload. Plain text will be sent as is, whereas objects will be converted to a urlencoded form.", 2232 examples=[ 2233 '[{"clause": {"type": "timestamp", "operator": 10, "parameters":\n [{"value": {{ stream_interval[\'start_time\'] | int * 1000 }} }]\n }, "orderBy": 1, "columnName": "Timestamp"}]/\n' 2234 ], 2235 title="Request Body Payload (Non-JSON)", 2236 ) 2237 request_body_json: Optional[Union[str, Dict[str, Any]]] = Field( 2238 None, 2239 description="Specifies how to populate the body of the request with a JSON payload. Can contain nested objects.", 2240 examples=[ 2241 {"sort_order": "ASC", "sort_field": "CREATED_AT"}, 2242 {"key": "{{ config['value'] }}"}, 2243 {"sort": {"field": "updated_at", "order": "ascending"}}, 2244 ], 2245 title="Request Body JSON Payload", 2246 ) 2247 request_headers: Optional[Union[str, Dict[str, str]]] = Field( 2248 None, 2249 description="Return any non-auth headers. Authentication headers will overwrite any overlapping headers returned from this method.", 2250 examples=[{"Output-Format": "JSON"}, {"Version": "{{ config['version'] }}"}], 2251 title="Request Headers", 2252 ) 2253 request_parameters: Optional[Union[str, Dict[str, Union[str, Any]]]] = Field( 2254 None, 2255 description="Specifies the query parameters that should be set on an outgoing HTTP request given the inputs.", 2256 examples=[ 2257 {"unit": "day"}, 2258 { 2259 "query": 'last_event_time BETWEEN TIMESTAMP "{{ stream_interval.start_time }}" AND TIMESTAMP "{{ stream_interval.end_time }}"' 2260 }, 2261 {"searchIn": "{{ ','.join(config.get('search_in', [])) }}"}, 2262 {"sort_by[asc]": "updated_at"}, 2263 ], 2264 title="Query Parameters", 2265 ) 2266 use_cache: Optional[bool] = Field( 2267 False, 2268 description="Enables stream requests caching. This field is automatically set by the CDK.", 2269 title="Use Cache", 2270 ) 2271 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters")
authenticator: Union[ApiKeyAuthenticator, BasicHttpAuthenticator, BearerAuthenticator, CustomAuthenticator, OAuthAuthenticator, JwtAuthenticator, NoAuth, SessionTokenAuthenticator, LegacySessionTokenAuthenticator, SelectiveAuthenticator, NoneType]
error_handler: Union[DefaultErrorHandler, CustomErrorHandler, CompositeErrorHandler, NoneType]
http_method: Optional[HttpMethod]
class
DynamicSchemaLoader(pydantic.v1.main.BaseModel):
2274class DynamicSchemaLoader(BaseModel): 2275 type: Literal["DynamicSchemaLoader"] 2276 retriever: Union[AsyncRetriever, CustomRetriever, SimpleRetriever] = Field( 2277 ..., 2278 description="Component used to coordinate how records are extracted across stream slices and request pages.", 2279 title="Retriever", 2280 ) 2281 schema_transformations: Optional[ 2282 List[ 2283 Union[ 2284 AddFields, 2285 CustomTransformation, 2286 RemoveFields, 2287 KeysToLower, 2288 KeysToSnakeCase, 2289 FlattenFields, 2290 DpathFlattenFields, 2291 KeysReplace, 2292 ] 2293 ] 2294 ] = Field( 2295 None, 2296 description="A list of transformations to be applied to the schema.", 2297 title="Schema Transformations", 2298 ) 2299 schema_type_identifier: SchemaTypeIdentifier 2300 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters")
retriever: Union[AsyncRetriever, CustomRetriever, SimpleRetriever]
schema_transformations: Optional[List[Union[AddFields, CustomTransformation, RemoveFields, KeysToLower, KeysToSnakeCase, FlattenFields, DpathFlattenFields, KeysReplace]]]
schema_type_identifier: SchemaTypeIdentifier
class
ParentStreamConfig(pydantic.v1.main.BaseModel):
2303class ParentStreamConfig(BaseModel): 2304 type: Literal["ParentStreamConfig"] 2305 lazy_read_pointer: Optional[List[str]] = Field( 2306 [], 2307 description="If set, this will enable lazy reading, using the initial read of parent records to extract child records.", 2308 title="Lazy Read Pointer", 2309 ) 2310 parent_key: str = Field( 2311 ..., 2312 description="The primary key of records from the parent stream that will be used during the retrieval of records for the current substream. This parent identifier field is typically a characteristic of the child records being extracted from the source API.", 2313 examples=["id", "{{ config['parent_record_id'] }}"], 2314 title="Parent Key", 2315 ) 2316 stream: Union[DeclarativeStream, StateDelegatingStream] = Field( 2317 ..., description="Reference to the parent stream.", title="Parent Stream" 2318 ) 2319 partition_field: str = Field( 2320 ..., 2321 description="While iterating over parent records during a sync, the parent_key value can be referenced by using this field.", 2322 examples=["parent_id", "{{ config['parent_partition_field'] }}"], 2323 title="Current Parent Key Value Identifier", 2324 ) 2325 request_option: Optional[RequestOption] = Field( 2326 None, 2327 description="A request option describing where the parent key value should be injected into and under what field name if applicable.", 2328 title="Request Option", 2329 ) 2330 incremental_dependency: Optional[bool] = Field( 2331 False, 2332 description="Indicates whether the parent stream should be read incrementally based on updates in the child stream.", 2333 title="Incremental Dependency", 2334 ) 2335 extra_fields: Optional[List[List[str]]] = Field( 2336 None, 2337 description="Array of field paths to include as additional fields in the stream slice. Each path is an array of strings representing keys to access fields in the respective parent record. Accessible via `stream_slice.extra_fields`. Missing fields are set to `None`.", 2338 title="Extra Fields", 2339 ) 2340 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters")
stream: Union[DeclarativeStream, StateDelegatingStream]
request_option: Optional[RequestOption]
class
PropertiesFromEndpoint(pydantic.v1.main.BaseModel):
2343class PropertiesFromEndpoint(BaseModel): 2344 type: Literal["PropertiesFromEndpoint"] 2345 property_field_path: List[str] = Field( 2346 ..., 2347 description="Describes the path to the field that should be extracted", 2348 examples=[["name"]], 2349 ) 2350 retriever: Union[CustomRetriever, SimpleRetriever] = Field( 2351 ..., 2352 description="Requester component that describes how to fetch the properties to query from a remote API endpoint.", 2353 ) 2354 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters")
retriever: Union[CustomRetriever, SimpleRetriever]
class
QueryProperties(pydantic.v1.main.BaseModel):
2357class QueryProperties(BaseModel): 2358 type: Literal["QueryProperties"] 2359 property_list: Union[List[str], PropertiesFromEndpoint] = Field( 2360 ..., 2361 description="The set of properties that will be queried for in the outbound request. This can either be statically defined or dynamic based on an API endpoint", 2362 title="Property List", 2363 ) 2364 always_include_properties: Optional[List[str]] = Field( 2365 None, 2366 description="The list of properties that should be included in every set of properties when multiple chunks of properties are being requested.", 2367 title="Always Include Properties", 2368 ) 2369 property_chunking: Optional[PropertyChunking] = Field( 2370 None, 2371 description="Defines how query properties will be grouped into smaller sets for APIs with limitations on the number of properties fetched per API request.", 2372 title="Property Chunking", 2373 ) 2374 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters")
property_list: Union[List[str], PropertiesFromEndpoint]
property_chunking: Optional[PropertyChunking]
class
StateDelegatingStream(pydantic.v1.main.BaseModel):
2377class StateDelegatingStream(BaseModel): 2378 type: Literal["StateDelegatingStream"] 2379 name: str = Field(..., description="The stream name.", example=["Users"], title="Name") 2380 full_refresh_stream: DeclarativeStream = Field( 2381 ..., 2382 description="Component used to coordinate how records are extracted across stream slices and request pages when the state is empty or not provided.", 2383 title="Retriever", 2384 ) 2385 incremental_stream: DeclarativeStream = Field( 2386 ..., 2387 description="Component used to coordinate how records are extracted across stream slices and request pages when the state provided.", 2388 title="Retriever", 2389 ) 2390 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters")
full_refresh_stream: DeclarativeStream
incremental_stream: DeclarativeStream
class
SimpleRetriever(pydantic.v1.main.BaseModel):
2393class SimpleRetriever(BaseModel): 2394 type: Literal["SimpleRetriever"] 2395 record_selector: RecordSelector = Field( 2396 ..., 2397 description="Component that describes how to extract records from a HTTP response.", 2398 ) 2399 requester: Union[CustomRequester, HttpRequester] = Field( 2400 ..., 2401 description="Requester component that describes how to prepare HTTP requests to send to the source API.", 2402 ) 2403 paginator: Optional[Union[DefaultPaginator, NoPagination]] = Field( 2404 None, 2405 description="Paginator component that describes how to navigate through the API's pages.", 2406 ) 2407 ignore_stream_slicer_parameters_on_paginated_requests: Optional[bool] = Field( 2408 False, 2409 description="If true, the partition router and incremental request options will be ignored when paginating requests. Request options set directly on the requester will not be ignored.", 2410 ) 2411 partition_router: Optional[ 2412 Union[ 2413 CustomPartitionRouter, 2414 ListPartitionRouter, 2415 SubstreamPartitionRouter, 2416 GroupingPartitionRouter, 2417 List[ 2418 Union[ 2419 CustomPartitionRouter, 2420 ListPartitionRouter, 2421 SubstreamPartitionRouter, 2422 GroupingPartitionRouter, 2423 ] 2424 ], 2425 ] 2426 ] = Field( 2427 [], 2428 description="PartitionRouter component that describes how to partition the stream, enabling incremental syncs and checkpointing.", 2429 title="Partition Router", 2430 ) 2431 decoder: Optional[ 2432 Union[ 2433 CustomDecoder, 2434 CsvDecoder, 2435 GzipDecoder, 2436 JsonDecoder, 2437 JsonlDecoder, 2438 IterableDecoder, 2439 XmlDecoder, 2440 ZipfileDecoder, 2441 ] 2442 ] = Field( 2443 None, 2444 description="Component decoding the response so records can be extracted.", 2445 title="Decoder", 2446 ) 2447 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters")
record_selector: RecordSelector
requester: Union[CustomRequester, HttpRequester]
paginator: Union[DefaultPaginator, NoPagination, NoneType]
partition_router: Union[CustomPartitionRouter, ListPartitionRouter, SubstreamPartitionRouter, GroupingPartitionRouter, List[Union[CustomPartitionRouter, ListPartitionRouter, SubstreamPartitionRouter, GroupingPartitionRouter]], NoneType]
decoder: Union[CustomDecoder, CsvDecoder, GzipDecoder, JsonDecoder, JsonlDecoder, IterableDecoder, XmlDecoder, ZipfileDecoder, NoneType]
class
AsyncRetriever(pydantic.v1.main.BaseModel):
2450class AsyncRetriever(BaseModel): 2451 type: Literal["AsyncRetriever"] 2452 record_selector: RecordSelector = Field( 2453 ..., 2454 description="Component that describes how to extract records from a HTTP response.", 2455 ) 2456 status_mapping: AsyncJobStatusMap = Field( 2457 ..., description="Async Job Status to Airbyte CDK Async Job Status mapping." 2458 ) 2459 status_extractor: Union[CustomRecordExtractor, DpathExtractor] = Field( 2460 ..., description="Responsible for fetching the actual status of the async job." 2461 ) 2462 download_target_extractor: Union[CustomRecordExtractor, DpathExtractor] = Field( 2463 ..., 2464 description="Responsible for fetching the final result `urls` provided by the completed / finished / ready async job.", 2465 ) 2466 download_extractor: Optional[ 2467 Union[CustomRecordExtractor, DpathExtractor, ResponseToFileExtractor] 2468 ] = Field(None, description="Responsible for fetching the records from provided urls.") 2469 creation_requester: Union[CustomRequester, HttpRequester] = Field( 2470 ..., 2471 description="Requester component that describes how to prepare HTTP requests to send to the source API to create the async server-side job.", 2472 ) 2473 polling_requester: Union[CustomRequester, HttpRequester] = Field( 2474 ..., 2475 description="Requester component that describes how to prepare HTTP requests to send to the source API to fetch the status of the running async job.", 2476 ) 2477 polling_job_timeout: Optional[Union[int, str]] = Field( 2478 None, 2479 description="The time in minutes after which the single Async Job should be considered as Timed Out.", 2480 ) 2481 download_target_requester: Optional[Union[CustomRequester, HttpRequester]] = Field( 2482 None, 2483 description="Requester component that describes how to prepare HTTP requests to send to the source API to extract the url from polling response by the completed async job.", 2484 ) 2485 download_requester: Union[CustomRequester, HttpRequester] = Field( 2486 ..., 2487 description="Requester component that describes how to prepare HTTP requests to send to the source API to download the data provided by the completed async job.", 2488 ) 2489 download_paginator: Optional[Union[DefaultPaginator, NoPagination]] = Field( 2490 None, 2491 description="Paginator component that describes how to navigate through the API's pages during download.", 2492 ) 2493 abort_requester: Optional[Union[CustomRequester, HttpRequester]] = Field( 2494 None, 2495 description="Requester component that describes how to prepare HTTP requests to send to the source API to abort a job once it is timed out from the source's perspective.", 2496 ) 2497 delete_requester: Optional[Union[CustomRequester, HttpRequester]] = Field( 2498 None, 2499 description="Requester component that describes how to prepare HTTP requests to send to the source API to delete a job once the records are extracted.", 2500 ) 2501 partition_router: Optional[ 2502 Union[ 2503 CustomPartitionRouter, 2504 ListPartitionRouter, 2505 SubstreamPartitionRouter, 2506 GroupingPartitionRouter, 2507 List[ 2508 Union[ 2509 CustomPartitionRouter, 2510 ListPartitionRouter, 2511 SubstreamPartitionRouter, 2512 GroupingPartitionRouter, 2513 ] 2514 ], 2515 ] 2516 ] = Field( 2517 [], 2518 description="PartitionRouter component that describes how to partition the stream, enabling incremental syncs and checkpointing.", 2519 title="Partition Router", 2520 ) 2521 decoder: Optional[ 2522 Union[ 2523 CustomDecoder, 2524 CsvDecoder, 2525 GzipDecoder, 2526 JsonDecoder, 2527 JsonlDecoder, 2528 IterableDecoder, 2529 XmlDecoder, 2530 ZipfileDecoder, 2531 ] 2532 ] = Field( 2533 None, 2534 description="Component decoding the response so records can be extracted.", 2535 title="Decoder", 2536 ) 2537 download_decoder: Optional[ 2538 Union[ 2539 CustomDecoder, 2540 CsvDecoder, 2541 GzipDecoder, 2542 JsonDecoder, 2543 JsonlDecoder, 2544 IterableDecoder, 2545 XmlDecoder, 2546 ZipfileDecoder, 2547 ] 2548 ] = Field( 2549 None, 2550 description="Component decoding the download response so records can be extracted.", 2551 title="Download Decoder", 2552 ) 2553 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters")
record_selector: RecordSelector
status_mapping: AsyncJobStatusMap
status_extractor: Union[CustomRecordExtractor, DpathExtractor]
download_target_extractor: Union[CustomRecordExtractor, DpathExtractor]
download_extractor: Union[CustomRecordExtractor, DpathExtractor, ResponseToFileExtractor, NoneType]
creation_requester: Union[CustomRequester, HttpRequester]
polling_requester: Union[CustomRequester, HttpRequester]
download_target_requester: Union[CustomRequester, HttpRequester, NoneType]
download_requester: Union[CustomRequester, HttpRequester]
download_paginator: Union[DefaultPaginator, NoPagination, NoneType]
abort_requester: Union[CustomRequester, HttpRequester, NoneType]
delete_requester: Union[CustomRequester, HttpRequester, NoneType]
partition_router: Union[CustomPartitionRouter, ListPartitionRouter, SubstreamPartitionRouter, GroupingPartitionRouter, List[Union[CustomPartitionRouter, ListPartitionRouter, SubstreamPartitionRouter, GroupingPartitionRouter]], NoneType]
decoder: Union[CustomDecoder, CsvDecoder, GzipDecoder, JsonDecoder, JsonlDecoder, IterableDecoder, XmlDecoder, ZipfileDecoder, NoneType]
download_decoder: Union[CustomDecoder, CsvDecoder, GzipDecoder, JsonDecoder, JsonlDecoder, IterableDecoder, XmlDecoder, ZipfileDecoder, NoneType]
class
SubstreamPartitionRouter(pydantic.v1.main.BaseModel):
2556class SubstreamPartitionRouter(BaseModel): 2557 type: Literal["SubstreamPartitionRouter"] 2558 parent_stream_configs: List[ParentStreamConfig] = Field( 2559 ..., 2560 description="Specifies which parent streams are being iterated over and how parent records should be used to partition the child stream data set.", 2561 title="Parent Stream Configs", 2562 ) 2563 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters")
parent_stream_configs: List[ParentStreamConfig]
class
GroupingPartitionRouter(pydantic.v1.main.BaseModel):
2566class GroupingPartitionRouter(BaseModel): 2567 type: Literal["GroupingPartitionRouter"] 2568 group_size: int = Field( 2569 ..., 2570 description="The number of partitions to include in each group. This determines how many partition values are batched together in a single slice.", 2571 examples=[10, 50], 2572 title="Group Size", 2573 ) 2574 underlying_partition_router: Union[ 2575 CustomPartitionRouter, ListPartitionRouter, SubstreamPartitionRouter 2576 ] = Field( 2577 ..., 2578 description="The partition router whose output will be grouped. This can be any valid partition router component.", 2579 title="Underlying Partition Router", 2580 ) 2581 deduplicate: Optional[bool] = Field( 2582 True, 2583 description="If true, ensures that partitions are unique within each group by removing duplicates based on the partition key.", 2584 title="Deduplicate Partitions", 2585 ) 2586 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters")
underlying_partition_router: Union[CustomPartitionRouter, ListPartitionRouter, SubstreamPartitionRouter]
class
HttpComponentsResolver(pydantic.v1.main.BaseModel):
2589class HttpComponentsResolver(BaseModel): 2590 type: Literal["HttpComponentsResolver"] 2591 retriever: Union[AsyncRetriever, CustomRetriever, SimpleRetriever] = Field( 2592 ..., 2593 description="Component used to coordinate how records are extracted across stream slices and request pages.", 2594 title="Retriever", 2595 ) 2596 components_mapping: List[ComponentMappingDefinition] 2597 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters")
retriever: Union[AsyncRetriever, CustomRetriever, SimpleRetriever]
components_mapping: List[ComponentMappingDefinition]
class
DynamicDeclarativeStream(pydantic.v1.main.BaseModel):
2600class DynamicDeclarativeStream(BaseModel): 2601 type: Literal["DynamicDeclarativeStream"] 2602 name: Optional[str] = Field( 2603 "", description="The dynamic stream name.", example=["Tables"], title="Name" 2604 ) 2605 stream_template: DeclarativeStream = Field( 2606 ..., description="Reference to the stream template.", title="Stream Template" 2607 ) 2608 components_resolver: Union[HttpComponentsResolver, ConfigComponentsResolver] = Field( 2609 ..., 2610 description="Component resolve and populates stream templates with components values.", 2611 title="Components Resolver", 2612 )
stream_template: DeclarativeStream
components_resolver: Union[HttpComponentsResolver, ConfigComponentsResolver]