airbyte_cdk.sources.declarative.models.declarative_component_schema
1# generated by datamodel-codegen: 2# filename: declarative_component_schema.yaml 3 4from __future__ import annotations 5 6from enum import Enum 7from typing import Any, Dict, List, Literal, Optional, Union 8 9from pydantic.v1 import BaseModel, Extra, Field 10 11 12class AuthFlowType(Enum): 13 oauth2_0 = "oauth2.0" 14 oauth1_0 = "oauth1.0" 15 16 17class BasicHttpAuthenticator(BaseModel): 18 type: Literal["BasicHttpAuthenticator"] 19 username: str = Field( 20 ..., 21 description="The username that will be combined with the password, base64 encoded and used to make requests. Fill it in the user inputs.", 22 examples=["{{ config['username'] }}", "{{ config['api_key'] }}"], 23 title="Username", 24 ) 25 password: Optional[str] = Field( 26 "", 27 description="The password that will be combined with the username, base64 encoded and used to make requests. Fill it in the user inputs.", 28 examples=["{{ config['password'] }}", ""], 29 title="Password", 30 ) 31 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters") 32 33 34class BearerAuthenticator(BaseModel): 35 type: Literal["BearerAuthenticator"] 36 api_token: str = Field( 37 ..., 38 description="Token to inject as request header for authenticating with the API.", 39 examples=["{{ config['api_key'] }}", "{{ config['token'] }}"], 40 title="Bearer Token", 41 ) 42 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters") 43 44 45class CheckStream(BaseModel): 46 type: Literal["CheckStream"] 47 stream_names: List[str] = Field( 48 ..., 49 description="Names of the streams to try reading from when running a check operation.", 50 examples=[["users"], ["users", "contacts"]], 51 title="Stream Names", 52 ) 53 54 55class CheckDynamicStream(BaseModel): 56 type: Literal["CheckDynamicStream"] 57 stream_count: int = Field( 58 ..., 59 description="Numbers of the streams to try reading from when running a check operation.", 60 title="Stream Count", 61 ) 62 use_check_availability: Optional[bool] = Field( 63 True, 64 description="Enables stream check availability. This field is automatically set by the CDK.", 65 title="Use Check Availability", 66 ) 67 68 69class ConcurrencyLevel(BaseModel): 70 type: Optional[Literal["ConcurrencyLevel"]] = None 71 default_concurrency: Union[int, str] = Field( 72 ..., 73 description="The amount of concurrency that will applied during a sync. This value can be hardcoded or user-defined in the config if different users have varying volume thresholds in the target API.", 74 examples=[10, "{{ config['num_workers'] or 10 }}"], 75 title="Default Concurrency", 76 ) 77 max_concurrency: Optional[int] = Field( 78 None, 79 description="The maximum level of concurrency that will be used during a sync. This becomes a required field when the default_concurrency derives from the config, because it serves as a safeguard against a user-defined threshold that is too high.", 80 examples=[20, 100], 81 title="Max Concurrency", 82 ) 83 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters") 84 85 86class ConstantBackoffStrategy(BaseModel): 87 type: Literal["ConstantBackoffStrategy"] 88 backoff_time_in_seconds: Union[float, str] = Field( 89 ..., 90 description="Backoff time in seconds.", 91 examples=[30, 30.5, "{{ config['backoff_time'] }}"], 92 title="Backoff Time", 93 ) 94 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters") 95 96 97class CursorPagination(BaseModel): 98 type: Literal["CursorPagination"] 99 cursor_value: str = Field( 100 ..., 101 description="Value of the cursor defining the next page to fetch.", 102 examples=[ 103 "{{ headers.link.next.cursor }}", 104 "{{ last_record['key'] }}", 105 "{{ response['nextPage'] }}", 106 ], 107 title="Cursor Value", 108 ) 109 page_size: Optional[int] = Field( 110 None, 111 description="The number of records to include in each pages.", 112 examples=[100], 113 title="Page Size", 114 ) 115 stop_condition: Optional[str] = Field( 116 None, 117 description="Template string evaluating when to stop paginating.", 118 examples=[ 119 "{{ response.data.has_more is false }}", 120 "{{ 'next' not in headers['link'] }}", 121 ], 122 title="Stop Condition", 123 ) 124 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters") 125 126 127class CustomAuthenticator(BaseModel): 128 class Config: 129 extra = Extra.allow 130 131 type: Literal["CustomAuthenticator"] 132 class_name: str = Field( 133 ..., 134 description="Fully-qualified name of the class that will be implementing the custom authentication strategy. Has to be a sub class of DeclarativeAuthenticator. The format is `source_<name>.<package>.<class_name>`.", 135 examples=["source_railz.components.ShortLivedTokenAuthenticator"], 136 title="Class Name", 137 ) 138 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters") 139 140 141class CustomBackoffStrategy(BaseModel): 142 class Config: 143 extra = Extra.allow 144 145 type: Literal["CustomBackoffStrategy"] 146 class_name: str = Field( 147 ..., 148 description="Fully-qualified name of the class that will be implementing the custom backoff strategy. The format is `source_<name>.<package>.<class_name>`.", 149 examples=["source_railz.components.MyCustomBackoffStrategy"], 150 title="Class Name", 151 ) 152 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters") 153 154 155class CustomErrorHandler(BaseModel): 156 class Config: 157 extra = Extra.allow 158 159 type: Literal["CustomErrorHandler"] 160 class_name: str = Field( 161 ..., 162 description="Fully-qualified name of the class that will be implementing the custom error handler. The format is `source_<name>.<package>.<class_name>`.", 163 examples=["source_railz.components.MyCustomErrorHandler"], 164 title="Class Name", 165 ) 166 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters") 167 168 169class CustomIncrementalSync(BaseModel): 170 class Config: 171 extra = Extra.allow 172 173 type: Literal["CustomIncrementalSync"] 174 class_name: str = Field( 175 ..., 176 description="Fully-qualified name of the class that will be implementing the custom incremental sync. The format is `source_<name>.<package>.<class_name>`.", 177 examples=["source_railz.components.MyCustomIncrementalSync"], 178 title="Class Name", 179 ) 180 cursor_field: str = Field( 181 ..., 182 description="The location of the value on a record that will be used as a bookmark during sync.", 183 ) 184 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters") 185 186 187class CustomPaginationStrategy(BaseModel): 188 class Config: 189 extra = Extra.allow 190 191 type: Literal["CustomPaginationStrategy"] 192 class_name: str = Field( 193 ..., 194 description="Fully-qualified name of the class that will be implementing the custom pagination strategy. The format is `source_<name>.<package>.<class_name>`.", 195 examples=["source_railz.components.MyCustomPaginationStrategy"], 196 title="Class Name", 197 ) 198 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters") 199 200 201class CustomRecordExtractor(BaseModel): 202 class Config: 203 extra = Extra.allow 204 205 type: Literal["CustomRecordExtractor"] 206 class_name: str = Field( 207 ..., 208 description="Fully-qualified name of the class that will be implementing the custom record extraction strategy. The format is `source_<name>.<package>.<class_name>`.", 209 examples=["source_railz.components.MyCustomRecordExtractor"], 210 title="Class Name", 211 ) 212 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters") 213 214 215class CustomRecordFilter(BaseModel): 216 class Config: 217 extra = Extra.allow 218 219 type: Literal["CustomRecordFilter"] 220 class_name: str = Field( 221 ..., 222 description="Fully-qualified name of the class that will be implementing the custom record filter strategy. The format is `source_<name>.<package>.<class_name>`.", 223 examples=["source_railz.components.MyCustomCustomRecordFilter"], 224 title="Class Name", 225 ) 226 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters") 227 228 229class CustomRequester(BaseModel): 230 class Config: 231 extra = Extra.allow 232 233 type: Literal["CustomRequester"] 234 class_name: str = Field( 235 ..., 236 description="Fully-qualified name of the class that will be implementing the custom requester strategy. The format is `source_<name>.<package>.<class_name>`.", 237 examples=["source_railz.components.MyCustomRecordExtractor"], 238 title="Class Name", 239 ) 240 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters") 241 242 243class CustomRetriever(BaseModel): 244 class Config: 245 extra = Extra.allow 246 247 type: Literal["CustomRetriever"] 248 class_name: str = Field( 249 ..., 250 description="Fully-qualified name of the class that will be implementing the custom retriever strategy. The format is `source_<name>.<package>.<class_name>`.", 251 examples=["source_railz.components.MyCustomRetriever"], 252 title="Class Name", 253 ) 254 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters") 255 256 257class CustomPartitionRouter(BaseModel): 258 class Config: 259 extra = Extra.allow 260 261 type: Literal["CustomPartitionRouter"] 262 class_name: str = Field( 263 ..., 264 description="Fully-qualified name of the class that will be implementing the custom partition router. The format is `source_<name>.<package>.<class_name>`.", 265 examples=["source_railz.components.MyCustomPartitionRouter"], 266 title="Class Name", 267 ) 268 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters") 269 270 271class CustomSchemaLoader(BaseModel): 272 class Config: 273 extra = Extra.allow 274 275 type: Literal["CustomSchemaLoader"] 276 class_name: str = Field( 277 ..., 278 description="Fully-qualified name of the class that will be implementing the custom schema loader. The format is `source_<name>.<package>.<class_name>`.", 279 examples=["source_railz.components.MyCustomSchemaLoader"], 280 title="Class Name", 281 ) 282 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters") 283 284 285class CustomSchemaNormalization(BaseModel): 286 class Config: 287 extra = Extra.allow 288 289 type: Literal["CustomSchemaNormalization"] 290 class_name: str = Field( 291 ..., 292 description="Fully-qualified name of the class that will be implementing the custom normalization. The format is `source_<name>.<package>.<class_name>`.", 293 examples=[ 294 "source_amazon_seller_partner.components.LedgerDetailedViewReportsTypeTransformer" 295 ], 296 title="Class Name", 297 ) 298 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters") 299 300 301class CustomStateMigration(BaseModel): 302 class Config: 303 extra = Extra.allow 304 305 type: Literal["CustomStateMigration"] 306 class_name: str = Field( 307 ..., 308 description="Fully-qualified name of the class that will be implementing the custom state migration. The format is `source_<name>.<package>.<class_name>`.", 309 examples=["source_railz.components.MyCustomStateMigration"], 310 title="Class Name", 311 ) 312 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters") 313 314 315class CustomTransformation(BaseModel): 316 class Config: 317 extra = Extra.allow 318 319 type: Literal["CustomTransformation"] 320 class_name: str = Field( 321 ..., 322 description="Fully-qualified name of the class that will be implementing the custom transformation. The format is `source_<name>.<package>.<class_name>`.", 323 examples=["source_railz.components.MyCustomTransformation"], 324 title="Class Name", 325 ) 326 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters") 327 328 329class LegacyToPerPartitionStateMigration(BaseModel): 330 class Config: 331 extra = Extra.allow 332 333 type: Optional[Literal["LegacyToPerPartitionStateMigration"]] = None 334 335 336class Clamping(BaseModel): 337 target: str = Field( 338 ..., 339 description="The period of time that datetime windows will be clamped by", 340 examples=["DAY", "WEEK", "MONTH", "{{ config['target'] }}"], 341 title="Target", 342 ) 343 target_details: Optional[Dict[str, Any]] = None 344 345 346class Algorithm(Enum): 347 HS256 = "HS256" 348 HS384 = "HS384" 349 HS512 = "HS512" 350 ES256 = "ES256" 351 ES256K = "ES256K" 352 ES384 = "ES384" 353 ES512 = "ES512" 354 RS256 = "RS256" 355 RS384 = "RS384" 356 RS512 = "RS512" 357 PS256 = "PS256" 358 PS384 = "PS384" 359 PS512 = "PS512" 360 EdDSA = "EdDSA" 361 362 363class JwtHeaders(BaseModel): 364 class Config: 365 extra = Extra.forbid 366 367 kid: Optional[str] = Field( 368 None, 369 description="Private key ID for user account.", 370 examples=["{{ config['kid'] }}"], 371 title="Key Identifier", 372 ) 373 typ: Optional[str] = Field( 374 "JWT", 375 description="The media type of the complete JWT.", 376 examples=["JWT"], 377 title="Type", 378 ) 379 cty: Optional[str] = Field( 380 None, 381 description="Content type of JWT header.", 382 examples=["JWT"], 383 title="Content Type", 384 ) 385 386 387class JwtPayload(BaseModel): 388 class Config: 389 extra = Extra.forbid 390 391 iss: Optional[str] = Field( 392 None, 393 description="The user/principal that issued the JWT. Commonly a value unique to the user.", 394 examples=["{{ config['iss'] }}"], 395 title="Issuer", 396 ) 397 sub: Optional[str] = Field( 398 None, 399 description="The subject of the JWT. Commonly defined by the API.", 400 title="Subject", 401 ) 402 aud: Optional[str] = Field( 403 None, 404 description="The recipient that the JWT is intended for. Commonly defined by the API.", 405 examples=["appstoreconnect-v1"], 406 title="Audience", 407 ) 408 409 410class JwtAuthenticator(BaseModel): 411 type: Literal["JwtAuthenticator"] 412 secret_key: str = Field( 413 ..., 414 description="Secret used to sign the JSON web token.", 415 examples=["{{ config['secret_key'] }}"], 416 ) 417 base64_encode_secret_key: Optional[bool] = Field( 418 False, 419 description='When set to true, the secret key will be base64 encoded prior to being encoded as part of the JWT. Only set to "true" when required by the API.', 420 ) 421 algorithm: Algorithm = Field( 422 ..., 423 description="Algorithm used to sign the JSON web token.", 424 examples=["ES256", "HS256", "RS256", "{{ config['algorithm'] }}"], 425 ) 426 token_duration: Optional[int] = Field( 427 1200, 428 description="The amount of time in seconds a JWT token can be valid after being issued.", 429 examples=[1200, 3600], 430 title="Token Duration", 431 ) 432 header_prefix: Optional[str] = Field( 433 None, 434 description="The prefix to be used within the Authentication header.", 435 examples=["Bearer", "Basic"], 436 title="Header Prefix", 437 ) 438 jwt_headers: Optional[JwtHeaders] = Field( 439 None, 440 description="JWT headers used when signing JSON web token.", 441 title="JWT Headers", 442 ) 443 additional_jwt_headers: Optional[Dict[str, Any]] = Field( 444 None, 445 description="Additional headers to be included with the JWT headers object.", 446 title="Additional JWT Headers", 447 ) 448 jwt_payload: Optional[JwtPayload] = Field( 449 None, 450 description="JWT Payload used when signing JSON web token.", 451 title="JWT Payload", 452 ) 453 additional_jwt_payload: Optional[Dict[str, Any]] = Field( 454 None, 455 description="Additional properties to be added to the JWT payload.", 456 title="Additional JWT Payload Properties", 457 ) 458 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters") 459 460 461class RefreshTokenUpdater(BaseModel): 462 refresh_token_name: Optional[str] = Field( 463 "refresh_token", 464 description="The name of the property which contains the updated refresh token in the response from the token refresh endpoint.", 465 examples=["refresh_token"], 466 title="Refresh Token Property Name", 467 ) 468 access_token_config_path: Optional[List[str]] = Field( 469 ["credentials", "access_token"], 470 description="Config path to the access token. Make sure the field actually exists in the config.", 471 examples=[["credentials", "access_token"], ["access_token"]], 472 title="Config Path To Access Token", 473 ) 474 refresh_token_config_path: Optional[List[str]] = Field( 475 ["credentials", "refresh_token"], 476 description="Config path to the access token. Make sure the field actually exists in the config.", 477 examples=[["credentials", "refresh_token"], ["refresh_token"]], 478 title="Config Path To Refresh Token", 479 ) 480 token_expiry_date_config_path: Optional[List[str]] = Field( 481 ["credentials", "token_expiry_date"], 482 description="Config path to the expiry date. Make sure actually exists in the config.", 483 examples=[["credentials", "token_expiry_date"]], 484 title="Config Path To Expiry Date", 485 ) 486 refresh_token_error_status_codes: Optional[List[int]] = Field( 487 [], 488 description="Status Codes to Identify refresh token error in response (Refresh Token Error Key and Refresh Token Error Values should be also specified). Responses with one of the error status code and containing an error value will be flagged as a config error", 489 examples=[[400, 500]], 490 title="Refresh Token Error Status Codes", 491 ) 492 refresh_token_error_key: Optional[str] = Field( 493 "", 494 description="Key to Identify refresh token error in response (Refresh Token Error Status Codes and Refresh Token Error Values should be also specified).", 495 examples=["error"], 496 title="Refresh Token Error Key", 497 ) 498 refresh_token_error_values: Optional[List[str]] = Field( 499 [], 500 description='List of values to check for exception during token refresh process. Used to check if the error found in the response matches the key from the Refresh Token Error Key field (e.g. response={"error": "invalid_grant"}). Only responses with one of the error status code and containing an error value will be flagged as a config error', 501 examples=[["invalid_grant", "invalid_permissions"]], 502 title="Refresh Token Error Values", 503 ) 504 505 506class OAuthAuthenticator(BaseModel): 507 type: Literal["OAuthAuthenticator"] 508 client_id_name: Optional[str] = Field( 509 "client_id", 510 description="The name of the property to use to refresh the `access_token`.", 511 examples=["custom_app_id"], 512 title="Client ID Property Name", 513 ) 514 client_id: Optional[str] = Field( 515 None, 516 description="The OAuth client ID. Fill it in the user inputs.", 517 examples=["{{ config['client_id }}", "{{ config['credentials']['client_id }}"], 518 title="Client ID", 519 ) 520 client_secret_name: Optional[str] = Field( 521 "client_secret", 522 description="The name of the property to use to refresh the `access_token`.", 523 examples=["custom_app_secret"], 524 title="Client Secret Property Name", 525 ) 526 client_secret: Optional[str] = Field( 527 None, 528 description="The OAuth client secret. Fill it in the user inputs.", 529 examples=[ 530 "{{ config['client_secret }}", 531 "{{ config['credentials']['client_secret }}", 532 ], 533 title="Client Secret", 534 ) 535 refresh_token_name: Optional[str] = Field( 536 "refresh_token", 537 description="The name of the property to use to refresh the `access_token`.", 538 examples=["custom_app_refresh_value"], 539 title="Refresh Token Property Name", 540 ) 541 refresh_token: Optional[str] = Field( 542 None, 543 description="Credential artifact used to get a new access token.", 544 examples=[ 545 "{{ config['refresh_token'] }}", 546 "{{ config['credentials]['refresh_token'] }}", 547 ], 548 title="Refresh Token", 549 ) 550 token_refresh_endpoint: Optional[str] = Field( 551 None, 552 description="The full URL to call to obtain a new access token.", 553 examples=["https://connect.squareup.com/oauth2/token"], 554 title="Token Refresh Endpoint", 555 ) 556 access_token_name: Optional[str] = Field( 557 "access_token", 558 description="The name of the property which contains the access token in the response from the token refresh endpoint.", 559 examples=["access_token"], 560 title="Access Token Property Name", 561 ) 562 access_token_value: Optional[str] = Field( 563 None, 564 description="The value of the access_token to bypass the token refreshing using `refresh_token`.", 565 examples=["secret_access_token_value"], 566 title="Access Token Value", 567 ) 568 expires_in_name: Optional[str] = Field( 569 "expires_in", 570 description="The name of the property which contains the expiry date in the response from the token refresh endpoint.", 571 examples=["expires_in"], 572 title="Token Expiry Property Name", 573 ) 574 grant_type_name: Optional[str] = Field( 575 "grant_type", 576 description="The name of the property to use to refresh the `access_token`.", 577 examples=["custom_grant_type"], 578 title="Grant Type Property Name", 579 ) 580 grant_type: Optional[str] = Field( 581 "refresh_token", 582 description="Specifies the OAuth2 grant type. If set to refresh_token, the refresh_token needs to be provided as well. For client_credentials, only client id and secret are required. Other grant types are not officially supported.", 583 examples=["refresh_token", "client_credentials"], 584 title="Grant Type", 585 ) 586 refresh_request_body: Optional[Dict[str, Any]] = Field( 587 None, 588 description="Body of the request sent to get a new access token.", 589 examples=[ 590 { 591 "applicationId": "{{ config['application_id'] }}", 592 "applicationSecret": "{{ config['application_secret'] }}", 593 "token": "{{ config['token'] }}", 594 } 595 ], 596 title="Refresh Request Body", 597 ) 598 refresh_request_headers: Optional[Dict[str, Any]] = Field( 599 None, 600 description="Headers of the request sent to get a new access token.", 601 examples=[ 602 { 603 "Authorization": "<AUTH_TOKEN>", 604 "Content-Type": "application/x-www-form-urlencoded", 605 } 606 ], 607 title="Refresh Request Headers", 608 ) 609 scopes: Optional[List[str]] = Field( 610 None, 611 description="List of scopes that should be granted to the access token.", 612 examples=[["crm.list.read", "crm.objects.contacts.read", "crm.schema.contacts.read"]], 613 title="Scopes", 614 ) 615 token_expiry_date: Optional[str] = Field( 616 None, 617 description="The access token expiry date.", 618 examples=["2023-04-06T07:12:10.421833+00:00", 1680842386], 619 title="Token Expiry Date", 620 ) 621 token_expiry_date_format: Optional[str] = Field( 622 None, 623 description="The format of the time to expiration datetime. Provide it if the time is returned as a date-time string instead of seconds.", 624 examples=["%Y-%m-%d %H:%M:%S.%f+00:00"], 625 title="Token Expiry Date Format", 626 ) 627 refresh_token_updater: Optional[RefreshTokenUpdater] = Field( 628 None, 629 description="When the token updater is defined, new refresh tokens, access tokens and the access token expiry date are written back from the authentication response to the config object. This is important if the refresh token can only used once.", 630 title="Token Updater", 631 ) 632 profile_assertion: Optional[JwtAuthenticator] = Field( 633 None, 634 description="The authenticator being used to authenticate the client authenticator.", 635 title="Profile Assertion", 636 ) 637 use_profile_assertion: Optional[bool] = Field( 638 False, 639 description="Enable using profile assertion as a flow for OAuth authorization.", 640 title="Use Profile Assertion", 641 ) 642 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters") 643 644 645class Rate(BaseModel): 646 class Config: 647 extra = Extra.allow 648 649 limit: Union[int, str] = Field( 650 ..., 651 description="The maximum number of calls allowed within the interval.", 652 title="Limit", 653 ) 654 interval: str = Field( 655 ..., 656 description="The time interval for the rate limit.", 657 examples=["PT1H", "P1D"], 658 title="Interval", 659 ) 660 661 662class HttpRequestRegexMatcher(BaseModel): 663 class Config: 664 extra = Extra.allow 665 666 method: Optional[str] = Field( 667 None, description="The HTTP method to match (e.g., GET, POST).", title="Method" 668 ) 669 url_base: Optional[str] = Field( 670 None, 671 description='The base URL (scheme and host, e.g. "https://api.example.com") to match.', 672 title="URL Base", 673 ) 674 url_path_pattern: Optional[str] = Field( 675 None, 676 description="A regular expression pattern to match the URL path.", 677 title="URL Path Pattern", 678 ) 679 params: Optional[Dict[str, Any]] = Field( 680 None, description="The query parameters to match.", title="Parameters" 681 ) 682 headers: Optional[Dict[str, Any]] = Field( 683 None, description="The headers to match.", title="Headers" 684 ) 685 686 687class DpathExtractor(BaseModel): 688 type: Literal["DpathExtractor"] 689 field_path: List[str] = Field( 690 ..., 691 description='List of potentially nested fields describing the full path of the field to extract. Use "*" to extract all values from an array. See more info in the [docs](https://docs.airbyte.com/connector-development/config-based/understanding-the-yaml-file/record-selector).', 692 examples=[ 693 ["data"], 694 ["data", "records"], 695 ["data", "{{ parameters.name }}"], 696 ["data", "*", "record"], 697 ], 698 title="Field Path", 699 ) 700 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters") 701 702 703class ResponseToFileExtractor(BaseModel): 704 type: Literal["ResponseToFileExtractor"] 705 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters") 706 707 708class ExponentialBackoffStrategy(BaseModel): 709 type: Literal["ExponentialBackoffStrategy"] 710 factor: Optional[Union[float, str]] = Field( 711 5, 712 description="Multiplicative constant applied on each retry.", 713 examples=[5, 5.5, "10"], 714 title="Factor", 715 ) 716 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters") 717 718 719class SessionTokenRequestBearerAuthenticator(BaseModel): 720 type: Literal["Bearer"] 721 722 723class HttpMethod(Enum): 724 GET = "GET" 725 POST = "POST" 726 727 728class Action(Enum): 729 SUCCESS = "SUCCESS" 730 FAIL = "FAIL" 731 RETRY = "RETRY" 732 IGNORE = "IGNORE" 733 RATE_LIMITED = "RATE_LIMITED" 734 735 736class FailureType(Enum): 737 system_error = "system_error" 738 config_error = "config_error" 739 transient_error = "transient_error" 740 741 742class HttpResponseFilter(BaseModel): 743 type: Literal["HttpResponseFilter"] 744 action: Optional[Action] = Field( 745 None, 746 description="Action to execute if a response matches the filter.", 747 examples=["SUCCESS", "FAIL", "RETRY", "IGNORE", "RATE_LIMITED"], 748 title="Action", 749 ) 750 failure_type: Optional[FailureType] = Field( 751 None, 752 description="Failure type of traced exception if a response matches the filter.", 753 examples=["system_error", "config_error", "transient_error"], 754 title="Failure Type", 755 ) 756 error_message: Optional[str] = Field( 757 None, 758 description="Error Message to display if the response matches the filter.", 759 title="Error Message", 760 ) 761 error_message_contains: Optional[str] = Field( 762 None, 763 description="Match the response if its error message contains the substring.", 764 example=["This API operation is not enabled for this site"], 765 title="Error Message Substring", 766 ) 767 http_codes: Optional[List[int]] = Field( 768 None, 769 description="Match the response if its HTTP code is included in this list.", 770 examples=[[420, 429], [500]], 771 title="HTTP Codes", 772 unique_items=True, 773 ) 774 predicate: Optional[str] = Field( 775 None, 776 description="Match the response if the predicate evaluates to true.", 777 examples=[ 778 "{{ 'Too much requests' in response }}", 779 "{{ 'error_code' in response and response['error_code'] == 'ComplexityException' }}", 780 ], 781 title="Predicate", 782 ) 783 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters") 784 785 786class ComplexFieldType(BaseModel): 787 field_type: str 788 items: Optional[Union[str, ComplexFieldType]] = None 789 790 791class TypesMap(BaseModel): 792 target_type: Union[str, List[str], ComplexFieldType] 793 current_type: Union[str, List[str]] 794 condition: Optional[str] = None 795 796 797class SchemaTypeIdentifier(BaseModel): 798 type: Optional[Literal["SchemaTypeIdentifier"]] = None 799 schema_pointer: Optional[List[str]] = Field( 800 [], 801 description="List of nested fields defining the schema field path to extract. Defaults to [].", 802 title="Schema Path", 803 ) 804 key_pointer: List[str] = Field( 805 ..., 806 description="List of potentially nested fields describing the full path of the field key to extract.", 807 title="Key Path", 808 ) 809 type_pointer: Optional[List[str]] = Field( 810 None, 811 description="List of potentially nested fields describing the full path of the field type to extract.", 812 title="Type Path", 813 ) 814 types_mapping: Optional[List[TypesMap]] = None 815 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters") 816 817 818class InlineSchemaLoader(BaseModel): 819 type: Literal["InlineSchemaLoader"] 820 schema_: Optional[Dict[str, Any]] = Field( 821 None, 822 alias="schema", 823 description='Describes a streams\' schema. Refer to the <a href="https://docs.airbyte.com/understanding-airbyte/supported-data-types/">Data Types documentation</a> for more details on which types are valid.', 824 title="Schema", 825 ) 826 827 828class JsonFileSchemaLoader(BaseModel): 829 type: Literal["JsonFileSchemaLoader"] 830 file_path: Optional[str] = Field( 831 None, 832 description="Path to the JSON file defining the schema. The path is relative to the connector module's root.", 833 example=["./schemas/users.json"], 834 title="File Path", 835 ) 836 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters") 837 838 839class JsonDecoder(BaseModel): 840 type: Literal["JsonDecoder"] 841 842 843class JsonlDecoder(BaseModel): 844 type: Literal["JsonlDecoder"] 845 846 847class KeysToLower(BaseModel): 848 type: Literal["KeysToLower"] 849 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters") 850 851 852class KeysToSnakeCase(BaseModel): 853 type: Literal["KeysToSnakeCase"] 854 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters") 855 856 857class FlattenFields(BaseModel): 858 type: Literal["FlattenFields"] 859 flatten_lists: Optional[bool] = Field( 860 True, 861 description="Whether to flatten lists or leave it as is. Default is True.", 862 title="Flatten Lists", 863 ) 864 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters") 865 866 867class DpathFlattenFields(BaseModel): 868 type: Literal["DpathFlattenFields"] 869 field_path: List[str] = Field( 870 ..., 871 description="A path to field that needs to be flattened.", 872 examples=[["data"], ["data", "*", "field"]], 873 title="Field Path", 874 ) 875 delete_origin_value: Optional[bool] = Field( 876 None, 877 description="Whether to delete the origin value or keep it. Default is False.", 878 title="Delete Origin Value", 879 ) 880 replace_record: Optional[bool] = Field( 881 None, 882 description="Whether to replace the origin record or not. Default is False.", 883 title="Replace Origin Record", 884 ) 885 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters") 886 887 888class KeysReplace(BaseModel): 889 type: Literal["KeysReplace"] 890 old: str = Field( 891 ..., 892 description="Old value to replace.", 893 examples=[ 894 " ", 895 "{{ record.id }}", 896 "{{ config['id'] }}", 897 "{{ stream_slice['id'] }}", 898 ], 899 title="Old value", 900 ) 901 new: str = Field( 902 ..., 903 description="New value to set.", 904 examples=[ 905 "_", 906 "{{ record.id }}", 907 "{{ config['id'] }}", 908 "{{ stream_slice['id'] }}", 909 ], 910 title="New value", 911 ) 912 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters") 913 914 915class IterableDecoder(BaseModel): 916 type: Literal["IterableDecoder"] 917 918 919class XmlDecoder(BaseModel): 920 type: Literal["XmlDecoder"] 921 922 923class CustomDecoder(BaseModel): 924 class Config: 925 extra = Extra.allow 926 927 type: Literal["CustomDecoder"] 928 class_name: str = Field( 929 ..., 930 description="Fully-qualified name of the class that will be implementing the custom decoding. Has to be a sub class of Decoder. The format is `source_<name>.<package>.<class_name>`.", 931 examples=["source_amazon_ads.components.GzipJsonlDecoder"], 932 title="Class Name", 933 ) 934 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters") 935 936 937class MinMaxDatetime(BaseModel): 938 type: Literal["MinMaxDatetime"] 939 datetime: str = Field( 940 ..., 941 description="Datetime value.", 942 examples=["2021-01-01", "2021-01-01T00:00:00Z", "{{ config['start_time'] }}"], 943 title="Datetime", 944 ) 945 datetime_format: Optional[str] = Field( 946 "", 947 description='Format of the datetime value. Defaults to "%Y-%m-%dT%H:%M:%S.%f%z" if left empty. Use placeholders starting with "%" to describe the format the API is using. The following placeholders are available:\n * **%s**: Epoch unix timestamp - `1686218963`\n * **%s_as_float**: Epoch unix timestamp in seconds as float with microsecond precision - `1686218963.123456`\n * **%ms**: Epoch unix timestamp - `1686218963123`\n * **%a**: Weekday (abbreviated) - `Sun`\n * **%A**: Weekday (full) - `Sunday`\n * **%w**: Weekday (decimal) - `0` (Sunday), `6` (Saturday)\n * **%d**: Day of the month (zero-padded) - `01`, `02`, ..., `31`\n * **%b**: Month (abbreviated) - `Jan`\n * **%B**: Month (full) - `January`\n * **%m**: Month (zero-padded) - `01`, `02`, ..., `12`\n * **%y**: Year (without century, zero-padded) - `00`, `01`, ..., `99`\n * **%Y**: Year (with century) - `0001`, `0002`, ..., `9999`\n * **%H**: Hour (24-hour, zero-padded) - `00`, `01`, ..., `23`\n * **%I**: Hour (12-hour, zero-padded) - `01`, `02`, ..., `12`\n * **%p**: AM/PM indicator\n * **%M**: Minute (zero-padded) - `00`, `01`, ..., `59`\n * **%S**: Second (zero-padded) - `00`, `01`, ..., `59`\n * **%f**: Microsecond (zero-padded to 6 digits) - `000000`, `000001`, ..., `999999`\n * **%_ms**: Millisecond (zero-padded to 3 digits) - `000`, `001`, ..., `999`\n * **%z**: UTC offset - `(empty)`, `+0000`, `-04:00`\n * **%Z**: Time zone name - `(empty)`, `UTC`, `GMT`\n * **%j**: Day of the year (zero-padded) - `001`, `002`, ..., `366`\n * **%U**: Week number of the year (Sunday as first day) - `00`, `01`, ..., `53`\n * **%W**: Week number of the year (Monday as first day) - `00`, `01`, ..., `53`\n * **%c**: Date and time representation - `Tue Aug 16 21:30:00 1988`\n * **%x**: Date representation - `08/16/1988`\n * **%X**: Time representation - `21:30:00`\n * **%%**: Literal \'%\' character\n\n Some placeholders depend on the locale of the underlying system - in most cases this locale is configured as en/US. For more information see the [Python documentation](https://docs.python.org/3/library/datetime.html#strftime-and-strptime-format-codes).\n', 948 examples=["%Y-%m-%dT%H:%M:%S.%f%z", "%Y-%m-%d", "%s"], 949 title="Datetime Format", 950 ) 951 max_datetime: Optional[str] = Field( 952 None, 953 description="Ceiling applied on the datetime value. Must be formatted with the datetime_format field.", 954 examples=["2021-01-01T00:00:00Z", "2021-01-01"], 955 title="Max Datetime", 956 ) 957 min_datetime: Optional[str] = Field( 958 None, 959 description="Floor applied on the datetime value. Must be formatted with the datetime_format field.", 960 examples=["2010-01-01T00:00:00Z", "2010-01-01"], 961 title="Min Datetime", 962 ) 963 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters") 964 965 966class NoAuth(BaseModel): 967 type: Literal["NoAuth"] 968 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters") 969 970 971class NoPagination(BaseModel): 972 type: Literal["NoPagination"] 973 974 975class State(BaseModel): 976 class Config: 977 extra = Extra.allow 978 979 min: int 980 max: int 981 982 983class OauthConnectorInputSpecification(BaseModel): 984 class Config: 985 extra = Extra.allow 986 987 consent_url: str = Field( 988 ..., 989 description="The DeclarativeOAuth Specific string URL string template to initiate the authentication.\nThe placeholders are replaced during the processing to provide neccessary values.", 990 examples=[ 991 "https://domain.host.com/marketing_api/auth?{{client_id_key}}={{client_id_value}}&{{redirect_uri_key}}={{{{redirect_uri_value}} | urlEncoder}}&{{state_key}}={{state_value}}", 992 "https://endpoint.host.com/oauth2/authorize?{{client_id_key}}={{client_id_value}}&{{redirect_uri_key}}={{{{redirect_uri_value}} | urlEncoder}}&{{scope_key}}={{{{scope_value}} | urlEncoder}}&{{state_key}}={{state_value}}&subdomain={{subdomain}}", 993 ], 994 title="Consent URL", 995 ) 996 scope: Optional[str] = Field( 997 None, 998 description="The DeclarativeOAuth Specific string of the scopes needed to be grant for authenticated user.", 999 examples=["user:read user:read_orders workspaces:read"], 1000 title="Scopes", 1001 ) 1002 access_token_url: str = Field( 1003 ..., 1004 description="The DeclarativeOAuth Specific URL templated string to obtain the `access_token`, `refresh_token` etc.\nThe placeholders are replaced during the processing to provide neccessary values.", 1005 examples=[ 1006 "https://auth.host.com/oauth2/token?{{client_id_key}}={{client_id_value}}&{{client_secret_key}}={{client_secret_value}}&{{auth_code_key}}={{auth_code_value}}&{{redirect_uri_key}}={{{{redirect_uri_value}} | urlEncoder}}" 1007 ], 1008 title="Access Token URL", 1009 ) 1010 access_token_headers: Optional[Dict[str, Any]] = Field( 1011 None, 1012 description="The DeclarativeOAuth Specific optional headers to inject while exchanging the `auth_code` to `access_token` during `completeOAuthFlow` step.", 1013 examples=[ 1014 { 1015 "Authorization": "Basic {{ {{ client_id_value }}:{{ client_secret_value }} | base64Encoder }}" 1016 } 1017 ], 1018 title="Access Token Headers", 1019 ) 1020 access_token_params: Optional[Dict[str, Any]] = Field( 1021 None, 1022 description="The DeclarativeOAuth Specific optional query parameters to inject while exchanging the `auth_code` to `access_token` during `completeOAuthFlow` step.\nWhen this property is provided, the query params will be encoded as `Json` and included in the outgoing API request.", 1023 examples=[ 1024 { 1025 "{{ auth_code_key }}": "{{ auth_code_value }}", 1026 "{{ client_id_key }}": "{{ client_id_value }}", 1027 "{{ client_secret_key }}": "{{ client_secret_value }}", 1028 } 1029 ], 1030 title="Access Token Query Params (Json Encoded)", 1031 ) 1032 extract_output: Optional[List[str]] = Field( 1033 None, 1034 description="The DeclarativeOAuth Specific list of strings to indicate which keys should be extracted and returned back to the input config.", 1035 examples=[["access_token", "refresh_token", "other_field"]], 1036 title="Extract Output", 1037 ) 1038 state: Optional[State] = Field( 1039 None, 1040 description="The DeclarativeOAuth Specific object to provide the criteria of how the `state` query param should be constructed,\nincluding length and complexity.", 1041 examples=[{"min": 7, "max": 128}], 1042 title="Configurable State Query Param", 1043 ) 1044 client_id_key: Optional[str] = Field( 1045 None, 1046 description="The DeclarativeOAuth Specific optional override to provide the custom `client_id` key name, if required by data-provider.", 1047 examples=["my_custom_client_id_key_name"], 1048 title="Client ID Key Override", 1049 ) 1050 client_secret_key: Optional[str] = Field( 1051 None, 1052 description="The DeclarativeOAuth Specific optional override to provide the custom `client_secret` key name, if required by data-provider.", 1053 examples=["my_custom_client_secret_key_name"], 1054 title="Client Secret Key Override", 1055 ) 1056 scope_key: Optional[str] = Field( 1057 None, 1058 description="The DeclarativeOAuth Specific optional override to provide the custom `scope` key name, if required by data-provider.", 1059 examples=["my_custom_scope_key_key_name"], 1060 title="Scopes Key Override", 1061 ) 1062 state_key: Optional[str] = Field( 1063 None, 1064 description="The DeclarativeOAuth Specific optional override to provide the custom `state` key name, if required by data-provider.", 1065 examples=["my_custom_state_key_key_name"], 1066 title="State Key Override", 1067 ) 1068 auth_code_key: Optional[str] = Field( 1069 None, 1070 description="The DeclarativeOAuth Specific optional override to provide the custom `code` key name to something like `auth_code` or `custom_auth_code`, if required by data-provider.", 1071 examples=["my_custom_auth_code_key_name"], 1072 title="Auth Code Key Override", 1073 ) 1074 redirect_uri_key: Optional[str] = Field( 1075 None, 1076 description="The DeclarativeOAuth Specific optional override to provide the custom `redirect_uri` key name to something like `callback_uri`, if required by data-provider.", 1077 examples=["my_custom_redirect_uri_key_name"], 1078 title="Redirect URI Key Override", 1079 ) 1080 1081 1082class OAuthConfigSpecification(BaseModel): 1083 class Config: 1084 extra = Extra.allow 1085 1086 oauth_user_input_from_connector_config_specification: Optional[Dict[str, Any]] = Field( 1087 None, 1088 description="OAuth specific blob. This is a Json Schema used to validate Json configurations used as input to OAuth.\nMust be a valid non-nested JSON that refers to properties from ConnectorSpecification.connectionSpecification\nusing special annotation 'path_in_connector_config'.\nThese are input values the user is entering through the UI to authenticate to the connector, that might also shared\nas inputs for syncing data via the connector.\nExamples:\nif no connector values is shared during oauth flow, oauth_user_input_from_connector_config_specification=[]\nif connector values such as 'app_id' inside the top level are used to generate the API url for the oauth flow,\n oauth_user_input_from_connector_config_specification={\n app_id: {\n type: string\n path_in_connector_config: ['app_id']\n }\n }\nif connector values such as 'info.app_id' nested inside another object are used to generate the API url for the oauth flow,\n oauth_user_input_from_connector_config_specification={\n app_id: {\n type: string\n path_in_connector_config: ['info', 'app_id']\n }\n }", 1089 examples=[ 1090 {"app_id": {"type": "string", "path_in_connector_config": ["app_id"]}}, 1091 { 1092 "app_id": { 1093 "type": "string", 1094 "path_in_connector_config": ["info", "app_id"], 1095 } 1096 }, 1097 ], 1098 title="OAuth user input", 1099 ) 1100 oauth_connector_input_specification: Optional[OauthConnectorInputSpecification] = Field( 1101 None, 1102 description='The DeclarativeOAuth specific blob.\nPertains to the fields defined by the connector relating to the OAuth flow.\n\nInterpolation capabilities:\n- The variables placeholders are declared as `{{my_var}}`.\n- The nested resolution variables like `{{ {{my_nested_var}} }}` is allowed as well.\n\n- The allowed interpolation context is:\n + base64Encoder - encode to `base64`, {{ {{my_var_a}}:{{my_var_b}} | base64Encoder }}\n + base64Decorer - decode from `base64` encoded string, {{ {{my_string_variable_or_string_value}} | base64Decoder }}\n + urlEncoder - encode the input string to URL-like format, {{ https://test.host.com/endpoint | urlEncoder}}\n + urlDecorer - decode the input url-encoded string into text format, {{ urlDecoder:https%3A%2F%2Fairbyte.io | urlDecoder}}\n + codeChallengeS256 - get the `codeChallenge` encoded value to provide additional data-provider specific authorisation values, {{ {{state_value}} | codeChallengeS256 }}\n\nExamples:\n - The TikTok Marketing DeclarativeOAuth spec:\n {\n "oauth_connector_input_specification": {\n "type": "object",\n "additionalProperties": false,\n "properties": {\n "consent_url": "https://ads.tiktok.com/marketing_api/auth?{{client_id_key}}={{client_id_value}}&{{redirect_uri_key}}={{ {{redirect_uri_value}} | urlEncoder}}&{{state_key}}={{state_value}}",\n "access_token_url": "https://business-api.tiktok.com/open_api/v1.3/oauth2/access_token/",\n "access_token_params": {\n "{{ auth_code_key }}": "{{ auth_code_value }}",\n "{{ client_id_key }}": "{{ client_id_value }}",\n "{{ client_secret_key }}": "{{ client_secret_value }}"\n },\n "access_token_headers": {\n "Content-Type": "application/json",\n "Accept": "application/json"\n },\n "extract_output": ["data.access_token"],\n "client_id_key": "app_id",\n "client_secret_key": "secret",\n "auth_code_key": "auth_code"\n }\n }\n }', 1103 title="DeclarativeOAuth Connector Specification", 1104 ) 1105 complete_oauth_output_specification: Optional[Dict[str, Any]] = Field( 1106 None, 1107 description="OAuth specific blob. This is a Json Schema used to validate Json configurations produced by the OAuth flows as they are\nreturned by the distant OAuth APIs.\nMust be a valid JSON describing the fields to merge back to `ConnectorSpecification.connectionSpecification`.\nFor each field, a special annotation `path_in_connector_config` can be specified to determine where to merge it,\nExamples:\n complete_oauth_output_specification={\n refresh_token: {\n type: string,\n path_in_connector_config: ['credentials', 'refresh_token']\n }\n }", 1108 examples=[ 1109 { 1110 "refresh_token": { 1111 "type": "string,", 1112 "path_in_connector_config": ["credentials", "refresh_token"], 1113 } 1114 } 1115 ], 1116 title="OAuth output specification", 1117 ) 1118 complete_oauth_server_input_specification: Optional[Dict[str, Any]] = Field( 1119 None, 1120 description="OAuth specific blob. This is a Json Schema used to validate Json configurations persisted as Airbyte Server configurations.\nMust be a valid non-nested JSON describing additional fields configured by the Airbyte Instance or Workspace Admins to be used by the\nserver when completing an OAuth flow (typically exchanging an auth code for refresh token).\nExamples:\n complete_oauth_server_input_specification={\n client_id: {\n type: string\n },\n client_secret: {\n type: string\n }\n }", 1121 examples=[{"client_id": {"type": "string"}, "client_secret": {"type": "string"}}], 1122 title="OAuth input specification", 1123 ) 1124 complete_oauth_server_output_specification: Optional[Dict[str, Any]] = Field( 1125 None, 1126 description="OAuth specific blob. This is a Json Schema used to validate Json configurations persisted as Airbyte Server configurations that\nalso need to be merged back into the connector configuration at runtime.\nThis is a subset configuration of `complete_oauth_server_input_specification` that filters fields out to retain only the ones that\nare necessary for the connector to function with OAuth. (some fields could be used during oauth flows but not needed afterwards, therefore\nthey would be listed in the `complete_oauth_server_input_specification` but not `complete_oauth_server_output_specification`)\nMust be a valid non-nested JSON describing additional fields configured by the Airbyte Instance or Workspace Admins to be used by the\nconnector when using OAuth flow APIs.\nThese fields are to be merged back to `ConnectorSpecification.connectionSpecification`.\nFor each field, a special annotation `path_in_connector_config` can be specified to determine where to merge it,\nExamples:\n complete_oauth_server_output_specification={\n client_id: {\n type: string,\n path_in_connector_config: ['credentials', 'client_id']\n },\n client_secret: {\n type: string,\n path_in_connector_config: ['credentials', 'client_secret']\n }\n }", 1127 examples=[ 1128 { 1129 "client_id": { 1130 "type": "string,", 1131 "path_in_connector_config": ["credentials", "client_id"], 1132 }, 1133 "client_secret": { 1134 "type": "string,", 1135 "path_in_connector_config": ["credentials", "client_secret"], 1136 }, 1137 } 1138 ], 1139 title="OAuth server output specification", 1140 ) 1141 1142 1143class OffsetIncrement(BaseModel): 1144 type: Literal["OffsetIncrement"] 1145 page_size: Optional[Union[int, str]] = Field( 1146 None, 1147 description="The number of records to include in each pages.", 1148 examples=[100, "{{ config['page_size'] }}"], 1149 title="Limit", 1150 ) 1151 inject_on_first_request: Optional[bool] = Field( 1152 False, 1153 description="Using the `offset` with value `0` during the first request", 1154 title="Inject Offset", 1155 ) 1156 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters") 1157 1158 1159class PageIncrement(BaseModel): 1160 type: Literal["PageIncrement"] 1161 page_size: Optional[Union[int, str]] = Field( 1162 None, 1163 description="The number of records to include in each pages.", 1164 examples=[100, "100", "{{ config['page_size'] }}"], 1165 title="Page Size", 1166 ) 1167 start_from_page: Optional[int] = Field( 1168 0, 1169 description="Index of the first page to request.", 1170 examples=[0, 1], 1171 title="Start From Page", 1172 ) 1173 inject_on_first_request: Optional[bool] = Field( 1174 False, 1175 description="Using the `page number` with value defined by `start_from_page` during the first request", 1176 title="Inject Page Number", 1177 ) 1178 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters") 1179 1180 1181class PrimaryKey(BaseModel): 1182 __root__: Union[str, List[str], List[List[str]]] = Field( 1183 ..., 1184 description="The stream field to be used to distinguish unique records. Can either be a single field, an array of fields representing a composite key, or an array of arrays representing a composite key where the fields are nested fields.", 1185 examples=["id", ["code", "type"]], 1186 title="Primary Key", 1187 ) 1188 1189 1190class RecordFilter(BaseModel): 1191 type: Literal["RecordFilter"] 1192 condition: Optional[str] = Field( 1193 "", 1194 description="The predicate to filter a record. Records will be removed if evaluated to False.", 1195 examples=[ 1196 "{{ record['created_at'] >= stream_interval['start_time'] }}", 1197 "{{ record.status in ['active', 'expired'] }}", 1198 ], 1199 ) 1200 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters") 1201 1202 1203class SchemaNormalization(Enum): 1204 None_ = "None" 1205 Default = "Default" 1206 1207 1208class RemoveFields(BaseModel): 1209 type: Literal["RemoveFields"] 1210 condition: Optional[str] = Field( 1211 "", 1212 description="The predicate to filter a property by a property value. Property will be removed if it is empty OR expression is evaluated to True.,", 1213 examples=[ 1214 "{{ property|string == '' }}", 1215 "{{ property is integer }}", 1216 "{{ property|length > 5 }}", 1217 "{{ property == 'some_string_to_match' }}", 1218 ], 1219 ) 1220 field_pointers: List[List[str]] = Field( 1221 ..., 1222 description="Array of paths defining the field to remove. Each item is an array whose field describe the path of a field to remove.", 1223 examples=[["tags"], [["content", "html"], ["content", "plain_text"]]], 1224 title="Field Paths", 1225 ) 1226 1227 1228class RequestPath(BaseModel): 1229 type: Literal["RequestPath"] 1230 1231 1232class InjectInto(Enum): 1233 request_parameter = "request_parameter" 1234 header = "header" 1235 body_data = "body_data" 1236 body_json = "body_json" 1237 1238 1239class RequestOption(BaseModel): 1240 type: Literal["RequestOption"] 1241 field_name: Optional[str] = Field( 1242 None, 1243 description="Configures which key should be used in the location that the descriptor is being injected into. We hope to eventually deprecate this field in favor of `field_path` for all request_options, but must currently maintain it for backwards compatibility in the Builder.", 1244 examples=["segment_id"], 1245 title="Field Name", 1246 ) 1247 field_path: Optional[List[str]] = Field( 1248 None, 1249 description="Configures a path to be used for nested structures in JSON body requests (e.g. GraphQL queries)", 1250 examples=[["data", "viewer", "id"]], 1251 title="Field Path", 1252 ) 1253 inject_into: InjectInto = Field( 1254 ..., 1255 description="Configures where the descriptor should be set on the HTTP requests. Note that request parameters that are already encoded in the URL path will not be duplicated.", 1256 examples=["request_parameter", "header", "body_data", "body_json"], 1257 title="Inject Into", 1258 ) 1259 1260 1261class Schemas(BaseModel): 1262 pass 1263 1264 class Config: 1265 extra = Extra.allow 1266 1267 1268class LegacySessionTokenAuthenticator(BaseModel): 1269 type: Literal["LegacySessionTokenAuthenticator"] 1270 header: str = Field( 1271 ..., 1272 description="The name of the session token header that will be injected in the request", 1273 examples=["X-Session"], 1274 title="Session Request Header", 1275 ) 1276 login_url: str = Field( 1277 ..., 1278 description="Path of the login URL (do not include the base URL)", 1279 examples=["session"], 1280 title="Login Path", 1281 ) 1282 session_token: Optional[str] = Field( 1283 None, 1284 description="Session token to use if using a pre-defined token. Not needed if authenticating with username + password pair", 1285 example=["{{ config['session_token'] }}"], 1286 title="Session Token", 1287 ) 1288 session_token_response_key: str = Field( 1289 ..., 1290 description="Name of the key of the session token to be extracted from the response", 1291 examples=["id"], 1292 title="Response Token Response Key", 1293 ) 1294 username: Optional[str] = Field( 1295 None, 1296 description="Username used to authenticate and obtain a session token", 1297 examples=[" {{ config['username'] }}"], 1298 title="Username", 1299 ) 1300 password: Optional[str] = Field( 1301 "", 1302 description="Password used to authenticate and obtain a session token", 1303 examples=["{{ config['password'] }}", ""], 1304 title="Password", 1305 ) 1306 validate_session_url: str = Field( 1307 ..., 1308 description="Path of the URL to use to validate that the session token is valid (do not include the base URL)", 1309 examples=["user/current"], 1310 title="Validate Session Path", 1311 ) 1312 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters") 1313 1314 1315class CsvDecoder(BaseModel): 1316 type: Literal["CsvDecoder"] 1317 encoding: Optional[str] = "utf-8" 1318 delimiter: Optional[str] = "," 1319 1320 1321class AsyncJobStatusMap(BaseModel): 1322 type: Optional[Literal["AsyncJobStatusMap"]] = None 1323 running: List[str] 1324 completed: List[str] 1325 failed: List[str] 1326 timeout: List[str] 1327 1328 1329class ValueType(Enum): 1330 string = "string" 1331 number = "number" 1332 integer = "integer" 1333 boolean = "boolean" 1334 1335 1336class WaitTimeFromHeader(BaseModel): 1337 type: Literal["WaitTimeFromHeader"] 1338 header: str = Field( 1339 ..., 1340 description="The name of the response header defining how long to wait before retrying.", 1341 examples=["Retry-After"], 1342 title="Response Header Name", 1343 ) 1344 regex: Optional[str] = Field( 1345 None, 1346 description="Optional regex to apply on the header to extract its value. The regex should define a capture group defining the wait time.", 1347 examples=["([-+]?\\d+)"], 1348 title="Extraction Regex", 1349 ) 1350 max_waiting_time_in_seconds: Optional[float] = Field( 1351 None, 1352 description="Given the value extracted from the header is greater than this value, stop the stream.", 1353 examples=[3600], 1354 title="Max Waiting Time in Seconds", 1355 ) 1356 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters") 1357 1358 1359class WaitUntilTimeFromHeader(BaseModel): 1360 type: Literal["WaitUntilTimeFromHeader"] 1361 header: str = Field( 1362 ..., 1363 description="The name of the response header defining how long to wait before retrying.", 1364 examples=["wait_time"], 1365 title="Response Header", 1366 ) 1367 min_wait: Optional[Union[float, str]] = Field( 1368 None, 1369 description="Minimum time to wait before retrying.", 1370 examples=[10, "60"], 1371 title="Minimum Wait Time", 1372 ) 1373 regex: Optional[str] = Field( 1374 None, 1375 description="Optional regex to apply on the header to extract its value. The regex should define a capture group defining the wait time.", 1376 examples=["([-+]?\\d+)"], 1377 title="Extraction Regex", 1378 ) 1379 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters") 1380 1381 1382class ComponentMappingDefinition(BaseModel): 1383 type: Literal["ComponentMappingDefinition"] 1384 field_path: List[str] = Field( 1385 ..., 1386 description="A list of potentially nested fields indicating the full path where value will be added or updated.", 1387 examples=[ 1388 ["data"], 1389 ["data", "records"], 1390 ["data", 1, "name"], 1391 ["data", "{{ components_values.name }}"], 1392 ["data", "*", "record"], 1393 ["*", "**", "name"], 1394 ], 1395 title="Field Path", 1396 ) 1397 value: str = Field( 1398 ..., 1399 description="The dynamic or static value to assign to the key. Interpolated values can be used to dynamically determine the value during runtime.", 1400 examples=[ 1401 "{{ components_values['updates'] }}", 1402 "{{ components_values['MetaData']['LastUpdatedTime'] }}", 1403 "{{ config['segment_id'] }}", 1404 "{{ stream_slice['parent_id'] }}", 1405 "{{ stream_slice['extra_fields']['name'] }}", 1406 ], 1407 title="Value", 1408 ) 1409 value_type: Optional[ValueType] = Field( 1410 None, 1411 description="The expected data type of the value. If omitted, the type will be inferred from the value provided.", 1412 title="Value Type", 1413 ) 1414 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters") 1415 1416 1417class StreamConfig(BaseModel): 1418 type: Literal["StreamConfig"] 1419 configs_pointer: List[str] = Field( 1420 ..., 1421 description="A list of potentially nested fields indicating the full path in source config file where streams configs located.", 1422 examples=[["data"], ["data", "streams"], ["data", "{{ parameters.name }}"]], 1423 title="Configs Pointer", 1424 ) 1425 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters") 1426 1427 1428class ConfigComponentsResolver(BaseModel): 1429 type: Literal["ConfigComponentsResolver"] 1430 stream_config: StreamConfig 1431 components_mapping: List[ComponentMappingDefinition] 1432 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters") 1433 1434 1435class AddedFieldDefinition(BaseModel): 1436 type: Literal["AddedFieldDefinition"] 1437 path: List[str] = Field( 1438 ..., 1439 description="List of strings defining the path where to add the value on the record.", 1440 examples=[["segment_id"], ["metadata", "segment_id"]], 1441 title="Path", 1442 ) 1443 value: str = Field( 1444 ..., 1445 description="Value of the new field. Use {{ record['existing_field'] }} syntax to refer to other fields in the record.", 1446 examples=[ 1447 "{{ record['updates'] }}", 1448 "{{ record['MetaData']['LastUpdatedTime'] }}", 1449 "{{ stream_partition['segment_id'] }}", 1450 ], 1451 title="Value", 1452 ) 1453 value_type: Optional[ValueType] = Field( 1454 None, 1455 description="Type of the value. If not specified, the type will be inferred from the value.", 1456 title="Value Type", 1457 ) 1458 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters") 1459 1460 1461class AddFields(BaseModel): 1462 type: Literal["AddFields"] 1463 fields: List[AddedFieldDefinition] = Field( 1464 ..., 1465 description="List of transformations (path and corresponding value) that will be added to the record.", 1466 title="Fields", 1467 ) 1468 condition: Optional[str] = Field( 1469 "", 1470 description="Fields will be added if expression is evaluated to True.", 1471 examples=[ 1472 "{{ property|string == '' }}", 1473 "{{ property is integer }}", 1474 "{{ property|length > 5 }}", 1475 "{{ property == 'some_string_to_match' }}", 1476 ], 1477 ) 1478 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters") 1479 1480 1481class ApiKeyAuthenticator(BaseModel): 1482 type: Literal["ApiKeyAuthenticator"] 1483 api_token: Optional[str] = Field( 1484 None, 1485 description="The API key to inject in the request. Fill it in the user inputs.", 1486 examples=["{{ config['api_key'] }}", "Token token={{ config['api_key'] }}"], 1487 title="API Key", 1488 ) 1489 header: Optional[str] = Field( 1490 None, 1491 description="The name of the HTTP header that will be set to the API key. This setting is deprecated, use inject_into instead. Header and inject_into can not be defined at the same time.", 1492 examples=["Authorization", "Api-Token", "X-Auth-Token"], 1493 title="Header Name", 1494 ) 1495 inject_into: Optional[RequestOption] = Field( 1496 None, 1497 description="Configure how the API Key will be sent in requests to the source API. Either inject_into or header has to be defined.", 1498 examples=[ 1499 {"inject_into": "header", "field_name": "Authorization"}, 1500 {"inject_into": "request_parameter", "field_name": "authKey"}, 1501 ], 1502 title="Inject API Key Into Outgoing HTTP Request", 1503 ) 1504 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters") 1505 1506 1507class AuthFlow(BaseModel): 1508 auth_flow_type: Optional[AuthFlowType] = Field( 1509 None, description="The type of auth to use", title="Auth flow type" 1510 ) 1511 predicate_key: Optional[List[str]] = Field( 1512 None, 1513 description="JSON path to a field in the connectorSpecification that should exist for the advanced auth to be applicable.", 1514 examples=[["credentials", "auth_type"]], 1515 title="Predicate key", 1516 ) 1517 predicate_value: Optional[str] = Field( 1518 None, 1519 description="Value of the predicate_key fields for the advanced auth to be applicable.", 1520 examples=["Oauth"], 1521 title="Predicate value", 1522 ) 1523 oauth_config_specification: Optional[OAuthConfigSpecification] = None 1524 1525 1526class IncrementingCountCursor(BaseModel): 1527 type: Literal["IncrementingCountCursor"] 1528 cursor_field: str = Field( 1529 ..., 1530 description="The location of the value on a record that will be used as a bookmark during sync. To ensure no data loss, the API must return records in ascending order based on the cursor field. Nested fields are not supported, so the field must be at the top level of the record. You can use a combination of Add Field and Remove Field transformations to move the nested field to the top.", 1531 examples=["created_at", "{{ config['record_cursor'] }}"], 1532 title="Cursor Field", 1533 ) 1534 start_value: Optional[Union[str, int]] = Field( 1535 None, 1536 description="The value that determines the earliest record that should be synced.", 1537 examples=[0, "{{ config['start_value'] }}"], 1538 title="Start Value", 1539 ) 1540 start_value_option: Optional[RequestOption] = Field( 1541 None, 1542 description="Optionally configures how the start value will be sent in requests to the source API.", 1543 title="Inject Start Value Into Outgoing HTTP Request", 1544 ) 1545 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters") 1546 1547 1548class DatetimeBasedCursor(BaseModel): 1549 type: Literal["DatetimeBasedCursor"] 1550 clamping: Optional[Clamping] = Field( 1551 None, 1552 description="This option is used to adjust the upper and lower boundaries of each datetime window to beginning and end of the provided target period (day, week, month)", 1553 title="Date Range Clamping", 1554 ) 1555 cursor_field: str = Field( 1556 ..., 1557 description="The location of the value on a record that will be used as a bookmark during sync. To ensure no data loss, the API must return records in ascending order based on the cursor field. Nested fields are not supported, so the field must be at the top level of the record. You can use a combination of Add Field and Remove Field transformations to move the nested field to the top.", 1558 examples=["created_at", "{{ config['record_cursor'] }}"], 1559 title="Cursor Field", 1560 ) 1561 datetime_format: str = Field( 1562 ..., 1563 description="The datetime format used to format the datetime values that are sent in outgoing requests to the API. Use placeholders starting with \"%\" to describe the format the API is using. The following placeholders are available:\n * **%s**: Epoch unix timestamp - `1686218963`\n * **%s_as_float**: Epoch unix timestamp in seconds as float with microsecond precision - `1686218963.123456`\n * **%ms**: Epoch unix timestamp (milliseconds) - `1686218963123`\n * **%a**: Weekday (abbreviated) - `Sun`\n * **%A**: Weekday (full) - `Sunday`\n * **%w**: Weekday (decimal) - `0` (Sunday), `6` (Saturday)\n * **%d**: Day of the month (zero-padded) - `01`, `02`, ..., `31`\n * **%b**: Month (abbreviated) - `Jan`\n * **%B**: Month (full) - `January`\n * **%m**: Month (zero-padded) - `01`, `02`, ..., `12`\n * **%y**: Year (without century, zero-padded) - `00`, `01`, ..., `99`\n * **%Y**: Year (with century) - `0001`, `0002`, ..., `9999`\n * **%H**: Hour (24-hour, zero-padded) - `00`, `01`, ..., `23`\n * **%I**: Hour (12-hour, zero-padded) - `01`, `02`, ..., `12`\n * **%p**: AM/PM indicator\n * **%M**: Minute (zero-padded) - `00`, `01`, ..., `59`\n * **%S**: Second (zero-padded) - `00`, `01`, ..., `59`\n * **%f**: Microsecond (zero-padded to 6 digits) - `000000`\n * **%_ms**: Millisecond (zero-padded to 3 digits) - `000`\n * **%z**: UTC offset - `(empty)`, `+0000`, `-04:00`\n * **%Z**: Time zone name - `(empty)`, `UTC`, `GMT`\n * **%j**: Day of the year (zero-padded) - `001`, `002`, ..., `366`\n * **%U**: Week number of the year (starting Sunday) - `00`, ..., `53`\n * **%W**: Week number of the year (starting Monday) - `00`, ..., `53`\n * **%c**: Date and time - `Tue Aug 16 21:30:00 1988`\n * **%x**: Date standard format - `08/16/1988`\n * **%X**: Time standard format - `21:30:00`\n * **%%**: Literal '%' character\n\n Some placeholders depend on the locale of the underlying system - in most cases this locale is configured as en/US. For more information see the [Python documentation](https://docs.python.org/3/library/datetime.html#strftime-and-strptime-format-codes).\n", 1564 examples=["%Y-%m-%dT%H:%M:%S.%f%z", "%Y-%m-%d", "%s", "%ms", "%s_as_float"], 1565 title="Outgoing Datetime Format", 1566 ) 1567 start_datetime: Union[str, MinMaxDatetime] = Field( 1568 ..., 1569 description="The datetime that determines the earliest record that should be synced.", 1570 examples=["2020-01-1T00:00:00Z", "{{ config['start_time'] }}"], 1571 title="Start Datetime", 1572 ) 1573 cursor_datetime_formats: Optional[List[str]] = Field( 1574 None, 1575 description="The possible formats for the cursor field, in order of preference. The first format that matches the cursor field value will be used to parse it. If not provided, the `datetime_format` will be used.", 1576 title="Cursor Datetime Formats", 1577 ) 1578 cursor_granularity: Optional[str] = Field( 1579 None, 1580 description="Smallest increment the datetime_format has (ISO 8601 duration) that is used to ensure the start of a slice does not overlap with the end of the previous one, e.g. for %Y-%m-%d the granularity should be P1D, for %Y-%m-%dT%H:%M:%SZ the granularity should be PT1S. Given this field is provided, `step` needs to be provided as well.", 1581 examples=["PT1S"], 1582 title="Cursor Granularity", 1583 ) 1584 end_datetime: Optional[Union[str, MinMaxDatetime]] = Field( 1585 None, 1586 description="The datetime that determines the last record that should be synced. If not provided, `{{ now_utc() }}` will be used.", 1587 examples=["2021-01-1T00:00:00Z", "{{ now_utc() }}", "{{ day_delta(-1) }}"], 1588 title="End Datetime", 1589 ) 1590 end_time_option: Optional[RequestOption] = Field( 1591 None, 1592 description="Optionally configures how the end datetime will be sent in requests to the source API.", 1593 title="Inject End Time Into Outgoing HTTP Request", 1594 ) 1595 is_data_feed: Optional[bool] = Field( 1596 None, 1597 description="A data feed API is an API that does not allow filtering and paginates the content from the most recent to the least recent. Given this, the CDK needs to know when to stop paginating and this field will generate a stop condition for pagination.", 1598 title="Whether the target API is formatted as a data feed", 1599 ) 1600 is_client_side_incremental: Optional[bool] = Field( 1601 None, 1602 description="If the target API endpoint does not take cursor values to filter records and returns all records anyway, the connector with this cursor will filter out records locally, and only emit new records from the last sync, hence incremental. This means that all records would be read from the API, but only new records will be emitted to the destination.", 1603 title="Whether the target API does not support filtering and returns all data (the cursor filters records in the client instead of the API side)", 1604 ) 1605 is_compare_strictly: Optional[bool] = Field( 1606 False, 1607 description="Set to True if the target API does not accept queries where the start time equal the end time.", 1608 title="Whether to skip requests if the start time equals the end time", 1609 ) 1610 global_substream_cursor: Optional[bool] = Field( 1611 False, 1612 description="This setting optimizes performance when the parent stream has thousands of partitions by storing the cursor as a single value rather than per partition. Notably, the substream state is updated only at the end of the sync, which helps prevent data loss in case of a sync failure. See more info in the [docs](https://docs.airbyte.com/connector-development/config-based/understanding-the-yaml-file/incremental-syncs).", 1613 title="Whether to store cursor as one value instead of per partition", 1614 ) 1615 lookback_window: Optional[str] = Field( 1616 None, 1617 description="Time interval before the start_datetime to read data for, e.g. P1M for looking back one month.", 1618 examples=["P1D", "P{{ config['lookback_days'] }}D"], 1619 title="Lookback Window", 1620 ) 1621 partition_field_end: Optional[str] = Field( 1622 None, 1623 description="Name of the partition start time field.", 1624 examples=["ending_time"], 1625 title="Partition Field End", 1626 ) 1627 partition_field_start: Optional[str] = Field( 1628 None, 1629 description="Name of the partition end time field.", 1630 examples=["starting_time"], 1631 title="Partition Field Start", 1632 ) 1633 start_time_option: Optional[RequestOption] = Field( 1634 None, 1635 description="Optionally configures how the start datetime will be sent in requests to the source API.", 1636 title="Inject Start Time Into Outgoing HTTP Request", 1637 ) 1638 step: Optional[str] = Field( 1639 None, 1640 description="The size of the time window (ISO8601 duration). Given this field is provided, `cursor_granularity` needs to be provided as well.", 1641 examples=["P1W", "{{ config['step_increment'] }}"], 1642 title="Step", 1643 ) 1644 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters") 1645 1646 1647class FixedWindowCallRatePolicy(BaseModel): 1648 class Config: 1649 extra = Extra.allow 1650 1651 type: Literal["FixedWindowCallRatePolicy"] 1652 period: str = Field( 1653 ..., description="The time interval for the rate limit window.", title="Period" 1654 ) 1655 call_limit: int = Field( 1656 ..., 1657 description="The maximum number of calls allowed within the period.", 1658 title="Call Limit", 1659 ) 1660 matchers: List[HttpRequestRegexMatcher] = Field( 1661 ..., 1662 description="List of matchers that define which requests this policy applies to.", 1663 title="Matchers", 1664 ) 1665 1666 1667class MovingWindowCallRatePolicy(BaseModel): 1668 class Config: 1669 extra = Extra.allow 1670 1671 type: Literal["MovingWindowCallRatePolicy"] 1672 rates: List[Rate] = Field( 1673 ..., 1674 description="List of rates that define the call limits for different time intervals.", 1675 title="Rates", 1676 ) 1677 matchers: List[HttpRequestRegexMatcher] = Field( 1678 ..., 1679 description="List of matchers that define which requests this policy applies to.", 1680 title="Matchers", 1681 ) 1682 1683 1684class UnlimitedCallRatePolicy(BaseModel): 1685 class Config: 1686 extra = Extra.allow 1687 1688 type: Literal["UnlimitedCallRatePolicy"] 1689 matchers: List[HttpRequestRegexMatcher] = Field( 1690 ..., 1691 description="List of matchers that define which requests this policy applies to.", 1692 title="Matchers", 1693 ) 1694 1695 1696class DefaultErrorHandler(BaseModel): 1697 type: Literal["DefaultErrorHandler"] 1698 backoff_strategies: Optional[ 1699 List[ 1700 Union[ 1701 ConstantBackoffStrategy, 1702 CustomBackoffStrategy, 1703 ExponentialBackoffStrategy, 1704 WaitTimeFromHeader, 1705 WaitUntilTimeFromHeader, 1706 ] 1707 ] 1708 ] = Field( 1709 None, 1710 description="List of backoff strategies to use to determine how long to wait before retrying a retryable request.", 1711 title="Backoff Strategies", 1712 ) 1713 max_retries: Optional[int] = Field( 1714 5, 1715 description="The maximum number of time to retry a retryable request before giving up and failing.", 1716 examples=[5, 0, 10], 1717 title="Max Retry Count", 1718 ) 1719 response_filters: Optional[List[HttpResponseFilter]] = Field( 1720 None, 1721 description="List of response filters to iterate on when deciding how to handle an error. When using an array of multiple filters, the filters will be applied sequentially and the response will be selected if it matches any of the filter's predicate.", 1722 title="Response Filters", 1723 ) 1724 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters") 1725 1726 1727class DefaultPaginator(BaseModel): 1728 type: Literal["DefaultPaginator"] 1729 pagination_strategy: Union[ 1730 CursorPagination, CustomPaginationStrategy, OffsetIncrement, PageIncrement 1731 ] = Field( 1732 ..., 1733 description="Strategy defining how records are paginated.", 1734 title="Pagination Strategy", 1735 ) 1736 page_size_option: Optional[RequestOption] = None 1737 page_token_option: Optional[Union[RequestOption, RequestPath]] = None 1738 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters") 1739 1740 1741class SessionTokenRequestApiKeyAuthenticator(BaseModel): 1742 type: Literal["ApiKey"] 1743 inject_into: RequestOption = Field( 1744 ..., 1745 description="Configure how the API Key will be sent in requests to the source API.", 1746 examples=[ 1747 {"inject_into": "header", "field_name": "Authorization"}, 1748 {"inject_into": "request_parameter", "field_name": "authKey"}, 1749 ], 1750 title="Inject API Key Into Outgoing HTTP Request", 1751 ) 1752 1753 1754class ListPartitionRouter(BaseModel): 1755 type: Literal["ListPartitionRouter"] 1756 cursor_field: str = Field( 1757 ..., 1758 description='While iterating over list values, the name of field used to reference a list value. The partition value can be accessed with string interpolation. e.g. "{{ stream_partition[\'my_key\'] }}" where "my_key" is the value of the cursor_field.', 1759 examples=["section", "{{ config['section_key'] }}"], 1760 title="Current Partition Value Identifier", 1761 ) 1762 values: Union[str, List[str]] = Field( 1763 ..., 1764 description="The list of attributes being iterated over and used as input for the requests made to the source API.", 1765 examples=[["section_a", "section_b", "section_c"], "{{ config['sections'] }}"], 1766 title="Partition Values", 1767 ) 1768 request_option: Optional[RequestOption] = Field( 1769 None, 1770 description="A request option describing where the list value should be injected into and under what field name if applicable.", 1771 title="Inject Partition Value Into Outgoing HTTP Request", 1772 ) 1773 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters") 1774 1775 1776class RecordSelector(BaseModel): 1777 type: Literal["RecordSelector"] 1778 extractor: Union[CustomRecordExtractor, DpathExtractor] 1779 record_filter: Optional[Union[CustomRecordFilter, RecordFilter]] = Field( 1780 None, 1781 description="Responsible for filtering records to be emitted by the Source.", 1782 title="Record Filter", 1783 ) 1784 schema_normalization: Optional[Union[SchemaNormalization, CustomSchemaNormalization]] = Field( 1785 SchemaNormalization.None_, 1786 description="Responsible for normalization according to the schema.", 1787 title="Schema Normalization", 1788 ) 1789 transform_before_filtering: Optional[bool] = Field( 1790 False, 1791 description="If true, transformation will be applied before record filtering.", 1792 ) 1793 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters") 1794 1795 1796class GzipDecoder(BaseModel): 1797 type: Literal["GzipDecoder"] 1798 decoder: Union[CsvDecoder, GzipDecoder, JsonDecoder, JsonlDecoder] 1799 1800 1801class Spec(BaseModel): 1802 type: Literal["Spec"] 1803 connection_specification: Dict[str, Any] = Field( 1804 ..., 1805 description="A connection specification describing how a the connector can be configured.", 1806 title="Connection Specification", 1807 ) 1808 documentation_url: Optional[str] = Field( 1809 None, 1810 description="URL of the connector's documentation page.", 1811 examples=["https://docs.airbyte.com/integrations/sources/dremio"], 1812 title="Documentation URL", 1813 ) 1814 advanced_auth: Optional[AuthFlow] = Field( 1815 None, 1816 description="Advanced specification for configuring the authentication flow.", 1817 title="Advanced Auth", 1818 ) 1819 1820 1821class CompositeErrorHandler(BaseModel): 1822 type: Literal["CompositeErrorHandler"] 1823 error_handlers: List[Union[CompositeErrorHandler, DefaultErrorHandler]] = Field( 1824 ..., 1825 description="List of error handlers to iterate on to determine how to handle a failed response.", 1826 title="Error Handlers", 1827 ) 1828 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters") 1829 1830 1831class HTTPAPIBudget(BaseModel): 1832 class Config: 1833 extra = Extra.allow 1834 1835 type: Literal["HTTPAPIBudget"] 1836 policies: List[ 1837 Union[ 1838 FixedWindowCallRatePolicy, 1839 MovingWindowCallRatePolicy, 1840 UnlimitedCallRatePolicy, 1841 ] 1842 ] = Field( 1843 ..., 1844 description="List of call rate policies that define how many calls are allowed.", 1845 title="Policies", 1846 ) 1847 ratelimit_reset_header: Optional[str] = Field( 1848 "ratelimit-reset", 1849 description="The HTTP response header name that indicates when the rate limit resets.", 1850 title="Rate Limit Reset Header", 1851 ) 1852 ratelimit_remaining_header: Optional[str] = Field( 1853 "ratelimit-remaining", 1854 description="The HTTP response header name that indicates the number of remaining allowed calls.", 1855 title="Rate Limit Remaining Header", 1856 ) 1857 status_codes_for_ratelimit_hit: Optional[List[int]] = Field( 1858 [429], 1859 description="List of HTTP status codes that indicate a rate limit has been hit.", 1860 title="Status Codes for Rate Limit Hit", 1861 ) 1862 1863 1864class ZipfileDecoder(BaseModel): 1865 class Config: 1866 extra = Extra.allow 1867 1868 type: Literal["ZipfileDecoder"] 1869 decoder: Union[CsvDecoder, GzipDecoder, JsonDecoder, JsonlDecoder] = Field( 1870 ..., 1871 description="Parser to parse the decompressed data from the zipfile(s).", 1872 title="Parser", 1873 ) 1874 1875 1876class DeclarativeSource1(BaseModel): 1877 class Config: 1878 extra = Extra.forbid 1879 1880 type: Literal["DeclarativeSource"] 1881 check: Union[CheckStream, CheckDynamicStream] 1882 streams: List[Union[DeclarativeStream, StateDelegatingStream]] 1883 dynamic_streams: Optional[List[DynamicDeclarativeStream]] = None 1884 version: str = Field( 1885 ..., 1886 description="The version of the Airbyte CDK used to build and test the source.", 1887 ) 1888 schemas: Optional[Schemas] = None 1889 definitions: Optional[Dict[str, Any]] = None 1890 spec: Optional[Spec] = None 1891 concurrency_level: Optional[ConcurrencyLevel] = None 1892 api_budget: Optional[HTTPAPIBudget] = None 1893 max_concurrent_async_job_count: Optional[Union[int, str]] = Field( 1894 None, 1895 description="Maximum number of concurrent asynchronous jobs to run. This property is only relevant for sources/streams that support asynchronous job execution through the AsyncRetriever (e.g. a report-based stream that initiates a job, polls the job status, and then fetches the job results). This is often set by the API's maximum number of concurrent jobs on the account level. Refer to the API's documentation for this information.", 1896 examples=[3, "{{ config['max_concurrent_async_job_count'] }}"], 1897 title="Maximum Concurrent Asynchronous Jobs", 1898 ) 1899 metadata: Optional[Dict[str, Any]] = Field( 1900 None, 1901 description="For internal Airbyte use only - DO NOT modify manually. Used by consumers of declarative manifests for storing related metadata.", 1902 ) 1903 description: Optional[str] = Field( 1904 None, 1905 description="A description of the connector. It will be presented on the Source documentation page.", 1906 ) 1907 1908 1909class DeclarativeSource2(BaseModel): 1910 class Config: 1911 extra = Extra.forbid 1912 1913 type: Literal["DeclarativeSource"] 1914 check: Union[CheckStream, CheckDynamicStream] 1915 streams: Optional[List[Union[DeclarativeStream, StateDelegatingStream]]] = None 1916 dynamic_streams: List[DynamicDeclarativeStream] 1917 version: str = Field( 1918 ..., 1919 description="The version of the Airbyte CDK used to build and test the source.", 1920 ) 1921 schemas: Optional[Schemas] = None 1922 definitions: Optional[Dict[str, Any]] = None 1923 spec: Optional[Spec] = None 1924 concurrency_level: Optional[ConcurrencyLevel] = None 1925 api_budget: Optional[HTTPAPIBudget] = None 1926 max_concurrent_async_job_count: Optional[Union[int, str]] = Field( 1927 None, 1928 description="Maximum number of concurrent asynchronous jobs to run. This property is only relevant for sources/streams that support asynchronous job execution through the AsyncRetriever (e.g. a report-based stream that initiates a job, polls the job status, and then fetches the job results). This is often set by the API's maximum number of concurrent jobs on the account level. Refer to the API's documentation for this information.", 1929 examples=[3, "{{ config['max_concurrent_async_job_count'] }}"], 1930 title="Maximum Concurrent Asynchronous Jobs", 1931 ) 1932 metadata: Optional[Dict[str, Any]] = Field( 1933 None, 1934 description="For internal Airbyte use only - DO NOT modify manually. Used by consumers of declarative manifests for storing related metadata.", 1935 ) 1936 description: Optional[str] = Field( 1937 None, 1938 description="A description of the connector. It will be presented on the Source documentation page.", 1939 ) 1940 1941 1942class DeclarativeSource(BaseModel): 1943 class Config: 1944 extra = Extra.forbid 1945 1946 __root__: Union[DeclarativeSource1, DeclarativeSource2] = Field( 1947 ..., 1948 description="An API source that extracts data according to its declarative components.", 1949 title="DeclarativeSource", 1950 ) 1951 1952 1953class SelectiveAuthenticator(BaseModel): 1954 class Config: 1955 extra = Extra.allow 1956 1957 type: Literal["SelectiveAuthenticator"] 1958 authenticator_selection_path: List[str] = Field( 1959 ..., 1960 description="Path of the field in config with selected authenticator name", 1961 examples=[["auth"], ["auth", "type"]], 1962 title="Authenticator Selection Path", 1963 ) 1964 authenticators: Dict[ 1965 str, 1966 Union[ 1967 ApiKeyAuthenticator, 1968 BasicHttpAuthenticator, 1969 BearerAuthenticator, 1970 CustomAuthenticator, 1971 OAuthAuthenticator, 1972 JwtAuthenticator, 1973 NoAuth, 1974 SessionTokenAuthenticator, 1975 LegacySessionTokenAuthenticator, 1976 ], 1977 ] = Field( 1978 ..., 1979 description="Authenticators to select from.", 1980 examples=[ 1981 { 1982 "authenticators": { 1983 "token": "#/definitions/ApiKeyAuthenticator", 1984 "oauth": "#/definitions/OAuthAuthenticator", 1985 "jwt": "#/definitions/JwtAuthenticator", 1986 } 1987 } 1988 ], 1989 title="Authenticators", 1990 ) 1991 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters") 1992 1993 1994class DeclarativeStream(BaseModel): 1995 class Config: 1996 extra = Extra.allow 1997 1998 type: Literal["DeclarativeStream"] 1999 retriever: Union[AsyncRetriever, CustomRetriever, SimpleRetriever] = Field( 2000 ..., 2001 description="Component used to coordinate how records are extracted across stream slices and request pages.", 2002 title="Retriever", 2003 ) 2004 incremental_sync: Optional[ 2005 Union[CustomIncrementalSync, DatetimeBasedCursor, IncrementingCountCursor] 2006 ] = Field( 2007 None, 2008 description="Component used to fetch data incrementally based on a time field in the data.", 2009 title="Incremental Sync", 2010 ) 2011 name: Optional[str] = Field("", description="The stream name.", example=["Users"], title="Name") 2012 primary_key: Optional[PrimaryKey] = Field( 2013 "", description="The primary key of the stream.", title="Primary Key" 2014 ) 2015 schema_loader: Optional[ 2016 Union[ 2017 DynamicSchemaLoader, 2018 InlineSchemaLoader, 2019 JsonFileSchemaLoader, 2020 CustomSchemaLoader, 2021 ] 2022 ] = Field( 2023 None, 2024 description="Component used to retrieve the schema for the current stream.", 2025 title="Schema Loader", 2026 ) 2027 transformations: Optional[ 2028 List[ 2029 Union[ 2030 AddFields, 2031 CustomTransformation, 2032 RemoveFields, 2033 KeysToLower, 2034 KeysToSnakeCase, 2035 FlattenFields, 2036 DpathFlattenFields, 2037 KeysReplace, 2038 ] 2039 ] 2040 ] = Field( 2041 None, 2042 description="A list of transformations to be applied to each output record.", 2043 title="Transformations", 2044 ) 2045 state_migrations: Optional[ 2046 List[Union[LegacyToPerPartitionStateMigration, CustomStateMigration]] 2047 ] = Field( 2048 [], 2049 description="Array of state migrations to be applied on the input state", 2050 title="State Migrations", 2051 ) 2052 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters") 2053 2054 2055class SessionTokenAuthenticator(BaseModel): 2056 type: Literal["SessionTokenAuthenticator"] 2057 login_requester: HttpRequester = Field( 2058 ..., 2059 description="Description of the request to perform to obtain a session token to perform data requests. The response body is expected to be a JSON object with a session token property.", 2060 examples=[ 2061 { 2062 "type": "HttpRequester", 2063 "url_base": "https://my_api.com", 2064 "path": "/login", 2065 "authenticator": { 2066 "type": "BasicHttpAuthenticator", 2067 "username": "{{ config.username }}", 2068 "password": "{{ config.password }}", 2069 }, 2070 } 2071 ], 2072 title="Login Requester", 2073 ) 2074 session_token_path: List[str] = Field( 2075 ..., 2076 description="The path in the response body returned from the login requester to the session token.", 2077 examples=[["access_token"], ["result", "token"]], 2078 title="Session Token Path", 2079 ) 2080 expiration_duration: Optional[str] = Field( 2081 None, 2082 description="The duration in ISO 8601 duration notation after which the session token expires, starting from the time it was obtained. Omitting it will result in the session token being refreshed for every request.", 2083 examples=["PT1H", "P1D"], 2084 title="Expiration Duration", 2085 ) 2086 request_authentication: Union[ 2087 SessionTokenRequestApiKeyAuthenticator, SessionTokenRequestBearerAuthenticator 2088 ] = Field( 2089 ..., 2090 description="Authentication method to use for requests sent to the API, specifying how to inject the session token.", 2091 title="Data Request Authentication", 2092 ) 2093 decoder: Optional[Union[JsonDecoder, XmlDecoder]] = Field( 2094 None, description="Component used to decode the response.", title="Decoder" 2095 ) 2096 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters") 2097 2098 2099class HttpRequester(BaseModel): 2100 type: Literal["HttpRequester"] 2101 url_base: str = Field( 2102 ..., 2103 description="Base URL of the API source. Do not put sensitive information (e.g. API tokens) into this field - Use the Authentication component for this.", 2104 examples=[ 2105 "https://connect.squareup.com/v2", 2106 "{{ config['base_url'] or 'https://app.posthog.com'}}/api", 2107 "https://connect.squareup.com/v2/quotes/{{ stream_partition['id'] }}/quote_line_groups", 2108 "https://example.com/api/v1/resource/{{ next_page_token['id'] }}", 2109 ], 2110 title="API Base URL", 2111 ) 2112 path: Optional[str] = Field( 2113 None, 2114 description="Path the specific API endpoint that this stream represents. Do not put sensitive information (e.g. API tokens) into this field - Use the Authentication component for this.", 2115 examples=[ 2116 "/products", 2117 "/quotes/{{ stream_partition['id'] }}/quote_line_groups", 2118 "/trades/{{ config['symbol_id'] }}/history", 2119 ], 2120 title="URL Path", 2121 ) 2122 authenticator: Optional[ 2123 Union[ 2124 ApiKeyAuthenticator, 2125 BasicHttpAuthenticator, 2126 BearerAuthenticator, 2127 CustomAuthenticator, 2128 OAuthAuthenticator, 2129 JwtAuthenticator, 2130 NoAuth, 2131 SessionTokenAuthenticator, 2132 LegacySessionTokenAuthenticator, 2133 SelectiveAuthenticator, 2134 ] 2135 ] = Field( 2136 None, 2137 description="Authentication method to use for requests sent to the API.", 2138 title="Authenticator", 2139 ) 2140 error_handler: Optional[ 2141 Union[DefaultErrorHandler, CustomErrorHandler, CompositeErrorHandler] 2142 ] = Field( 2143 None, 2144 description="Error handler component that defines how to handle errors.", 2145 title="Error Handler", 2146 ) 2147 http_method: Optional[HttpMethod] = Field( 2148 HttpMethod.GET, 2149 description="The HTTP method used to fetch data from the source (can be GET or POST).", 2150 examples=["GET", "POST"], 2151 title="HTTP Method", 2152 ) 2153 request_body_data: Optional[Union[str, Dict[str, str]]] = Field( 2154 None, 2155 description="Specifies how to populate the body of the request with a non-JSON payload. Plain text will be sent as is, whereas objects will be converted to a urlencoded form.", 2156 examples=[ 2157 '[{"clause": {"type": "timestamp", "operator": 10, "parameters":\n [{"value": {{ stream_interval[\'start_time\'] | int * 1000 }} }]\n }, "orderBy": 1, "columnName": "Timestamp"}]/\n' 2158 ], 2159 title="Request Body Payload (Non-JSON)", 2160 ) 2161 request_body_json: Optional[Union[str, Dict[str, Any]]] = Field( 2162 None, 2163 description="Specifies how to populate the body of the request with a JSON payload. Can contain nested objects.", 2164 examples=[ 2165 {"sort_order": "ASC", "sort_field": "CREATED_AT"}, 2166 {"key": "{{ config['value'] }}"}, 2167 {"sort": {"field": "updated_at", "order": "ascending"}}, 2168 ], 2169 title="Request Body JSON Payload", 2170 ) 2171 request_headers: Optional[Union[str, Dict[str, str]]] = Field( 2172 None, 2173 description="Return any non-auth headers. Authentication headers will overwrite any overlapping headers returned from this method.", 2174 examples=[{"Output-Format": "JSON"}, {"Version": "{{ config['version'] }}"}], 2175 title="Request Headers", 2176 ) 2177 request_parameters: Optional[Union[str, Dict[str, str]]] = Field( 2178 None, 2179 description="Specifies the query parameters that should be set on an outgoing HTTP request given the inputs.", 2180 examples=[ 2181 {"unit": "day"}, 2182 { 2183 "query": 'last_event_time BETWEEN TIMESTAMP "{{ stream_interval.start_time }}" AND TIMESTAMP "{{ stream_interval.end_time }}"' 2184 }, 2185 {"searchIn": "{{ ','.join(config.get('search_in', [])) }}"}, 2186 {"sort_by[asc]": "updated_at"}, 2187 ], 2188 title="Query Parameters", 2189 ) 2190 use_cache: Optional[bool] = Field( 2191 False, 2192 description="Enables stream requests caching. This field is automatically set by the CDK.", 2193 title="Use Cache", 2194 ) 2195 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters") 2196 2197 2198class DynamicSchemaLoader(BaseModel): 2199 type: Literal["DynamicSchemaLoader"] 2200 retriever: Union[AsyncRetriever, CustomRetriever, SimpleRetriever] = Field( 2201 ..., 2202 description="Component used to coordinate how records are extracted across stream slices and request pages.", 2203 title="Retriever", 2204 ) 2205 schema_transformations: Optional[ 2206 List[ 2207 Union[ 2208 AddFields, 2209 CustomTransformation, 2210 RemoveFields, 2211 KeysToLower, 2212 KeysToSnakeCase, 2213 FlattenFields, 2214 DpathFlattenFields, 2215 KeysReplace, 2216 ] 2217 ] 2218 ] = Field( 2219 None, 2220 description="A list of transformations to be applied to the schema.", 2221 title="Schema Transformations", 2222 ) 2223 schema_type_identifier: SchemaTypeIdentifier 2224 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters") 2225 2226 2227class ParentStreamConfig(BaseModel): 2228 type: Literal["ParentStreamConfig"] 2229 lazy_read_pointer: Optional[List[str]] = Field( 2230 [], 2231 description="If set, this will enable lazy reading, using the initial read of parent records to extract child records.", 2232 title="Lazy Read Pointer", 2233 ) 2234 parent_key: str = Field( 2235 ..., 2236 description="The primary key of records from the parent stream that will be used during the retrieval of records for the current substream. This parent identifier field is typically a characteristic of the child records being extracted from the source API.", 2237 examples=["id", "{{ config['parent_record_id'] }}"], 2238 title="Parent Key", 2239 ) 2240 stream: Union[DeclarativeStream, StateDelegatingStream] = Field( 2241 ..., description="Reference to the parent stream.", title="Parent Stream" 2242 ) 2243 partition_field: str = Field( 2244 ..., 2245 description="While iterating over parent records during a sync, the parent_key value can be referenced by using this field.", 2246 examples=["parent_id", "{{ config['parent_partition_field'] }}"], 2247 title="Current Parent Key Value Identifier", 2248 ) 2249 request_option: Optional[RequestOption] = Field( 2250 None, 2251 description="A request option describing where the parent key value should be injected into and under what field name if applicable.", 2252 title="Request Option", 2253 ) 2254 incremental_dependency: Optional[bool] = Field( 2255 False, 2256 description="Indicates whether the parent stream should be read incrementally based on updates in the child stream.", 2257 title="Incremental Dependency", 2258 ) 2259 extra_fields: Optional[List[List[str]]] = Field( 2260 None, 2261 description="Array of field paths to include as additional fields in the stream slice. Each path is an array of strings representing keys to access fields in the respective parent record. Accessible via `stream_slice.extra_fields`. Missing fields are set to `None`.", 2262 title="Extra Fields", 2263 ) 2264 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters") 2265 2266 2267class StateDelegatingStream(BaseModel): 2268 type: Literal["StateDelegatingStream"] 2269 name: str = Field(..., description="The stream name.", example=["Users"], title="Name") 2270 full_refresh_stream: DeclarativeStream = Field( 2271 ..., 2272 description="Component used to coordinate how records are extracted across stream slices and request pages when the state is empty or not provided.", 2273 title="Retriever", 2274 ) 2275 incremental_stream: DeclarativeStream = Field( 2276 ..., 2277 description="Component used to coordinate how records are extracted across stream slices and request pages when the state provided.", 2278 title="Retriever", 2279 ) 2280 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters") 2281 2282 2283class SimpleRetriever(BaseModel): 2284 type: Literal["SimpleRetriever"] 2285 record_selector: RecordSelector = Field( 2286 ..., 2287 description="Component that describes how to extract records from a HTTP response.", 2288 ) 2289 requester: Union[CustomRequester, HttpRequester] = Field( 2290 ..., 2291 description="Requester component that describes how to prepare HTTP requests to send to the source API.", 2292 ) 2293 paginator: Optional[Union[DefaultPaginator, NoPagination]] = Field( 2294 None, 2295 description="Paginator component that describes how to navigate through the API's pages.", 2296 ) 2297 ignore_stream_slicer_parameters_on_paginated_requests: Optional[bool] = Field( 2298 False, 2299 description="If true, the partition router and incremental request options will be ignored when paginating requests. Request options set directly on the requester will not be ignored.", 2300 ) 2301 partition_router: Optional[ 2302 Union[ 2303 CustomPartitionRouter, 2304 ListPartitionRouter, 2305 SubstreamPartitionRouter, 2306 List[Union[CustomPartitionRouter, ListPartitionRouter, SubstreamPartitionRouter]], 2307 ] 2308 ] = Field( 2309 [], 2310 description="PartitionRouter component that describes how to partition the stream, enabling incremental syncs and checkpointing.", 2311 title="Partition Router", 2312 ) 2313 decoder: Optional[ 2314 Union[ 2315 CustomDecoder, 2316 CsvDecoder, 2317 GzipDecoder, 2318 JsonDecoder, 2319 JsonlDecoder, 2320 IterableDecoder, 2321 XmlDecoder, 2322 ZipfileDecoder, 2323 ] 2324 ] = Field( 2325 None, 2326 description="Component decoding the response so records can be extracted.", 2327 title="Decoder", 2328 ) 2329 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters") 2330 2331 2332class AsyncRetriever(BaseModel): 2333 type: Literal["AsyncRetriever"] 2334 record_selector: RecordSelector = Field( 2335 ..., 2336 description="Component that describes how to extract records from a HTTP response.", 2337 ) 2338 status_mapping: AsyncJobStatusMap = Field( 2339 ..., description="Async Job Status to Airbyte CDK Async Job Status mapping." 2340 ) 2341 status_extractor: Union[CustomRecordExtractor, DpathExtractor] = Field( 2342 ..., description="Responsible for fetching the actual status of the async job." 2343 ) 2344 download_target_extractor: Union[CustomRecordExtractor, DpathExtractor] = Field( 2345 ..., 2346 description="Responsible for fetching the final result `urls` provided by the completed / finished / ready async job.", 2347 ) 2348 download_extractor: Optional[ 2349 Union[CustomRecordExtractor, DpathExtractor, ResponseToFileExtractor] 2350 ] = Field(None, description="Responsible for fetching the records from provided urls.") 2351 creation_requester: Union[CustomRequester, HttpRequester] = Field( 2352 ..., 2353 description="Requester component that describes how to prepare HTTP requests to send to the source API to create the async server-side job.", 2354 ) 2355 polling_requester: Union[CustomRequester, HttpRequester] = Field( 2356 ..., 2357 description="Requester component that describes how to prepare HTTP requests to send to the source API to fetch the status of the running async job.", 2358 ) 2359 polling_job_timeout: Optional[Union[int, str]] = Field( 2360 None, 2361 description="The time in minutes after which the single Async Job should be considered as Timed Out.", 2362 ) 2363 download_target_requester: Optional[Union[CustomRequester, HttpRequester]] = Field( 2364 None, 2365 description="Requester component that describes how to prepare HTTP requests to send to the source API to extract the url from polling response by the completed async job.", 2366 ) 2367 download_requester: Union[CustomRequester, HttpRequester] = Field( 2368 ..., 2369 description="Requester component that describes how to prepare HTTP requests to send to the source API to download the data provided by the completed async job.", 2370 ) 2371 download_paginator: Optional[Union[DefaultPaginator, NoPagination]] = Field( 2372 None, 2373 description="Paginator component that describes how to navigate through the API's pages during download.", 2374 ) 2375 abort_requester: Optional[Union[CustomRequester, HttpRequester]] = Field( 2376 None, 2377 description="Requester component that describes how to prepare HTTP requests to send to the source API to abort a job once it is timed out from the source's perspective.", 2378 ) 2379 delete_requester: Optional[Union[CustomRequester, HttpRequester]] = Field( 2380 None, 2381 description="Requester component that describes how to prepare HTTP requests to send to the source API to delete a job once the records are extracted.", 2382 ) 2383 partition_router: Optional[ 2384 Union[ 2385 CustomPartitionRouter, 2386 ListPartitionRouter, 2387 SubstreamPartitionRouter, 2388 List[Union[CustomPartitionRouter, ListPartitionRouter, SubstreamPartitionRouter]], 2389 ] 2390 ] = Field( 2391 [], 2392 description="PartitionRouter component that describes how to partition the stream, enabling incremental syncs and checkpointing.", 2393 title="Partition Router", 2394 ) 2395 decoder: Optional[ 2396 Union[ 2397 CustomDecoder, 2398 CsvDecoder, 2399 GzipDecoder, 2400 JsonDecoder, 2401 JsonlDecoder, 2402 IterableDecoder, 2403 XmlDecoder, 2404 ZipfileDecoder, 2405 ] 2406 ] = Field( 2407 None, 2408 description="Component decoding the response so records can be extracted.", 2409 title="Decoder", 2410 ) 2411 download_decoder: Optional[ 2412 Union[ 2413 CustomDecoder, 2414 CsvDecoder, 2415 GzipDecoder, 2416 JsonDecoder, 2417 JsonlDecoder, 2418 IterableDecoder, 2419 XmlDecoder, 2420 ZipfileDecoder, 2421 ] 2422 ] = Field( 2423 None, 2424 description="Component decoding the download response so records can be extracted.", 2425 title="Download Decoder", 2426 ) 2427 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters") 2428 2429 2430class SubstreamPartitionRouter(BaseModel): 2431 type: Literal["SubstreamPartitionRouter"] 2432 parent_stream_configs: List[ParentStreamConfig] = Field( 2433 ..., 2434 description="Specifies which parent streams are being iterated over and how parent records should be used to partition the child stream data set.", 2435 title="Parent Stream Configs", 2436 ) 2437 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters") 2438 2439 2440class HttpComponentsResolver(BaseModel): 2441 type: Literal["HttpComponentsResolver"] 2442 retriever: Union[AsyncRetriever, CustomRetriever, SimpleRetriever] = Field( 2443 ..., 2444 description="Component used to coordinate how records are extracted across stream slices and request pages.", 2445 title="Retriever", 2446 ) 2447 components_mapping: List[ComponentMappingDefinition] 2448 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters") 2449 2450 2451class DynamicDeclarativeStream(BaseModel): 2452 type: Literal["DynamicDeclarativeStream"] 2453 stream_template: DeclarativeStream = Field( 2454 ..., description="Reference to the stream template.", title="Stream Template" 2455 ) 2456 components_resolver: Union[HttpComponentsResolver, ConfigComponentsResolver] = Field( 2457 ..., 2458 description="Component resolve and populates stream templates with components values.", 2459 title="Components Resolver", 2460 ) 2461 2462 2463ComplexFieldType.update_forward_refs() 2464GzipDecoder.update_forward_refs() 2465CompositeErrorHandler.update_forward_refs() 2466DeclarativeSource1.update_forward_refs() 2467DeclarativeSource2.update_forward_refs() 2468SelectiveAuthenticator.update_forward_refs() 2469DeclarativeStream.update_forward_refs() 2470SessionTokenAuthenticator.update_forward_refs() 2471DynamicSchemaLoader.update_forward_refs() 2472ParentStreamConfig.update_forward_refs() 2473SimpleRetriever.update_forward_refs() 2474AsyncRetriever.update_forward_refs()
class
AuthFlowType(enum.Enum):
An enumeration.
oauth2_0 =
<AuthFlowType.oauth2_0: 'oauth2.0'>
oauth1_0 =
<AuthFlowType.oauth1_0: 'oauth1.0'>
class
BasicHttpAuthenticator(pydantic.v1.main.BaseModel):
18class BasicHttpAuthenticator(BaseModel): 19 type: Literal["BasicHttpAuthenticator"] 20 username: str = Field( 21 ..., 22 description="The username that will be combined with the password, base64 encoded and used to make requests. Fill it in the user inputs.", 23 examples=["{{ config['username'] }}", "{{ config['api_key'] }}"], 24 title="Username", 25 ) 26 password: Optional[str] = Field( 27 "", 28 description="The password that will be combined with the username, base64 encoded and used to make requests. Fill it in the user inputs.", 29 examples=["{{ config['password'] }}", ""], 30 title="Password", 31 ) 32 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters")
class
BearerAuthenticator(pydantic.v1.main.BaseModel):
35class BearerAuthenticator(BaseModel): 36 type: Literal["BearerAuthenticator"] 37 api_token: str = Field( 38 ..., 39 description="Token to inject as request header for authenticating with the API.", 40 examples=["{{ config['api_key'] }}", "{{ config['token'] }}"], 41 title="Bearer Token", 42 ) 43 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters")
class
CheckStream(pydantic.v1.main.BaseModel):
class
CheckDynamicStream(pydantic.v1.main.BaseModel):
56class CheckDynamicStream(BaseModel): 57 type: Literal["CheckDynamicStream"] 58 stream_count: int = Field( 59 ..., 60 description="Numbers of the streams to try reading from when running a check operation.", 61 title="Stream Count", 62 ) 63 use_check_availability: Optional[bool] = Field( 64 True, 65 description="Enables stream check availability. This field is automatically set by the CDK.", 66 title="Use Check Availability", 67 )
class
ConcurrencyLevel(pydantic.v1.main.BaseModel):
70class ConcurrencyLevel(BaseModel): 71 type: Optional[Literal["ConcurrencyLevel"]] = None 72 default_concurrency: Union[int, str] = Field( 73 ..., 74 description="The amount of concurrency that will applied during a sync. This value can be hardcoded or user-defined in the config if different users have varying volume thresholds in the target API.", 75 examples=[10, "{{ config['num_workers'] or 10 }}"], 76 title="Default Concurrency", 77 ) 78 max_concurrency: Optional[int] = Field( 79 None, 80 description="The maximum level of concurrency that will be used during a sync. This becomes a required field when the default_concurrency derives from the config, because it serves as a safeguard against a user-defined threshold that is too high.", 81 examples=[20, 100], 82 title="Max Concurrency", 83 ) 84 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters")
class
ConstantBackoffStrategy(pydantic.v1.main.BaseModel):
87class ConstantBackoffStrategy(BaseModel): 88 type: Literal["ConstantBackoffStrategy"] 89 backoff_time_in_seconds: Union[float, str] = Field( 90 ..., 91 description="Backoff time in seconds.", 92 examples=[30, 30.5, "{{ config['backoff_time'] }}"], 93 title="Backoff Time", 94 ) 95 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters")
class
CursorPagination(pydantic.v1.main.BaseModel):
98class CursorPagination(BaseModel): 99 type: Literal["CursorPagination"] 100 cursor_value: str = Field( 101 ..., 102 description="Value of the cursor defining the next page to fetch.", 103 examples=[ 104 "{{ headers.link.next.cursor }}", 105 "{{ last_record['key'] }}", 106 "{{ response['nextPage'] }}", 107 ], 108 title="Cursor Value", 109 ) 110 page_size: Optional[int] = Field( 111 None, 112 description="The number of records to include in each pages.", 113 examples=[100], 114 title="Page Size", 115 ) 116 stop_condition: Optional[str] = Field( 117 None, 118 description="Template string evaluating when to stop paginating.", 119 examples=[ 120 "{{ response.data.has_more is false }}", 121 "{{ 'next' not in headers['link'] }}", 122 ], 123 title="Stop Condition", 124 ) 125 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters")
class
CustomAuthenticator(pydantic.v1.main.BaseModel):
128class CustomAuthenticator(BaseModel): 129 class Config: 130 extra = Extra.allow 131 132 type: Literal["CustomAuthenticator"] 133 class_name: str = Field( 134 ..., 135 description="Fully-qualified name of the class that will be implementing the custom authentication strategy. Has to be a sub class of DeclarativeAuthenticator. The format is `source_<name>.<package>.<class_name>`.", 136 examples=["source_railz.components.ShortLivedTokenAuthenticator"], 137 title="Class Name", 138 ) 139 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters")
class
CustomAuthenticator.Config:
class
CustomBackoffStrategy(pydantic.v1.main.BaseModel):
142class CustomBackoffStrategy(BaseModel): 143 class Config: 144 extra = Extra.allow 145 146 type: Literal["CustomBackoffStrategy"] 147 class_name: str = Field( 148 ..., 149 description="Fully-qualified name of the class that will be implementing the custom backoff strategy. The format is `source_<name>.<package>.<class_name>`.", 150 examples=["source_railz.components.MyCustomBackoffStrategy"], 151 title="Class Name", 152 ) 153 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters")
class
CustomBackoffStrategy.Config:
class
CustomErrorHandler(pydantic.v1.main.BaseModel):
156class CustomErrorHandler(BaseModel): 157 class Config: 158 extra = Extra.allow 159 160 type: Literal["CustomErrorHandler"] 161 class_name: str = Field( 162 ..., 163 description="Fully-qualified name of the class that will be implementing the custom error handler. The format is `source_<name>.<package>.<class_name>`.", 164 examples=["source_railz.components.MyCustomErrorHandler"], 165 title="Class Name", 166 ) 167 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters")
class
CustomErrorHandler.Config:
class
CustomIncrementalSync(pydantic.v1.main.BaseModel):
170class CustomIncrementalSync(BaseModel): 171 class Config: 172 extra = Extra.allow 173 174 type: Literal["CustomIncrementalSync"] 175 class_name: str = Field( 176 ..., 177 description="Fully-qualified name of the class that will be implementing the custom incremental sync. The format is `source_<name>.<package>.<class_name>`.", 178 examples=["source_railz.components.MyCustomIncrementalSync"], 179 title="Class Name", 180 ) 181 cursor_field: str = Field( 182 ..., 183 description="The location of the value on a record that will be used as a bookmark during sync.", 184 ) 185 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters")
class
CustomIncrementalSync.Config:
class
CustomPaginationStrategy(pydantic.v1.main.BaseModel):
188class CustomPaginationStrategy(BaseModel): 189 class Config: 190 extra = Extra.allow 191 192 type: Literal["CustomPaginationStrategy"] 193 class_name: str = Field( 194 ..., 195 description="Fully-qualified name of the class that will be implementing the custom pagination strategy. The format is `source_<name>.<package>.<class_name>`.", 196 examples=["source_railz.components.MyCustomPaginationStrategy"], 197 title="Class Name", 198 ) 199 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters")
class
CustomPaginationStrategy.Config:
class
CustomRecordExtractor(pydantic.v1.main.BaseModel):
202class CustomRecordExtractor(BaseModel): 203 class Config: 204 extra = Extra.allow 205 206 type: Literal["CustomRecordExtractor"] 207 class_name: str = Field( 208 ..., 209 description="Fully-qualified name of the class that will be implementing the custom record extraction strategy. The format is `source_<name>.<package>.<class_name>`.", 210 examples=["source_railz.components.MyCustomRecordExtractor"], 211 title="Class Name", 212 ) 213 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters")
class
CustomRecordExtractor.Config:
class
CustomRecordFilter(pydantic.v1.main.BaseModel):
216class CustomRecordFilter(BaseModel): 217 class Config: 218 extra = Extra.allow 219 220 type: Literal["CustomRecordFilter"] 221 class_name: str = Field( 222 ..., 223 description="Fully-qualified name of the class that will be implementing the custom record filter strategy. The format is `source_<name>.<package>.<class_name>`.", 224 examples=["source_railz.components.MyCustomCustomRecordFilter"], 225 title="Class Name", 226 ) 227 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters")
class
CustomRecordFilter.Config:
class
CustomRequester(pydantic.v1.main.BaseModel):
230class CustomRequester(BaseModel): 231 class Config: 232 extra = Extra.allow 233 234 type: Literal["CustomRequester"] 235 class_name: str = Field( 236 ..., 237 description="Fully-qualified name of the class that will be implementing the custom requester strategy. The format is `source_<name>.<package>.<class_name>`.", 238 examples=["source_railz.components.MyCustomRecordExtractor"], 239 title="Class Name", 240 ) 241 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters")
class
CustomRequester.Config:
class
CustomRetriever(pydantic.v1.main.BaseModel):
244class CustomRetriever(BaseModel): 245 class Config: 246 extra = Extra.allow 247 248 type: Literal["CustomRetriever"] 249 class_name: str = Field( 250 ..., 251 description="Fully-qualified name of the class that will be implementing the custom retriever strategy. The format is `source_<name>.<package>.<class_name>`.", 252 examples=["source_railz.components.MyCustomRetriever"], 253 title="Class Name", 254 ) 255 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters")
class
CustomRetriever.Config:
class
CustomPartitionRouter(pydantic.v1.main.BaseModel):
258class CustomPartitionRouter(BaseModel): 259 class Config: 260 extra = Extra.allow 261 262 type: Literal["CustomPartitionRouter"] 263 class_name: str = Field( 264 ..., 265 description="Fully-qualified name of the class that will be implementing the custom partition router. The format is `source_<name>.<package>.<class_name>`.", 266 examples=["source_railz.components.MyCustomPartitionRouter"], 267 title="Class Name", 268 ) 269 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters")
class
CustomPartitionRouter.Config:
class
CustomSchemaLoader(pydantic.v1.main.BaseModel):
272class CustomSchemaLoader(BaseModel): 273 class Config: 274 extra = Extra.allow 275 276 type: Literal["CustomSchemaLoader"] 277 class_name: str = Field( 278 ..., 279 description="Fully-qualified name of the class that will be implementing the custom schema loader. The format is `source_<name>.<package>.<class_name>`.", 280 examples=["source_railz.components.MyCustomSchemaLoader"], 281 title="Class Name", 282 ) 283 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters")
class
CustomSchemaLoader.Config:
class
CustomSchemaNormalization(pydantic.v1.main.BaseModel):
286class CustomSchemaNormalization(BaseModel): 287 class Config: 288 extra = Extra.allow 289 290 type: Literal["CustomSchemaNormalization"] 291 class_name: str = Field( 292 ..., 293 description="Fully-qualified name of the class that will be implementing the custom normalization. The format is `source_<name>.<package>.<class_name>`.", 294 examples=[ 295 "source_amazon_seller_partner.components.LedgerDetailedViewReportsTypeTransformer" 296 ], 297 title="Class Name", 298 ) 299 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters")
class
CustomSchemaNormalization.Config:
class
CustomStateMigration(pydantic.v1.main.BaseModel):
302class CustomStateMigration(BaseModel): 303 class Config: 304 extra = Extra.allow 305 306 type: Literal["CustomStateMigration"] 307 class_name: str = Field( 308 ..., 309 description="Fully-qualified name of the class that will be implementing the custom state migration. The format is `source_<name>.<package>.<class_name>`.", 310 examples=["source_railz.components.MyCustomStateMigration"], 311 title="Class Name", 312 ) 313 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters")
class
CustomStateMigration.Config:
class
CustomTransformation(pydantic.v1.main.BaseModel):
316class CustomTransformation(BaseModel): 317 class Config: 318 extra = Extra.allow 319 320 type: Literal["CustomTransformation"] 321 class_name: str = Field( 322 ..., 323 description="Fully-qualified name of the class that will be implementing the custom transformation. The format is `source_<name>.<package>.<class_name>`.", 324 examples=["source_railz.components.MyCustomTransformation"], 325 title="Class Name", 326 ) 327 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters")
class
CustomTransformation.Config:
class
LegacyToPerPartitionStateMigration(pydantic.v1.main.BaseModel):
class
LegacyToPerPartitionStateMigration.Config:
class
Clamping(pydantic.v1.main.BaseModel):
class
Algorithm(enum.Enum):
347class Algorithm(Enum): 348 HS256 = "HS256" 349 HS384 = "HS384" 350 HS512 = "HS512" 351 ES256 = "ES256" 352 ES256K = "ES256K" 353 ES384 = "ES384" 354 ES512 = "ES512" 355 RS256 = "RS256" 356 RS384 = "RS384" 357 RS512 = "RS512" 358 PS256 = "PS256" 359 PS384 = "PS384" 360 PS512 = "PS512" 361 EdDSA = "EdDSA"
An enumeration.
HS256 =
<Algorithm.HS256: 'HS256'>
HS384 =
<Algorithm.HS384: 'HS384'>
HS512 =
<Algorithm.HS512: 'HS512'>
ES256 =
<Algorithm.ES256: 'ES256'>
ES256K =
<Algorithm.ES256K: 'ES256K'>
ES384 =
<Algorithm.ES384: 'ES384'>
ES512 =
<Algorithm.ES512: 'ES512'>
RS256 =
<Algorithm.RS256: 'RS256'>
RS384 =
<Algorithm.RS384: 'RS384'>
RS512 =
<Algorithm.RS512: 'RS512'>
PS256 =
<Algorithm.PS256: 'PS256'>
PS384 =
<Algorithm.PS384: 'PS384'>
PS512 =
<Algorithm.PS512: 'PS512'>
EdDSA =
<Algorithm.EdDSA: 'EdDSA'>
class
JwtHeaders(pydantic.v1.main.BaseModel):
364class JwtHeaders(BaseModel): 365 class Config: 366 extra = Extra.forbid 367 368 kid: Optional[str] = Field( 369 None, 370 description="Private key ID for user account.", 371 examples=["{{ config['kid'] }}"], 372 title="Key Identifier", 373 ) 374 typ: Optional[str] = Field( 375 "JWT", 376 description="The media type of the complete JWT.", 377 examples=["JWT"], 378 title="Type", 379 ) 380 cty: Optional[str] = Field( 381 None, 382 description="Content type of JWT header.", 383 examples=["JWT"], 384 title="Content Type", 385 )
class
JwtHeaders.Config:
class
JwtPayload(pydantic.v1.main.BaseModel):
388class JwtPayload(BaseModel): 389 class Config: 390 extra = Extra.forbid 391 392 iss: Optional[str] = Field( 393 None, 394 description="The user/principal that issued the JWT. Commonly a value unique to the user.", 395 examples=["{{ config['iss'] }}"], 396 title="Issuer", 397 ) 398 sub: Optional[str] = Field( 399 None, 400 description="The subject of the JWT. Commonly defined by the API.", 401 title="Subject", 402 ) 403 aud: Optional[str] = Field( 404 None, 405 description="The recipient that the JWT is intended for. Commonly defined by the API.", 406 examples=["appstoreconnect-v1"], 407 title="Audience", 408 )
class
JwtPayload.Config:
class
JwtAuthenticator(pydantic.v1.main.BaseModel):
411class JwtAuthenticator(BaseModel): 412 type: Literal["JwtAuthenticator"] 413 secret_key: str = Field( 414 ..., 415 description="Secret used to sign the JSON web token.", 416 examples=["{{ config['secret_key'] }}"], 417 ) 418 base64_encode_secret_key: Optional[bool] = Field( 419 False, 420 description='When set to true, the secret key will be base64 encoded prior to being encoded as part of the JWT. Only set to "true" when required by the API.', 421 ) 422 algorithm: Algorithm = Field( 423 ..., 424 description="Algorithm used to sign the JSON web token.", 425 examples=["ES256", "HS256", "RS256", "{{ config['algorithm'] }}"], 426 ) 427 token_duration: Optional[int] = Field( 428 1200, 429 description="The amount of time in seconds a JWT token can be valid after being issued.", 430 examples=[1200, 3600], 431 title="Token Duration", 432 ) 433 header_prefix: Optional[str] = Field( 434 None, 435 description="The prefix to be used within the Authentication header.", 436 examples=["Bearer", "Basic"], 437 title="Header Prefix", 438 ) 439 jwt_headers: Optional[JwtHeaders] = Field( 440 None, 441 description="JWT headers used when signing JSON web token.", 442 title="JWT Headers", 443 ) 444 additional_jwt_headers: Optional[Dict[str, Any]] = Field( 445 None, 446 description="Additional headers to be included with the JWT headers object.", 447 title="Additional JWT Headers", 448 ) 449 jwt_payload: Optional[JwtPayload] = Field( 450 None, 451 description="JWT Payload used when signing JSON web token.", 452 title="JWT Payload", 453 ) 454 additional_jwt_payload: Optional[Dict[str, Any]] = Field( 455 None, 456 description="Additional properties to be added to the JWT payload.", 457 title="Additional JWT Payload Properties", 458 ) 459 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters")
algorithm: Algorithm
jwt_headers: Optional[JwtHeaders]
jwt_payload: Optional[JwtPayload]
class
RefreshTokenUpdater(pydantic.v1.main.BaseModel):
462class RefreshTokenUpdater(BaseModel): 463 refresh_token_name: Optional[str] = Field( 464 "refresh_token", 465 description="The name of the property which contains the updated refresh token in the response from the token refresh endpoint.", 466 examples=["refresh_token"], 467 title="Refresh Token Property Name", 468 ) 469 access_token_config_path: Optional[List[str]] = Field( 470 ["credentials", "access_token"], 471 description="Config path to the access token. Make sure the field actually exists in the config.", 472 examples=[["credentials", "access_token"], ["access_token"]], 473 title="Config Path To Access Token", 474 ) 475 refresh_token_config_path: Optional[List[str]] = Field( 476 ["credentials", "refresh_token"], 477 description="Config path to the access token. Make sure the field actually exists in the config.", 478 examples=[["credentials", "refresh_token"], ["refresh_token"]], 479 title="Config Path To Refresh Token", 480 ) 481 token_expiry_date_config_path: Optional[List[str]] = Field( 482 ["credentials", "token_expiry_date"], 483 description="Config path to the expiry date. Make sure actually exists in the config.", 484 examples=[["credentials", "token_expiry_date"]], 485 title="Config Path To Expiry Date", 486 ) 487 refresh_token_error_status_codes: Optional[List[int]] = Field( 488 [], 489 description="Status Codes to Identify refresh token error in response (Refresh Token Error Key and Refresh Token Error Values should be also specified). Responses with one of the error status code and containing an error value will be flagged as a config error", 490 examples=[[400, 500]], 491 title="Refresh Token Error Status Codes", 492 ) 493 refresh_token_error_key: Optional[str] = Field( 494 "", 495 description="Key to Identify refresh token error in response (Refresh Token Error Status Codes and Refresh Token Error Values should be also specified).", 496 examples=["error"], 497 title="Refresh Token Error Key", 498 ) 499 refresh_token_error_values: Optional[List[str]] = Field( 500 [], 501 description='List of values to check for exception during token refresh process. Used to check if the error found in the response matches the key from the Refresh Token Error Key field (e.g. response={"error": "invalid_grant"}). Only responses with one of the error status code and containing an error value will be flagged as a config error', 502 examples=[["invalid_grant", "invalid_permissions"]], 503 title="Refresh Token Error Values", 504 )
class
OAuthAuthenticator(pydantic.v1.main.BaseModel):
507class OAuthAuthenticator(BaseModel): 508 type: Literal["OAuthAuthenticator"] 509 client_id_name: Optional[str] = Field( 510 "client_id", 511 description="The name of the property to use to refresh the `access_token`.", 512 examples=["custom_app_id"], 513 title="Client ID Property Name", 514 ) 515 client_id: Optional[str] = Field( 516 None, 517 description="The OAuth client ID. Fill it in the user inputs.", 518 examples=["{{ config['client_id }}", "{{ config['credentials']['client_id }}"], 519 title="Client ID", 520 ) 521 client_secret_name: Optional[str] = Field( 522 "client_secret", 523 description="The name of the property to use to refresh the `access_token`.", 524 examples=["custom_app_secret"], 525 title="Client Secret Property Name", 526 ) 527 client_secret: Optional[str] = Field( 528 None, 529 description="The OAuth client secret. Fill it in the user inputs.", 530 examples=[ 531 "{{ config['client_secret }}", 532 "{{ config['credentials']['client_secret }}", 533 ], 534 title="Client Secret", 535 ) 536 refresh_token_name: Optional[str] = Field( 537 "refresh_token", 538 description="The name of the property to use to refresh the `access_token`.", 539 examples=["custom_app_refresh_value"], 540 title="Refresh Token Property Name", 541 ) 542 refresh_token: Optional[str] = Field( 543 None, 544 description="Credential artifact used to get a new access token.", 545 examples=[ 546 "{{ config['refresh_token'] }}", 547 "{{ config['credentials]['refresh_token'] }}", 548 ], 549 title="Refresh Token", 550 ) 551 token_refresh_endpoint: Optional[str] = Field( 552 None, 553 description="The full URL to call to obtain a new access token.", 554 examples=["https://connect.squareup.com/oauth2/token"], 555 title="Token Refresh Endpoint", 556 ) 557 access_token_name: Optional[str] = Field( 558 "access_token", 559 description="The name of the property which contains the access token in the response from the token refresh endpoint.", 560 examples=["access_token"], 561 title="Access Token Property Name", 562 ) 563 access_token_value: Optional[str] = Field( 564 None, 565 description="The value of the access_token to bypass the token refreshing using `refresh_token`.", 566 examples=["secret_access_token_value"], 567 title="Access Token Value", 568 ) 569 expires_in_name: Optional[str] = Field( 570 "expires_in", 571 description="The name of the property which contains the expiry date in the response from the token refresh endpoint.", 572 examples=["expires_in"], 573 title="Token Expiry Property Name", 574 ) 575 grant_type_name: Optional[str] = Field( 576 "grant_type", 577 description="The name of the property to use to refresh the `access_token`.", 578 examples=["custom_grant_type"], 579 title="Grant Type Property Name", 580 ) 581 grant_type: Optional[str] = Field( 582 "refresh_token", 583 description="Specifies the OAuth2 grant type. If set to refresh_token, the refresh_token needs to be provided as well. For client_credentials, only client id and secret are required. Other grant types are not officially supported.", 584 examples=["refresh_token", "client_credentials"], 585 title="Grant Type", 586 ) 587 refresh_request_body: Optional[Dict[str, Any]] = Field( 588 None, 589 description="Body of the request sent to get a new access token.", 590 examples=[ 591 { 592 "applicationId": "{{ config['application_id'] }}", 593 "applicationSecret": "{{ config['application_secret'] }}", 594 "token": "{{ config['token'] }}", 595 } 596 ], 597 title="Refresh Request Body", 598 ) 599 refresh_request_headers: Optional[Dict[str, Any]] = Field( 600 None, 601 description="Headers of the request sent to get a new access token.", 602 examples=[ 603 { 604 "Authorization": "<AUTH_TOKEN>", 605 "Content-Type": "application/x-www-form-urlencoded", 606 } 607 ], 608 title="Refresh Request Headers", 609 ) 610 scopes: Optional[List[str]] = Field( 611 None, 612 description="List of scopes that should be granted to the access token.", 613 examples=[["crm.list.read", "crm.objects.contacts.read", "crm.schema.contacts.read"]], 614 title="Scopes", 615 ) 616 token_expiry_date: Optional[str] = Field( 617 None, 618 description="The access token expiry date.", 619 examples=["2023-04-06T07:12:10.421833+00:00", 1680842386], 620 title="Token Expiry Date", 621 ) 622 token_expiry_date_format: Optional[str] = Field( 623 None, 624 description="The format of the time to expiration datetime. Provide it if the time is returned as a date-time string instead of seconds.", 625 examples=["%Y-%m-%d %H:%M:%S.%f+00:00"], 626 title="Token Expiry Date Format", 627 ) 628 refresh_token_updater: Optional[RefreshTokenUpdater] = Field( 629 None, 630 description="When the token updater is defined, new refresh tokens, access tokens and the access token expiry date are written back from the authentication response to the config object. This is important if the refresh token can only used once.", 631 title="Token Updater", 632 ) 633 profile_assertion: Optional[JwtAuthenticator] = Field( 634 None, 635 description="The authenticator being used to authenticate the client authenticator.", 636 title="Profile Assertion", 637 ) 638 use_profile_assertion: Optional[bool] = Field( 639 False, 640 description="Enable using profile assertion as a flow for OAuth authorization.", 641 title="Use Profile Assertion", 642 ) 643 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters")
refresh_token_updater: Optional[RefreshTokenUpdater]
profile_assertion: Optional[JwtAuthenticator]
class
Rate(pydantic.v1.main.BaseModel):
646class Rate(BaseModel): 647 class Config: 648 extra = Extra.allow 649 650 limit: Union[int, str] = Field( 651 ..., 652 description="The maximum number of calls allowed within the interval.", 653 title="Limit", 654 ) 655 interval: str = Field( 656 ..., 657 description="The time interval for the rate limit.", 658 examples=["PT1H", "P1D"], 659 title="Interval", 660 )
class
Rate.Config:
class
HttpRequestRegexMatcher(pydantic.v1.main.BaseModel):
663class HttpRequestRegexMatcher(BaseModel): 664 class Config: 665 extra = Extra.allow 666 667 method: Optional[str] = Field( 668 None, description="The HTTP method to match (e.g., GET, POST).", title="Method" 669 ) 670 url_base: Optional[str] = Field( 671 None, 672 description='The base URL (scheme and host, e.g. "https://api.example.com") to match.', 673 title="URL Base", 674 ) 675 url_path_pattern: Optional[str] = Field( 676 None, 677 description="A regular expression pattern to match the URL path.", 678 title="URL Path Pattern", 679 ) 680 params: Optional[Dict[str, Any]] = Field( 681 None, description="The query parameters to match.", title="Parameters" 682 ) 683 headers: Optional[Dict[str, Any]] = Field( 684 None, description="The headers to match.", title="Headers" 685 )
class
HttpRequestRegexMatcher.Config:
class
DpathExtractor(pydantic.v1.main.BaseModel):
688class DpathExtractor(BaseModel): 689 type: Literal["DpathExtractor"] 690 field_path: List[str] = Field( 691 ..., 692 description='List of potentially nested fields describing the full path of the field to extract. Use "*" to extract all values from an array. See more info in the [docs](https://docs.airbyte.com/connector-development/config-based/understanding-the-yaml-file/record-selector).', 693 examples=[ 694 ["data"], 695 ["data", "records"], 696 ["data", "{{ parameters.name }}"], 697 ["data", "*", "record"], 698 ], 699 title="Field Path", 700 ) 701 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters")
class
ResponseToFileExtractor(pydantic.v1.main.BaseModel):
class
ExponentialBackoffStrategy(pydantic.v1.main.BaseModel):
709class ExponentialBackoffStrategy(BaseModel): 710 type: Literal["ExponentialBackoffStrategy"] 711 factor: Optional[Union[float, str]] = Field( 712 5, 713 description="Multiplicative constant applied on each retry.", 714 examples=[5, 5.5, "10"], 715 title="Factor", 716 ) 717 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters")
class
SessionTokenRequestBearerAuthenticator(pydantic.v1.main.BaseModel):
class
HttpMethod(enum.Enum):
An enumeration.
GET =
<HttpMethod.GET: 'GET'>
POST =
<HttpMethod.POST: 'POST'>
class
Action(enum.Enum):
729class Action(Enum): 730 SUCCESS = "SUCCESS" 731 FAIL = "FAIL" 732 RETRY = "RETRY" 733 IGNORE = "IGNORE" 734 RATE_LIMITED = "RATE_LIMITED"
An enumeration.
SUCCESS =
<Action.SUCCESS: 'SUCCESS'>
FAIL =
<Action.FAIL: 'FAIL'>
RETRY =
<Action.RETRY: 'RETRY'>
IGNORE =
<Action.IGNORE: 'IGNORE'>
RATE_LIMITED =
<Action.RATE_LIMITED: 'RATE_LIMITED'>
class
FailureType(enum.Enum):
737class FailureType(Enum): 738 system_error = "system_error" 739 config_error = "config_error" 740 transient_error = "transient_error"
An enumeration.
system_error =
<FailureType.system_error: 'system_error'>
config_error =
<FailureType.config_error: 'config_error'>
transient_error =
<FailureType.transient_error: 'transient_error'>
class
HttpResponseFilter(pydantic.v1.main.BaseModel):
743class HttpResponseFilter(BaseModel): 744 type: Literal["HttpResponseFilter"] 745 action: Optional[Action] = Field( 746 None, 747 description="Action to execute if a response matches the filter.", 748 examples=["SUCCESS", "FAIL", "RETRY", "IGNORE", "RATE_LIMITED"], 749 title="Action", 750 ) 751 failure_type: Optional[FailureType] = Field( 752 None, 753 description="Failure type of traced exception if a response matches the filter.", 754 examples=["system_error", "config_error", "transient_error"], 755 title="Failure Type", 756 ) 757 error_message: Optional[str] = Field( 758 None, 759 description="Error Message to display if the response matches the filter.", 760 title="Error Message", 761 ) 762 error_message_contains: Optional[str] = Field( 763 None, 764 description="Match the response if its error message contains the substring.", 765 example=["This API operation is not enabled for this site"], 766 title="Error Message Substring", 767 ) 768 http_codes: Optional[List[int]] = Field( 769 None, 770 description="Match the response if its HTTP code is included in this list.", 771 examples=[[420, 429], [500]], 772 title="HTTP Codes", 773 unique_items=True, 774 ) 775 predicate: Optional[str] = Field( 776 None, 777 description="Match the response if the predicate evaluates to true.", 778 examples=[ 779 "{{ 'Too much requests' in response }}", 780 "{{ 'error_code' in response and response['error_code'] == 'ComplexityException' }}", 781 ], 782 title="Predicate", 783 ) 784 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters")
action: Optional[Action]
failure_type: Optional[FailureType]
class
ComplexFieldType(pydantic.v1.main.BaseModel):
787class ComplexFieldType(BaseModel): 788 field_type: str 789 items: Optional[Union[str, ComplexFieldType]] = None
items: Union[str, ComplexFieldType, NoneType]
class
TypesMap(pydantic.v1.main.BaseModel):
792class TypesMap(BaseModel): 793 target_type: Union[str, List[str], ComplexFieldType] 794 current_type: Union[str, List[str]] 795 condition: Optional[str] = None
target_type: Union[str, List[str], ComplexFieldType]
class
SchemaTypeIdentifier(pydantic.v1.main.BaseModel):
798class SchemaTypeIdentifier(BaseModel): 799 type: Optional[Literal["SchemaTypeIdentifier"]] = None 800 schema_pointer: Optional[List[str]] = Field( 801 [], 802 description="List of nested fields defining the schema field path to extract. Defaults to [].", 803 title="Schema Path", 804 ) 805 key_pointer: List[str] = Field( 806 ..., 807 description="List of potentially nested fields describing the full path of the field key to extract.", 808 title="Key Path", 809 ) 810 type_pointer: Optional[List[str]] = Field( 811 None, 812 description="List of potentially nested fields describing the full path of the field type to extract.", 813 title="Type Path", 814 ) 815 types_mapping: Optional[List[TypesMap]] = None 816 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters")
types_mapping: Optional[List[TypesMap]]
class
InlineSchemaLoader(pydantic.v1.main.BaseModel):
819class InlineSchemaLoader(BaseModel): 820 type: Literal["InlineSchemaLoader"] 821 schema_: Optional[Dict[str, Any]] = Field( 822 None, 823 alias="schema", 824 description='Describes a streams\' schema. Refer to the <a href="https://docs.airbyte.com/understanding-airbyte/supported-data-types/">Data Types documentation</a> for more details on which types are valid.', 825 title="Schema", 826 )
class
JsonFileSchemaLoader(pydantic.v1.main.BaseModel):
829class JsonFileSchemaLoader(BaseModel): 830 type: Literal["JsonFileSchemaLoader"] 831 file_path: Optional[str] = Field( 832 None, 833 description="Path to the JSON file defining the schema. The path is relative to the connector module's root.", 834 example=["./schemas/users.json"], 835 title="File Path", 836 ) 837 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters")
class
JsonDecoder(pydantic.v1.main.BaseModel):
class
JsonlDecoder(pydantic.v1.main.BaseModel):
class
KeysToLower(pydantic.v1.main.BaseModel):
class
KeysToSnakeCase(pydantic.v1.main.BaseModel):
class
FlattenFields(pydantic.v1.main.BaseModel):
858class FlattenFields(BaseModel): 859 type: Literal["FlattenFields"] 860 flatten_lists: Optional[bool] = Field( 861 True, 862 description="Whether to flatten lists or leave it as is. Default is True.", 863 title="Flatten Lists", 864 ) 865 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters")
class
DpathFlattenFields(pydantic.v1.main.BaseModel):
868class DpathFlattenFields(BaseModel): 869 type: Literal["DpathFlattenFields"] 870 field_path: List[str] = Field( 871 ..., 872 description="A path to field that needs to be flattened.", 873 examples=[["data"], ["data", "*", "field"]], 874 title="Field Path", 875 ) 876 delete_origin_value: Optional[bool] = Field( 877 None, 878 description="Whether to delete the origin value or keep it. Default is False.", 879 title="Delete Origin Value", 880 ) 881 replace_record: Optional[bool] = Field( 882 None, 883 description="Whether to replace the origin record or not. Default is False.", 884 title="Replace Origin Record", 885 ) 886 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters")
class
KeysReplace(pydantic.v1.main.BaseModel):
889class KeysReplace(BaseModel): 890 type: Literal["KeysReplace"] 891 old: str = Field( 892 ..., 893 description="Old value to replace.", 894 examples=[ 895 " ", 896 "{{ record.id }}", 897 "{{ config['id'] }}", 898 "{{ stream_slice['id'] }}", 899 ], 900 title="Old value", 901 ) 902 new: str = Field( 903 ..., 904 description="New value to set.", 905 examples=[ 906 "_", 907 "{{ record.id }}", 908 "{{ config['id'] }}", 909 "{{ stream_slice['id'] }}", 910 ], 911 title="New value", 912 ) 913 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters")
class
IterableDecoder(pydantic.v1.main.BaseModel):
class
XmlDecoder(pydantic.v1.main.BaseModel):
class
CustomDecoder(pydantic.v1.main.BaseModel):
924class CustomDecoder(BaseModel): 925 class Config: 926 extra = Extra.allow 927 928 type: Literal["CustomDecoder"] 929 class_name: str = Field( 930 ..., 931 description="Fully-qualified name of the class that will be implementing the custom decoding. Has to be a sub class of Decoder. The format is `source_<name>.<package>.<class_name>`.", 932 examples=["source_amazon_ads.components.GzipJsonlDecoder"], 933 title="Class Name", 934 ) 935 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters")
class
CustomDecoder.Config:
class
MinMaxDatetime(pydantic.v1.main.BaseModel):
938class MinMaxDatetime(BaseModel): 939 type: Literal["MinMaxDatetime"] 940 datetime: str = Field( 941 ..., 942 description="Datetime value.", 943 examples=["2021-01-01", "2021-01-01T00:00:00Z", "{{ config['start_time'] }}"], 944 title="Datetime", 945 ) 946 datetime_format: Optional[str] = Field( 947 "", 948 description='Format of the datetime value. Defaults to "%Y-%m-%dT%H:%M:%S.%f%z" if left empty. Use placeholders starting with "%" to describe the format the API is using. The following placeholders are available:\n * **%s**: Epoch unix timestamp - `1686218963`\n * **%s_as_float**: Epoch unix timestamp in seconds as float with microsecond precision - `1686218963.123456`\n * **%ms**: Epoch unix timestamp - `1686218963123`\n * **%a**: Weekday (abbreviated) - `Sun`\n * **%A**: Weekday (full) - `Sunday`\n * **%w**: Weekday (decimal) - `0` (Sunday), `6` (Saturday)\n * **%d**: Day of the month (zero-padded) - `01`, `02`, ..., `31`\n * **%b**: Month (abbreviated) - `Jan`\n * **%B**: Month (full) - `January`\n * **%m**: Month (zero-padded) - `01`, `02`, ..., `12`\n * **%y**: Year (without century, zero-padded) - `00`, `01`, ..., `99`\n * **%Y**: Year (with century) - `0001`, `0002`, ..., `9999`\n * **%H**: Hour (24-hour, zero-padded) - `00`, `01`, ..., `23`\n * **%I**: Hour (12-hour, zero-padded) - `01`, `02`, ..., `12`\n * **%p**: AM/PM indicator\n * **%M**: Minute (zero-padded) - `00`, `01`, ..., `59`\n * **%S**: Second (zero-padded) - `00`, `01`, ..., `59`\n * **%f**: Microsecond (zero-padded to 6 digits) - `000000`, `000001`, ..., `999999`\n * **%_ms**: Millisecond (zero-padded to 3 digits) - `000`, `001`, ..., `999`\n * **%z**: UTC offset - `(empty)`, `+0000`, `-04:00`\n * **%Z**: Time zone name - `(empty)`, `UTC`, `GMT`\n * **%j**: Day of the year (zero-padded) - `001`, `002`, ..., `366`\n * **%U**: Week number of the year (Sunday as first day) - `00`, `01`, ..., `53`\n * **%W**: Week number of the year (Monday as first day) - `00`, `01`, ..., `53`\n * **%c**: Date and time representation - `Tue Aug 16 21:30:00 1988`\n * **%x**: Date representation - `08/16/1988`\n * **%X**: Time representation - `21:30:00`\n * **%%**: Literal \'%\' character\n\n Some placeholders depend on the locale of the underlying system - in most cases this locale is configured as en/US. For more information see the [Python documentation](https://docs.python.org/3/library/datetime.html#strftime-and-strptime-format-codes).\n', 949 examples=["%Y-%m-%dT%H:%M:%S.%f%z", "%Y-%m-%d", "%s"], 950 title="Datetime Format", 951 ) 952 max_datetime: Optional[str] = Field( 953 None, 954 description="Ceiling applied on the datetime value. Must be formatted with the datetime_format field.", 955 examples=["2021-01-01T00:00:00Z", "2021-01-01"], 956 title="Max Datetime", 957 ) 958 min_datetime: Optional[str] = Field( 959 None, 960 description="Floor applied on the datetime value. Must be formatted with the datetime_format field.", 961 examples=["2010-01-01T00:00:00Z", "2010-01-01"], 962 title="Min Datetime", 963 ) 964 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters")
class
NoAuth(pydantic.v1.main.BaseModel):
class
NoPagination(pydantic.v1.main.BaseModel):
class
State(pydantic.v1.main.BaseModel):
class
State.Config:
class
OauthConnectorInputSpecification(pydantic.v1.main.BaseModel):
984class OauthConnectorInputSpecification(BaseModel): 985 class Config: 986 extra = Extra.allow 987 988 consent_url: str = Field( 989 ..., 990 description="The DeclarativeOAuth Specific string URL string template to initiate the authentication.\nThe placeholders are replaced during the processing to provide neccessary values.", 991 examples=[ 992 "https://domain.host.com/marketing_api/auth?{{client_id_key}}={{client_id_value}}&{{redirect_uri_key}}={{{{redirect_uri_value}} | urlEncoder}}&{{state_key}}={{state_value}}", 993 "https://endpoint.host.com/oauth2/authorize?{{client_id_key}}={{client_id_value}}&{{redirect_uri_key}}={{{{redirect_uri_value}} | urlEncoder}}&{{scope_key}}={{{{scope_value}} | urlEncoder}}&{{state_key}}={{state_value}}&subdomain={{subdomain}}", 994 ], 995 title="Consent URL", 996 ) 997 scope: Optional[str] = Field( 998 None, 999 description="The DeclarativeOAuth Specific string of the scopes needed to be grant for authenticated user.", 1000 examples=["user:read user:read_orders workspaces:read"], 1001 title="Scopes", 1002 ) 1003 access_token_url: str = Field( 1004 ..., 1005 description="The DeclarativeOAuth Specific URL templated string to obtain the `access_token`, `refresh_token` etc.\nThe placeholders are replaced during the processing to provide neccessary values.", 1006 examples=[ 1007 "https://auth.host.com/oauth2/token?{{client_id_key}}={{client_id_value}}&{{client_secret_key}}={{client_secret_value}}&{{auth_code_key}}={{auth_code_value}}&{{redirect_uri_key}}={{{{redirect_uri_value}} | urlEncoder}}" 1008 ], 1009 title="Access Token URL", 1010 ) 1011 access_token_headers: Optional[Dict[str, Any]] = Field( 1012 None, 1013 description="The DeclarativeOAuth Specific optional headers to inject while exchanging the `auth_code` to `access_token` during `completeOAuthFlow` step.", 1014 examples=[ 1015 { 1016 "Authorization": "Basic {{ {{ client_id_value }}:{{ client_secret_value }} | base64Encoder }}" 1017 } 1018 ], 1019 title="Access Token Headers", 1020 ) 1021 access_token_params: Optional[Dict[str, Any]] = Field( 1022 None, 1023 description="The DeclarativeOAuth Specific optional query parameters to inject while exchanging the `auth_code` to `access_token` during `completeOAuthFlow` step.\nWhen this property is provided, the query params will be encoded as `Json` and included in the outgoing API request.", 1024 examples=[ 1025 { 1026 "{{ auth_code_key }}": "{{ auth_code_value }}", 1027 "{{ client_id_key }}": "{{ client_id_value }}", 1028 "{{ client_secret_key }}": "{{ client_secret_value }}", 1029 } 1030 ], 1031 title="Access Token Query Params (Json Encoded)", 1032 ) 1033 extract_output: Optional[List[str]] = Field( 1034 None, 1035 description="The DeclarativeOAuth Specific list of strings to indicate which keys should be extracted and returned back to the input config.", 1036 examples=[["access_token", "refresh_token", "other_field"]], 1037 title="Extract Output", 1038 ) 1039 state: Optional[State] = Field( 1040 None, 1041 description="The DeclarativeOAuth Specific object to provide the criteria of how the `state` query param should be constructed,\nincluding length and complexity.", 1042 examples=[{"min": 7, "max": 128}], 1043 title="Configurable State Query Param", 1044 ) 1045 client_id_key: Optional[str] = Field( 1046 None, 1047 description="The DeclarativeOAuth Specific optional override to provide the custom `client_id` key name, if required by data-provider.", 1048 examples=["my_custom_client_id_key_name"], 1049 title="Client ID Key Override", 1050 ) 1051 client_secret_key: Optional[str] = Field( 1052 None, 1053 description="The DeclarativeOAuth Specific optional override to provide the custom `client_secret` key name, if required by data-provider.", 1054 examples=["my_custom_client_secret_key_name"], 1055 title="Client Secret Key Override", 1056 ) 1057 scope_key: Optional[str] = Field( 1058 None, 1059 description="The DeclarativeOAuth Specific optional override to provide the custom `scope` key name, if required by data-provider.", 1060 examples=["my_custom_scope_key_key_name"], 1061 title="Scopes Key Override", 1062 ) 1063 state_key: Optional[str] = Field( 1064 None, 1065 description="The DeclarativeOAuth Specific optional override to provide the custom `state` key name, if required by data-provider.", 1066 examples=["my_custom_state_key_key_name"], 1067 title="State Key Override", 1068 ) 1069 auth_code_key: Optional[str] = Field( 1070 None, 1071 description="The DeclarativeOAuth Specific optional override to provide the custom `code` key name to something like `auth_code` or `custom_auth_code`, if required by data-provider.", 1072 examples=["my_custom_auth_code_key_name"], 1073 title="Auth Code Key Override", 1074 ) 1075 redirect_uri_key: Optional[str] = Field( 1076 None, 1077 description="The DeclarativeOAuth Specific optional override to provide the custom `redirect_uri` key name to something like `callback_uri`, if required by data-provider.", 1078 examples=["my_custom_redirect_uri_key_name"], 1079 title="Redirect URI Key Override", 1080 )
state: Optional[State]
class
OauthConnectorInputSpecification.Config:
class
OAuthConfigSpecification(pydantic.v1.main.BaseModel):
1083class OAuthConfigSpecification(BaseModel): 1084 class Config: 1085 extra = Extra.allow 1086 1087 oauth_user_input_from_connector_config_specification: Optional[Dict[str, Any]] = Field( 1088 None, 1089 description="OAuth specific blob. This is a Json Schema used to validate Json configurations used as input to OAuth.\nMust be a valid non-nested JSON that refers to properties from ConnectorSpecification.connectionSpecification\nusing special annotation 'path_in_connector_config'.\nThese are input values the user is entering through the UI to authenticate to the connector, that might also shared\nas inputs for syncing data via the connector.\nExamples:\nif no connector values is shared during oauth flow, oauth_user_input_from_connector_config_specification=[]\nif connector values such as 'app_id' inside the top level are used to generate the API url for the oauth flow,\n oauth_user_input_from_connector_config_specification={\n app_id: {\n type: string\n path_in_connector_config: ['app_id']\n }\n }\nif connector values such as 'info.app_id' nested inside another object are used to generate the API url for the oauth flow,\n oauth_user_input_from_connector_config_specification={\n app_id: {\n type: string\n path_in_connector_config: ['info', 'app_id']\n }\n }", 1090 examples=[ 1091 {"app_id": {"type": "string", "path_in_connector_config": ["app_id"]}}, 1092 { 1093 "app_id": { 1094 "type": "string", 1095 "path_in_connector_config": ["info", "app_id"], 1096 } 1097 }, 1098 ], 1099 title="OAuth user input", 1100 ) 1101 oauth_connector_input_specification: Optional[OauthConnectorInputSpecification] = Field( 1102 None, 1103 description='The DeclarativeOAuth specific blob.\nPertains to the fields defined by the connector relating to the OAuth flow.\n\nInterpolation capabilities:\n- The variables placeholders are declared as `{{my_var}}`.\n- The nested resolution variables like `{{ {{my_nested_var}} }}` is allowed as well.\n\n- The allowed interpolation context is:\n + base64Encoder - encode to `base64`, {{ {{my_var_a}}:{{my_var_b}} | base64Encoder }}\n + base64Decorer - decode from `base64` encoded string, {{ {{my_string_variable_or_string_value}} | base64Decoder }}\n + urlEncoder - encode the input string to URL-like format, {{ https://test.host.com/endpoint | urlEncoder}}\n + urlDecorer - decode the input url-encoded string into text format, {{ urlDecoder:https%3A%2F%2Fairbyte.io | urlDecoder}}\n + codeChallengeS256 - get the `codeChallenge` encoded value to provide additional data-provider specific authorisation values, {{ {{state_value}} | codeChallengeS256 }}\n\nExamples:\n - The TikTok Marketing DeclarativeOAuth spec:\n {\n "oauth_connector_input_specification": {\n "type": "object",\n "additionalProperties": false,\n "properties": {\n "consent_url": "https://ads.tiktok.com/marketing_api/auth?{{client_id_key}}={{client_id_value}}&{{redirect_uri_key}}={{ {{redirect_uri_value}} | urlEncoder}}&{{state_key}}={{state_value}}",\n "access_token_url": "https://business-api.tiktok.com/open_api/v1.3/oauth2/access_token/",\n "access_token_params": {\n "{{ auth_code_key }}": "{{ auth_code_value }}",\n "{{ client_id_key }}": "{{ client_id_value }}",\n "{{ client_secret_key }}": "{{ client_secret_value }}"\n },\n "access_token_headers": {\n "Content-Type": "application/json",\n "Accept": "application/json"\n },\n "extract_output": ["data.access_token"],\n "client_id_key": "app_id",\n "client_secret_key": "secret",\n "auth_code_key": "auth_code"\n }\n }\n }', 1104 title="DeclarativeOAuth Connector Specification", 1105 ) 1106 complete_oauth_output_specification: Optional[Dict[str, Any]] = Field( 1107 None, 1108 description="OAuth specific blob. This is a Json Schema used to validate Json configurations produced by the OAuth flows as they are\nreturned by the distant OAuth APIs.\nMust be a valid JSON describing the fields to merge back to `ConnectorSpecification.connectionSpecification`.\nFor each field, a special annotation `path_in_connector_config` can be specified to determine where to merge it,\nExamples:\n complete_oauth_output_specification={\n refresh_token: {\n type: string,\n path_in_connector_config: ['credentials', 'refresh_token']\n }\n }", 1109 examples=[ 1110 { 1111 "refresh_token": { 1112 "type": "string,", 1113 "path_in_connector_config": ["credentials", "refresh_token"], 1114 } 1115 } 1116 ], 1117 title="OAuth output specification", 1118 ) 1119 complete_oauth_server_input_specification: Optional[Dict[str, Any]] = Field( 1120 None, 1121 description="OAuth specific blob. This is a Json Schema used to validate Json configurations persisted as Airbyte Server configurations.\nMust be a valid non-nested JSON describing additional fields configured by the Airbyte Instance or Workspace Admins to be used by the\nserver when completing an OAuth flow (typically exchanging an auth code for refresh token).\nExamples:\n complete_oauth_server_input_specification={\n client_id: {\n type: string\n },\n client_secret: {\n type: string\n }\n }", 1122 examples=[{"client_id": {"type": "string"}, "client_secret": {"type": "string"}}], 1123 title="OAuth input specification", 1124 ) 1125 complete_oauth_server_output_specification: Optional[Dict[str, Any]] = Field( 1126 None, 1127 description="OAuth specific blob. This is a Json Schema used to validate Json configurations persisted as Airbyte Server configurations that\nalso need to be merged back into the connector configuration at runtime.\nThis is a subset configuration of `complete_oauth_server_input_specification` that filters fields out to retain only the ones that\nare necessary for the connector to function with OAuth. (some fields could be used during oauth flows but not needed afterwards, therefore\nthey would be listed in the `complete_oauth_server_input_specification` but not `complete_oauth_server_output_specification`)\nMust be a valid non-nested JSON describing additional fields configured by the Airbyte Instance or Workspace Admins to be used by the\nconnector when using OAuth flow APIs.\nThese fields are to be merged back to `ConnectorSpecification.connectionSpecification`.\nFor each field, a special annotation `path_in_connector_config` can be specified to determine where to merge it,\nExamples:\n complete_oauth_server_output_specification={\n client_id: {\n type: string,\n path_in_connector_config: ['credentials', 'client_id']\n },\n client_secret: {\n type: string,\n path_in_connector_config: ['credentials', 'client_secret']\n }\n }", 1128 examples=[ 1129 { 1130 "client_id": { 1131 "type": "string,", 1132 "path_in_connector_config": ["credentials", "client_id"], 1133 }, 1134 "client_secret": { 1135 "type": "string,", 1136 "path_in_connector_config": ["credentials", "client_secret"], 1137 }, 1138 } 1139 ], 1140 title="OAuth server output specification", 1141 )
oauth_connector_input_specification: Optional[OauthConnectorInputSpecification]
class
OAuthConfigSpecification.Config:
class
OffsetIncrement(pydantic.v1.main.BaseModel):
1144class OffsetIncrement(BaseModel): 1145 type: Literal["OffsetIncrement"] 1146 page_size: Optional[Union[int, str]] = Field( 1147 None, 1148 description="The number of records to include in each pages.", 1149 examples=[100, "{{ config['page_size'] }}"], 1150 title="Limit", 1151 ) 1152 inject_on_first_request: Optional[bool] = Field( 1153 False, 1154 description="Using the `offset` with value `0` during the first request", 1155 title="Inject Offset", 1156 ) 1157 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters")
class
PageIncrement(pydantic.v1.main.BaseModel):
1160class PageIncrement(BaseModel): 1161 type: Literal["PageIncrement"] 1162 page_size: Optional[Union[int, str]] = Field( 1163 None, 1164 description="The number of records to include in each pages.", 1165 examples=[100, "100", "{{ config['page_size'] }}"], 1166 title="Page Size", 1167 ) 1168 start_from_page: Optional[int] = Field( 1169 0, 1170 description="Index of the first page to request.", 1171 examples=[0, 1], 1172 title="Start From Page", 1173 ) 1174 inject_on_first_request: Optional[bool] = Field( 1175 False, 1176 description="Using the `page number` with value defined by `start_from_page` during the first request", 1177 title="Inject Page Number", 1178 ) 1179 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters")
class
PrimaryKey(pydantic.v1.main.BaseModel):
1182class PrimaryKey(BaseModel): 1183 __root__: Union[str, List[str], List[List[str]]] = Field( 1184 ..., 1185 description="The stream field to be used to distinguish unique records. Can either be a single field, an array of fields representing a composite key, or an array of arrays representing a composite key where the fields are nested fields.", 1186 examples=["id", ["code", "type"]], 1187 title="Primary Key", 1188 )
class
RecordFilter(pydantic.v1.main.BaseModel):
1191class RecordFilter(BaseModel): 1192 type: Literal["RecordFilter"] 1193 condition: Optional[str] = Field( 1194 "", 1195 description="The predicate to filter a record. Records will be removed if evaluated to False.", 1196 examples=[ 1197 "{{ record['created_at'] >= stream_interval['start_time'] }}", 1198 "{{ record.status in ['active', 'expired'] }}", 1199 ], 1200 ) 1201 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters")
class
SchemaNormalization(enum.Enum):
An enumeration.
None_ =
<SchemaNormalization.None_: 'None'>
Default =
<SchemaNormalization.Default: 'Default'>
class
RemoveFields(pydantic.v1.main.BaseModel):
1209class RemoveFields(BaseModel): 1210 type: Literal["RemoveFields"] 1211 condition: Optional[str] = Field( 1212 "", 1213 description="The predicate to filter a property by a property value. Property will be removed if it is empty OR expression is evaluated to True.,", 1214 examples=[ 1215 "{{ property|string == '' }}", 1216 "{{ property is integer }}", 1217 "{{ property|length > 5 }}", 1218 "{{ property == 'some_string_to_match' }}", 1219 ], 1220 ) 1221 field_pointers: List[List[str]] = Field( 1222 ..., 1223 description="Array of paths defining the field to remove. Each item is an array whose field describe the path of a field to remove.", 1224 examples=[["tags"], [["content", "html"], ["content", "plain_text"]]], 1225 title="Field Paths", 1226 )
class
RequestPath(pydantic.v1.main.BaseModel):
class
InjectInto(enum.Enum):
1233class InjectInto(Enum): 1234 request_parameter = "request_parameter" 1235 header = "header" 1236 body_data = "body_data" 1237 body_json = "body_json"
An enumeration.
request_parameter =
<InjectInto.request_parameter: 'request_parameter'>
header =
<InjectInto.header: 'header'>
body_data =
<InjectInto.body_data: 'body_data'>
body_json =
<InjectInto.body_json: 'body_json'>
class
RequestOption(pydantic.v1.main.BaseModel):
1240class RequestOption(BaseModel): 1241 type: Literal["RequestOption"] 1242 field_name: Optional[str] = Field( 1243 None, 1244 description="Configures which key should be used in the location that the descriptor is being injected into. We hope to eventually deprecate this field in favor of `field_path` for all request_options, but must currently maintain it for backwards compatibility in the Builder.", 1245 examples=["segment_id"], 1246 title="Field Name", 1247 ) 1248 field_path: Optional[List[str]] = Field( 1249 None, 1250 description="Configures a path to be used for nested structures in JSON body requests (e.g. GraphQL queries)", 1251 examples=[["data", "viewer", "id"]], 1252 title="Field Path", 1253 ) 1254 inject_into: InjectInto = Field( 1255 ..., 1256 description="Configures where the descriptor should be set on the HTTP requests. Note that request parameters that are already encoded in the URL path will not be duplicated.", 1257 examples=["request_parameter", "header", "body_data", "body_json"], 1258 title="Inject Into", 1259 )
inject_into: InjectInto
class
Schemas(pydantic.v1.main.BaseModel):
class
Schemas.Config:
class
LegacySessionTokenAuthenticator(pydantic.v1.main.BaseModel):
1269class LegacySessionTokenAuthenticator(BaseModel): 1270 type: Literal["LegacySessionTokenAuthenticator"] 1271 header: str = Field( 1272 ..., 1273 description="The name of the session token header that will be injected in the request", 1274 examples=["X-Session"], 1275 title="Session Request Header", 1276 ) 1277 login_url: str = Field( 1278 ..., 1279 description="Path of the login URL (do not include the base URL)", 1280 examples=["session"], 1281 title="Login Path", 1282 ) 1283 session_token: Optional[str] = Field( 1284 None, 1285 description="Session token to use if using a pre-defined token. Not needed if authenticating with username + password pair", 1286 example=["{{ config['session_token'] }}"], 1287 title="Session Token", 1288 ) 1289 session_token_response_key: str = Field( 1290 ..., 1291 description="Name of the key of the session token to be extracted from the response", 1292 examples=["id"], 1293 title="Response Token Response Key", 1294 ) 1295 username: Optional[str] = Field( 1296 None, 1297 description="Username used to authenticate and obtain a session token", 1298 examples=[" {{ config['username'] }}"], 1299 title="Username", 1300 ) 1301 password: Optional[str] = Field( 1302 "", 1303 description="Password used to authenticate and obtain a session token", 1304 examples=["{{ config['password'] }}", ""], 1305 title="Password", 1306 ) 1307 validate_session_url: str = Field( 1308 ..., 1309 description="Path of the URL to use to validate that the session token is valid (do not include the base URL)", 1310 examples=["user/current"], 1311 title="Validate Session Path", 1312 ) 1313 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters")
class
CsvDecoder(pydantic.v1.main.BaseModel):
class
AsyncJobStatusMap(pydantic.v1.main.BaseModel):
class
ValueType(enum.Enum):
1330class ValueType(Enum): 1331 string = "string" 1332 number = "number" 1333 integer = "integer" 1334 boolean = "boolean"
An enumeration.
string =
<ValueType.string: 'string'>
number =
<ValueType.number: 'number'>
integer =
<ValueType.integer: 'integer'>
boolean =
<ValueType.boolean: 'boolean'>
class
WaitTimeFromHeader(pydantic.v1.main.BaseModel):
1337class WaitTimeFromHeader(BaseModel): 1338 type: Literal["WaitTimeFromHeader"] 1339 header: str = Field( 1340 ..., 1341 description="The name of the response header defining how long to wait before retrying.", 1342 examples=["Retry-After"], 1343 title="Response Header Name", 1344 ) 1345 regex: Optional[str] = Field( 1346 None, 1347 description="Optional regex to apply on the header to extract its value. The regex should define a capture group defining the wait time.", 1348 examples=["([-+]?\\d+)"], 1349 title="Extraction Regex", 1350 ) 1351 max_waiting_time_in_seconds: Optional[float] = Field( 1352 None, 1353 description="Given the value extracted from the header is greater than this value, stop the stream.", 1354 examples=[3600], 1355 title="Max Waiting Time in Seconds", 1356 ) 1357 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters")
class
WaitUntilTimeFromHeader(pydantic.v1.main.BaseModel):
1360class WaitUntilTimeFromHeader(BaseModel): 1361 type: Literal["WaitUntilTimeFromHeader"] 1362 header: str = Field( 1363 ..., 1364 description="The name of the response header defining how long to wait before retrying.", 1365 examples=["wait_time"], 1366 title="Response Header", 1367 ) 1368 min_wait: Optional[Union[float, str]] = Field( 1369 None, 1370 description="Minimum time to wait before retrying.", 1371 examples=[10, "60"], 1372 title="Minimum Wait Time", 1373 ) 1374 regex: Optional[str] = Field( 1375 None, 1376 description="Optional regex to apply on the header to extract its value. The regex should define a capture group defining the wait time.", 1377 examples=["([-+]?\\d+)"], 1378 title="Extraction Regex", 1379 ) 1380 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters")
class
ComponentMappingDefinition(pydantic.v1.main.BaseModel):
1383class ComponentMappingDefinition(BaseModel): 1384 type: Literal["ComponentMappingDefinition"] 1385 field_path: List[str] = Field( 1386 ..., 1387 description="A list of potentially nested fields indicating the full path where value will be added or updated.", 1388 examples=[ 1389 ["data"], 1390 ["data", "records"], 1391 ["data", 1, "name"], 1392 ["data", "{{ components_values.name }}"], 1393 ["data", "*", "record"], 1394 ["*", "**", "name"], 1395 ], 1396 title="Field Path", 1397 ) 1398 value: str = Field( 1399 ..., 1400 description="The dynamic or static value to assign to the key. Interpolated values can be used to dynamically determine the value during runtime.", 1401 examples=[ 1402 "{{ components_values['updates'] }}", 1403 "{{ components_values['MetaData']['LastUpdatedTime'] }}", 1404 "{{ config['segment_id'] }}", 1405 "{{ stream_slice['parent_id'] }}", 1406 "{{ stream_slice['extra_fields']['name'] }}", 1407 ], 1408 title="Value", 1409 ) 1410 value_type: Optional[ValueType] = Field( 1411 None, 1412 description="The expected data type of the value. If omitted, the type will be inferred from the value provided.", 1413 title="Value Type", 1414 ) 1415 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters")
value_type: Optional[ValueType]
class
StreamConfig(pydantic.v1.main.BaseModel):
1418class StreamConfig(BaseModel): 1419 type: Literal["StreamConfig"] 1420 configs_pointer: List[str] = Field( 1421 ..., 1422 description="A list of potentially nested fields indicating the full path in source config file where streams configs located.", 1423 examples=[["data"], ["data", "streams"], ["data", "{{ parameters.name }}"]], 1424 title="Configs Pointer", 1425 ) 1426 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters")
class
ConfigComponentsResolver(pydantic.v1.main.BaseModel):
1429class ConfigComponentsResolver(BaseModel): 1430 type: Literal["ConfigComponentsResolver"] 1431 stream_config: StreamConfig 1432 components_mapping: List[ComponentMappingDefinition] 1433 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters")
stream_config: StreamConfig
components_mapping: List[ComponentMappingDefinition]
class
AddedFieldDefinition(pydantic.v1.main.BaseModel):
1436class AddedFieldDefinition(BaseModel): 1437 type: Literal["AddedFieldDefinition"] 1438 path: List[str] = Field( 1439 ..., 1440 description="List of strings defining the path where to add the value on the record.", 1441 examples=[["segment_id"], ["metadata", "segment_id"]], 1442 title="Path", 1443 ) 1444 value: str = Field( 1445 ..., 1446 description="Value of the new field. Use {{ record['existing_field'] }} syntax to refer to other fields in the record.", 1447 examples=[ 1448 "{{ record['updates'] }}", 1449 "{{ record['MetaData']['LastUpdatedTime'] }}", 1450 "{{ stream_partition['segment_id'] }}", 1451 ], 1452 title="Value", 1453 ) 1454 value_type: Optional[ValueType] = Field( 1455 None, 1456 description="Type of the value. If not specified, the type will be inferred from the value.", 1457 title="Value Type", 1458 ) 1459 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters")
value_type: Optional[ValueType]
class
AddFields(pydantic.v1.main.BaseModel):
1462class AddFields(BaseModel): 1463 type: Literal["AddFields"] 1464 fields: List[AddedFieldDefinition] = Field( 1465 ..., 1466 description="List of transformations (path and corresponding value) that will be added to the record.", 1467 title="Fields", 1468 ) 1469 condition: Optional[str] = Field( 1470 "", 1471 description="Fields will be added if expression is evaluated to True.", 1472 examples=[ 1473 "{{ property|string == '' }}", 1474 "{{ property is integer }}", 1475 "{{ property|length > 5 }}", 1476 "{{ property == 'some_string_to_match' }}", 1477 ], 1478 ) 1479 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters")
fields: List[AddedFieldDefinition]
class
ApiKeyAuthenticator(pydantic.v1.main.BaseModel):
1482class ApiKeyAuthenticator(BaseModel): 1483 type: Literal["ApiKeyAuthenticator"] 1484 api_token: Optional[str] = Field( 1485 None, 1486 description="The API key to inject in the request. Fill it in the user inputs.", 1487 examples=["{{ config['api_key'] }}", "Token token={{ config['api_key'] }}"], 1488 title="API Key", 1489 ) 1490 header: Optional[str] = Field( 1491 None, 1492 description="The name of the HTTP header that will be set to the API key. This setting is deprecated, use inject_into instead. Header and inject_into can not be defined at the same time.", 1493 examples=["Authorization", "Api-Token", "X-Auth-Token"], 1494 title="Header Name", 1495 ) 1496 inject_into: Optional[RequestOption] = Field( 1497 None, 1498 description="Configure how the API Key will be sent in requests to the source API. Either inject_into or header has to be defined.", 1499 examples=[ 1500 {"inject_into": "header", "field_name": "Authorization"}, 1501 {"inject_into": "request_parameter", "field_name": "authKey"}, 1502 ], 1503 title="Inject API Key Into Outgoing HTTP Request", 1504 ) 1505 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters")
inject_into: Optional[RequestOption]
class
AuthFlow(pydantic.v1.main.BaseModel):
1508class AuthFlow(BaseModel): 1509 auth_flow_type: Optional[AuthFlowType] = Field( 1510 None, description="The type of auth to use", title="Auth flow type" 1511 ) 1512 predicate_key: Optional[List[str]] = Field( 1513 None, 1514 description="JSON path to a field in the connectorSpecification that should exist for the advanced auth to be applicable.", 1515 examples=[["credentials", "auth_type"]], 1516 title="Predicate key", 1517 ) 1518 predicate_value: Optional[str] = Field( 1519 None, 1520 description="Value of the predicate_key fields for the advanced auth to be applicable.", 1521 examples=["Oauth"], 1522 title="Predicate value", 1523 ) 1524 oauth_config_specification: Optional[OAuthConfigSpecification] = None
auth_flow_type: Optional[AuthFlowType]
oauth_config_specification: Optional[OAuthConfigSpecification]
class
IncrementingCountCursor(pydantic.v1.main.BaseModel):
1527class IncrementingCountCursor(BaseModel): 1528 type: Literal["IncrementingCountCursor"] 1529 cursor_field: str = Field( 1530 ..., 1531 description="The location of the value on a record that will be used as a bookmark during sync. To ensure no data loss, the API must return records in ascending order based on the cursor field. Nested fields are not supported, so the field must be at the top level of the record. You can use a combination of Add Field and Remove Field transformations to move the nested field to the top.", 1532 examples=["created_at", "{{ config['record_cursor'] }}"], 1533 title="Cursor Field", 1534 ) 1535 start_value: Optional[Union[str, int]] = Field( 1536 None, 1537 description="The value that determines the earliest record that should be synced.", 1538 examples=[0, "{{ config['start_value'] }}"], 1539 title="Start Value", 1540 ) 1541 start_value_option: Optional[RequestOption] = Field( 1542 None, 1543 description="Optionally configures how the start value will be sent in requests to the source API.", 1544 title="Inject Start Value Into Outgoing HTTP Request", 1545 ) 1546 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters")
start_value_option: Optional[RequestOption]
class
DatetimeBasedCursor(pydantic.v1.main.BaseModel):
1549class DatetimeBasedCursor(BaseModel): 1550 type: Literal["DatetimeBasedCursor"] 1551 clamping: Optional[Clamping] = Field( 1552 None, 1553 description="This option is used to adjust the upper and lower boundaries of each datetime window to beginning and end of the provided target period (day, week, month)", 1554 title="Date Range Clamping", 1555 ) 1556 cursor_field: str = Field( 1557 ..., 1558 description="The location of the value on a record that will be used as a bookmark during sync. To ensure no data loss, the API must return records in ascending order based on the cursor field. Nested fields are not supported, so the field must be at the top level of the record. You can use a combination of Add Field and Remove Field transformations to move the nested field to the top.", 1559 examples=["created_at", "{{ config['record_cursor'] }}"], 1560 title="Cursor Field", 1561 ) 1562 datetime_format: str = Field( 1563 ..., 1564 description="The datetime format used to format the datetime values that are sent in outgoing requests to the API. Use placeholders starting with \"%\" to describe the format the API is using. The following placeholders are available:\n * **%s**: Epoch unix timestamp - `1686218963`\n * **%s_as_float**: Epoch unix timestamp in seconds as float with microsecond precision - `1686218963.123456`\n * **%ms**: Epoch unix timestamp (milliseconds) - `1686218963123`\n * **%a**: Weekday (abbreviated) - `Sun`\n * **%A**: Weekday (full) - `Sunday`\n * **%w**: Weekday (decimal) - `0` (Sunday), `6` (Saturday)\n * **%d**: Day of the month (zero-padded) - `01`, `02`, ..., `31`\n * **%b**: Month (abbreviated) - `Jan`\n * **%B**: Month (full) - `January`\n * **%m**: Month (zero-padded) - `01`, `02`, ..., `12`\n * **%y**: Year (without century, zero-padded) - `00`, `01`, ..., `99`\n * **%Y**: Year (with century) - `0001`, `0002`, ..., `9999`\n * **%H**: Hour (24-hour, zero-padded) - `00`, `01`, ..., `23`\n * **%I**: Hour (12-hour, zero-padded) - `01`, `02`, ..., `12`\n * **%p**: AM/PM indicator\n * **%M**: Minute (zero-padded) - `00`, `01`, ..., `59`\n * **%S**: Second (zero-padded) - `00`, `01`, ..., `59`\n * **%f**: Microsecond (zero-padded to 6 digits) - `000000`\n * **%_ms**: Millisecond (zero-padded to 3 digits) - `000`\n * **%z**: UTC offset - `(empty)`, `+0000`, `-04:00`\n * **%Z**: Time zone name - `(empty)`, `UTC`, `GMT`\n * **%j**: Day of the year (zero-padded) - `001`, `002`, ..., `366`\n * **%U**: Week number of the year (starting Sunday) - `00`, ..., `53`\n * **%W**: Week number of the year (starting Monday) - `00`, ..., `53`\n * **%c**: Date and time - `Tue Aug 16 21:30:00 1988`\n * **%x**: Date standard format - `08/16/1988`\n * **%X**: Time standard format - `21:30:00`\n * **%%**: Literal '%' character\n\n Some placeholders depend on the locale of the underlying system - in most cases this locale is configured as en/US. For more information see the [Python documentation](https://docs.python.org/3/library/datetime.html#strftime-and-strptime-format-codes).\n", 1565 examples=["%Y-%m-%dT%H:%M:%S.%f%z", "%Y-%m-%d", "%s", "%ms", "%s_as_float"], 1566 title="Outgoing Datetime Format", 1567 ) 1568 start_datetime: Union[str, MinMaxDatetime] = Field( 1569 ..., 1570 description="The datetime that determines the earliest record that should be synced.", 1571 examples=["2020-01-1T00:00:00Z", "{{ config['start_time'] }}"], 1572 title="Start Datetime", 1573 ) 1574 cursor_datetime_formats: Optional[List[str]] = Field( 1575 None, 1576 description="The possible formats for the cursor field, in order of preference. The first format that matches the cursor field value will be used to parse it. If not provided, the `datetime_format` will be used.", 1577 title="Cursor Datetime Formats", 1578 ) 1579 cursor_granularity: Optional[str] = Field( 1580 None, 1581 description="Smallest increment the datetime_format has (ISO 8601 duration) that is used to ensure the start of a slice does not overlap with the end of the previous one, e.g. for %Y-%m-%d the granularity should be P1D, for %Y-%m-%dT%H:%M:%SZ the granularity should be PT1S. Given this field is provided, `step` needs to be provided as well.", 1582 examples=["PT1S"], 1583 title="Cursor Granularity", 1584 ) 1585 end_datetime: Optional[Union[str, MinMaxDatetime]] = Field( 1586 None, 1587 description="The datetime that determines the last record that should be synced. If not provided, `{{ now_utc() }}` will be used.", 1588 examples=["2021-01-1T00:00:00Z", "{{ now_utc() }}", "{{ day_delta(-1) }}"], 1589 title="End Datetime", 1590 ) 1591 end_time_option: Optional[RequestOption] = Field( 1592 None, 1593 description="Optionally configures how the end datetime will be sent in requests to the source API.", 1594 title="Inject End Time Into Outgoing HTTP Request", 1595 ) 1596 is_data_feed: Optional[bool] = Field( 1597 None, 1598 description="A data feed API is an API that does not allow filtering and paginates the content from the most recent to the least recent. Given this, the CDK needs to know when to stop paginating and this field will generate a stop condition for pagination.", 1599 title="Whether the target API is formatted as a data feed", 1600 ) 1601 is_client_side_incremental: Optional[bool] = Field( 1602 None, 1603 description="If the target API endpoint does not take cursor values to filter records and returns all records anyway, the connector with this cursor will filter out records locally, and only emit new records from the last sync, hence incremental. This means that all records would be read from the API, but only new records will be emitted to the destination.", 1604 title="Whether the target API does not support filtering and returns all data (the cursor filters records in the client instead of the API side)", 1605 ) 1606 is_compare_strictly: Optional[bool] = Field( 1607 False, 1608 description="Set to True if the target API does not accept queries where the start time equal the end time.", 1609 title="Whether to skip requests if the start time equals the end time", 1610 ) 1611 global_substream_cursor: Optional[bool] = Field( 1612 False, 1613 description="This setting optimizes performance when the parent stream has thousands of partitions by storing the cursor as a single value rather than per partition. Notably, the substream state is updated only at the end of the sync, which helps prevent data loss in case of a sync failure. See more info in the [docs](https://docs.airbyte.com/connector-development/config-based/understanding-the-yaml-file/incremental-syncs).", 1614 title="Whether to store cursor as one value instead of per partition", 1615 ) 1616 lookback_window: Optional[str] = Field( 1617 None, 1618 description="Time interval before the start_datetime to read data for, e.g. P1M for looking back one month.", 1619 examples=["P1D", "P{{ config['lookback_days'] }}D"], 1620 title="Lookback Window", 1621 ) 1622 partition_field_end: Optional[str] = Field( 1623 None, 1624 description="Name of the partition start time field.", 1625 examples=["ending_time"], 1626 title="Partition Field End", 1627 ) 1628 partition_field_start: Optional[str] = Field( 1629 None, 1630 description="Name of the partition end time field.", 1631 examples=["starting_time"], 1632 title="Partition Field Start", 1633 ) 1634 start_time_option: Optional[RequestOption] = Field( 1635 None, 1636 description="Optionally configures how the start datetime will be sent in requests to the source API.", 1637 title="Inject Start Time Into Outgoing HTTP Request", 1638 ) 1639 step: Optional[str] = Field( 1640 None, 1641 description="The size of the time window (ISO8601 duration). Given this field is provided, `cursor_granularity` needs to be provided as well.", 1642 examples=["P1W", "{{ config['step_increment'] }}"], 1643 title="Step", 1644 ) 1645 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters")
clamping: Optional[Clamping]
start_datetime: Union[str, MinMaxDatetime]
end_datetime: Union[str, MinMaxDatetime, NoneType]
end_time_option: Optional[RequestOption]
start_time_option: Optional[RequestOption]
class
FixedWindowCallRatePolicy(pydantic.v1.main.BaseModel):
1648class FixedWindowCallRatePolicy(BaseModel): 1649 class Config: 1650 extra = Extra.allow 1651 1652 type: Literal["FixedWindowCallRatePolicy"] 1653 period: str = Field( 1654 ..., description="The time interval for the rate limit window.", title="Period" 1655 ) 1656 call_limit: int = Field( 1657 ..., 1658 description="The maximum number of calls allowed within the period.", 1659 title="Call Limit", 1660 ) 1661 matchers: List[HttpRequestRegexMatcher] = Field( 1662 ..., 1663 description="List of matchers that define which requests this policy applies to.", 1664 title="Matchers", 1665 )
matchers: List[HttpRequestRegexMatcher]
class
FixedWindowCallRatePolicy.Config:
class
MovingWindowCallRatePolicy(pydantic.v1.main.BaseModel):
1668class MovingWindowCallRatePolicy(BaseModel): 1669 class Config: 1670 extra = Extra.allow 1671 1672 type: Literal["MovingWindowCallRatePolicy"] 1673 rates: List[Rate] = Field( 1674 ..., 1675 description="List of rates that define the call limits for different time intervals.", 1676 title="Rates", 1677 ) 1678 matchers: List[HttpRequestRegexMatcher] = Field( 1679 ..., 1680 description="List of matchers that define which requests this policy applies to.", 1681 title="Matchers", 1682 )
rates: List[Rate]
matchers: List[HttpRequestRegexMatcher]
class
MovingWindowCallRatePolicy.Config:
class
UnlimitedCallRatePolicy(pydantic.v1.main.BaseModel):
1685class UnlimitedCallRatePolicy(BaseModel): 1686 class Config: 1687 extra = Extra.allow 1688 1689 type: Literal["UnlimitedCallRatePolicy"] 1690 matchers: List[HttpRequestRegexMatcher] = Field( 1691 ..., 1692 description="List of matchers that define which requests this policy applies to.", 1693 title="Matchers", 1694 )
matchers: List[HttpRequestRegexMatcher]
class
UnlimitedCallRatePolicy.Config:
class
DefaultErrorHandler(pydantic.v1.main.BaseModel):
1697class DefaultErrorHandler(BaseModel): 1698 type: Literal["DefaultErrorHandler"] 1699 backoff_strategies: Optional[ 1700 List[ 1701 Union[ 1702 ConstantBackoffStrategy, 1703 CustomBackoffStrategy, 1704 ExponentialBackoffStrategy, 1705 WaitTimeFromHeader, 1706 WaitUntilTimeFromHeader, 1707 ] 1708 ] 1709 ] = Field( 1710 None, 1711 description="List of backoff strategies to use to determine how long to wait before retrying a retryable request.", 1712 title="Backoff Strategies", 1713 ) 1714 max_retries: Optional[int] = Field( 1715 5, 1716 description="The maximum number of time to retry a retryable request before giving up and failing.", 1717 examples=[5, 0, 10], 1718 title="Max Retry Count", 1719 ) 1720 response_filters: Optional[List[HttpResponseFilter]] = Field( 1721 None, 1722 description="List of response filters to iterate on when deciding how to handle an error. When using an array of multiple filters, the filters will be applied sequentially and the response will be selected if it matches any of the filter's predicate.", 1723 title="Response Filters", 1724 ) 1725 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters")
backoff_strategies: Optional[List[Union[ConstantBackoffStrategy, CustomBackoffStrategy, ExponentialBackoffStrategy, WaitTimeFromHeader, WaitUntilTimeFromHeader]]]
response_filters: Optional[List[HttpResponseFilter]]
class
DefaultPaginator(pydantic.v1.main.BaseModel):
1728class DefaultPaginator(BaseModel): 1729 type: Literal["DefaultPaginator"] 1730 pagination_strategy: Union[ 1731 CursorPagination, CustomPaginationStrategy, OffsetIncrement, PageIncrement 1732 ] = Field( 1733 ..., 1734 description="Strategy defining how records are paginated.", 1735 title="Pagination Strategy", 1736 ) 1737 page_size_option: Optional[RequestOption] = None 1738 page_token_option: Optional[Union[RequestOption, RequestPath]] = None 1739 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters")
pagination_strategy: Union[CursorPagination, CustomPaginationStrategy, OffsetIncrement, PageIncrement]
page_size_option: Optional[RequestOption]
page_token_option: Union[RequestOption, RequestPath, NoneType]
class
SessionTokenRequestApiKeyAuthenticator(pydantic.v1.main.BaseModel):
1742class SessionTokenRequestApiKeyAuthenticator(BaseModel): 1743 type: Literal["ApiKey"] 1744 inject_into: RequestOption = Field( 1745 ..., 1746 description="Configure how the API Key will be sent in requests to the source API.", 1747 examples=[ 1748 {"inject_into": "header", "field_name": "Authorization"}, 1749 {"inject_into": "request_parameter", "field_name": "authKey"}, 1750 ], 1751 title="Inject API Key Into Outgoing HTTP Request", 1752 )
inject_into: RequestOption
class
ListPartitionRouter(pydantic.v1.main.BaseModel):
1755class ListPartitionRouter(BaseModel): 1756 type: Literal["ListPartitionRouter"] 1757 cursor_field: str = Field( 1758 ..., 1759 description='While iterating over list values, the name of field used to reference a list value. The partition value can be accessed with string interpolation. e.g. "{{ stream_partition[\'my_key\'] }}" where "my_key" is the value of the cursor_field.', 1760 examples=["section", "{{ config['section_key'] }}"], 1761 title="Current Partition Value Identifier", 1762 ) 1763 values: Union[str, List[str]] = Field( 1764 ..., 1765 description="The list of attributes being iterated over and used as input for the requests made to the source API.", 1766 examples=[["section_a", "section_b", "section_c"], "{{ config['sections'] }}"], 1767 title="Partition Values", 1768 ) 1769 request_option: Optional[RequestOption] = Field( 1770 None, 1771 description="A request option describing where the list value should be injected into and under what field name if applicable.", 1772 title="Inject Partition Value Into Outgoing HTTP Request", 1773 ) 1774 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters")
request_option: Optional[RequestOption]
class
RecordSelector(pydantic.v1.main.BaseModel):
1777class RecordSelector(BaseModel): 1778 type: Literal["RecordSelector"] 1779 extractor: Union[CustomRecordExtractor, DpathExtractor] 1780 record_filter: Optional[Union[CustomRecordFilter, RecordFilter]] = Field( 1781 None, 1782 description="Responsible for filtering records to be emitted by the Source.", 1783 title="Record Filter", 1784 ) 1785 schema_normalization: Optional[Union[SchemaNormalization, CustomSchemaNormalization]] = Field( 1786 SchemaNormalization.None_, 1787 description="Responsible for normalization according to the schema.", 1788 title="Schema Normalization", 1789 ) 1790 transform_before_filtering: Optional[bool] = Field( 1791 False, 1792 description="If true, transformation will be applied before record filtering.", 1793 ) 1794 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters")
extractor: Union[CustomRecordExtractor, DpathExtractor]
record_filter: Union[CustomRecordFilter, RecordFilter, NoneType]
schema_normalization: Union[SchemaNormalization, CustomSchemaNormalization, NoneType]
class
GzipDecoder(pydantic.v1.main.BaseModel):
1797class GzipDecoder(BaseModel): 1798 type: Literal["GzipDecoder"] 1799 decoder: Union[CsvDecoder, GzipDecoder, JsonDecoder, JsonlDecoder]
decoder: Union[CsvDecoder, GzipDecoder, JsonDecoder, JsonlDecoder]
class
Spec(pydantic.v1.main.BaseModel):
1802class Spec(BaseModel): 1803 type: Literal["Spec"] 1804 connection_specification: Dict[str, Any] = Field( 1805 ..., 1806 description="A connection specification describing how a the connector can be configured.", 1807 title="Connection Specification", 1808 ) 1809 documentation_url: Optional[str] = Field( 1810 None, 1811 description="URL of the connector's documentation page.", 1812 examples=["https://docs.airbyte.com/integrations/sources/dremio"], 1813 title="Documentation URL", 1814 ) 1815 advanced_auth: Optional[AuthFlow] = Field( 1816 None, 1817 description="Advanced specification for configuring the authentication flow.", 1818 title="Advanced Auth", 1819 )
advanced_auth: Optional[AuthFlow]
class
CompositeErrorHandler(pydantic.v1.main.BaseModel):
1822class CompositeErrorHandler(BaseModel): 1823 type: Literal["CompositeErrorHandler"] 1824 error_handlers: List[Union[CompositeErrorHandler, DefaultErrorHandler]] = Field( 1825 ..., 1826 description="List of error handlers to iterate on to determine how to handle a failed response.", 1827 title="Error Handlers", 1828 ) 1829 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters")
error_handlers: List[Union[CompositeErrorHandler, DefaultErrorHandler]]
class
HTTPAPIBudget(pydantic.v1.main.BaseModel):
1832class HTTPAPIBudget(BaseModel): 1833 class Config: 1834 extra = Extra.allow 1835 1836 type: Literal["HTTPAPIBudget"] 1837 policies: List[ 1838 Union[ 1839 FixedWindowCallRatePolicy, 1840 MovingWindowCallRatePolicy, 1841 UnlimitedCallRatePolicy, 1842 ] 1843 ] = Field( 1844 ..., 1845 description="List of call rate policies that define how many calls are allowed.", 1846 title="Policies", 1847 ) 1848 ratelimit_reset_header: Optional[str] = Field( 1849 "ratelimit-reset", 1850 description="The HTTP response header name that indicates when the rate limit resets.", 1851 title="Rate Limit Reset Header", 1852 ) 1853 ratelimit_remaining_header: Optional[str] = Field( 1854 "ratelimit-remaining", 1855 description="The HTTP response header name that indicates the number of remaining allowed calls.", 1856 title="Rate Limit Remaining Header", 1857 ) 1858 status_codes_for_ratelimit_hit: Optional[List[int]] = Field( 1859 [429], 1860 description="List of HTTP status codes that indicate a rate limit has been hit.", 1861 title="Status Codes for Rate Limit Hit", 1862 )
policies: List[Union[FixedWindowCallRatePolicy, MovingWindowCallRatePolicy, UnlimitedCallRatePolicy]]
class
HTTPAPIBudget.Config:
class
ZipfileDecoder(pydantic.v1.main.BaseModel):
1865class ZipfileDecoder(BaseModel): 1866 class Config: 1867 extra = Extra.allow 1868 1869 type: Literal["ZipfileDecoder"] 1870 decoder: Union[CsvDecoder, GzipDecoder, JsonDecoder, JsonlDecoder] = Field( 1871 ..., 1872 description="Parser to parse the decompressed data from the zipfile(s).", 1873 title="Parser", 1874 )
decoder: Union[CsvDecoder, GzipDecoder, JsonDecoder, JsonlDecoder]
class
ZipfileDecoder.Config:
class
DeclarativeSource1(pydantic.v1.main.BaseModel):
1877class DeclarativeSource1(BaseModel): 1878 class Config: 1879 extra = Extra.forbid 1880 1881 type: Literal["DeclarativeSource"] 1882 check: Union[CheckStream, CheckDynamicStream] 1883 streams: List[Union[DeclarativeStream, StateDelegatingStream]] 1884 dynamic_streams: Optional[List[DynamicDeclarativeStream]] = None 1885 version: str = Field( 1886 ..., 1887 description="The version of the Airbyte CDK used to build and test the source.", 1888 ) 1889 schemas: Optional[Schemas] = None 1890 definitions: Optional[Dict[str, Any]] = None 1891 spec: Optional[Spec] = None 1892 concurrency_level: Optional[ConcurrencyLevel] = None 1893 api_budget: Optional[HTTPAPIBudget] = None 1894 max_concurrent_async_job_count: Optional[Union[int, str]] = Field( 1895 None, 1896 description="Maximum number of concurrent asynchronous jobs to run. This property is only relevant for sources/streams that support asynchronous job execution through the AsyncRetriever (e.g. a report-based stream that initiates a job, polls the job status, and then fetches the job results). This is often set by the API's maximum number of concurrent jobs on the account level. Refer to the API's documentation for this information.", 1897 examples=[3, "{{ config['max_concurrent_async_job_count'] }}"], 1898 title="Maximum Concurrent Asynchronous Jobs", 1899 ) 1900 metadata: Optional[Dict[str, Any]] = Field( 1901 None, 1902 description="For internal Airbyte use only - DO NOT modify manually. Used by consumers of declarative manifests for storing related metadata.", 1903 ) 1904 description: Optional[str] = Field( 1905 None, 1906 description="A description of the connector. It will be presented on the Source documentation page.", 1907 )
check: Union[CheckStream, CheckDynamicStream]
streams: List[Union[DeclarativeStream, StateDelegatingStream]]
dynamic_streams: Optional[List[DynamicDeclarativeStream]]
schemas: Optional[Schemas]
spec: Optional[Spec]
concurrency_level: Optional[ConcurrencyLevel]
api_budget: Optional[HTTPAPIBudget]
class
DeclarativeSource1.Config:
class
DeclarativeSource2(pydantic.v1.main.BaseModel):
1910class DeclarativeSource2(BaseModel): 1911 class Config: 1912 extra = Extra.forbid 1913 1914 type: Literal["DeclarativeSource"] 1915 check: Union[CheckStream, CheckDynamicStream] 1916 streams: Optional[List[Union[DeclarativeStream, StateDelegatingStream]]] = None 1917 dynamic_streams: List[DynamicDeclarativeStream] 1918 version: str = Field( 1919 ..., 1920 description="The version of the Airbyte CDK used to build and test the source.", 1921 ) 1922 schemas: Optional[Schemas] = None 1923 definitions: Optional[Dict[str, Any]] = None 1924 spec: Optional[Spec] = None 1925 concurrency_level: Optional[ConcurrencyLevel] = None 1926 api_budget: Optional[HTTPAPIBudget] = None 1927 max_concurrent_async_job_count: Optional[Union[int, str]] = Field( 1928 None, 1929 description="Maximum number of concurrent asynchronous jobs to run. This property is only relevant for sources/streams that support asynchronous job execution through the AsyncRetriever (e.g. a report-based stream that initiates a job, polls the job status, and then fetches the job results). This is often set by the API's maximum number of concurrent jobs on the account level. Refer to the API's documentation for this information.", 1930 examples=[3, "{{ config['max_concurrent_async_job_count'] }}"], 1931 title="Maximum Concurrent Asynchronous Jobs", 1932 ) 1933 metadata: Optional[Dict[str, Any]] = Field( 1934 None, 1935 description="For internal Airbyte use only - DO NOT modify manually. Used by consumers of declarative manifests for storing related metadata.", 1936 ) 1937 description: Optional[str] = Field( 1938 None, 1939 description="A description of the connector. It will be presented on the Source documentation page.", 1940 )
check: Union[CheckStream, CheckDynamicStream]
streams: Optional[List[Union[DeclarativeStream, StateDelegatingStream]]]
dynamic_streams: List[DynamicDeclarativeStream]
schemas: Optional[Schemas]
spec: Optional[Spec]
concurrency_level: Optional[ConcurrencyLevel]
api_budget: Optional[HTTPAPIBudget]
class
DeclarativeSource2.Config:
class
DeclarativeSource(pydantic.v1.main.BaseModel):
1943class DeclarativeSource(BaseModel): 1944 class Config: 1945 extra = Extra.forbid 1946 1947 __root__: Union[DeclarativeSource1, DeclarativeSource2] = Field( 1948 ..., 1949 description="An API source that extracts data according to its declarative components.", 1950 title="DeclarativeSource", 1951 )
class
DeclarativeSource.Config:
class
SelectiveAuthenticator(pydantic.v1.main.BaseModel):
1954class SelectiveAuthenticator(BaseModel): 1955 class Config: 1956 extra = Extra.allow 1957 1958 type: Literal["SelectiveAuthenticator"] 1959 authenticator_selection_path: List[str] = Field( 1960 ..., 1961 description="Path of the field in config with selected authenticator name", 1962 examples=[["auth"], ["auth", "type"]], 1963 title="Authenticator Selection Path", 1964 ) 1965 authenticators: Dict[ 1966 str, 1967 Union[ 1968 ApiKeyAuthenticator, 1969 BasicHttpAuthenticator, 1970 BearerAuthenticator, 1971 CustomAuthenticator, 1972 OAuthAuthenticator, 1973 JwtAuthenticator, 1974 NoAuth, 1975 SessionTokenAuthenticator, 1976 LegacySessionTokenAuthenticator, 1977 ], 1978 ] = Field( 1979 ..., 1980 description="Authenticators to select from.", 1981 examples=[ 1982 { 1983 "authenticators": { 1984 "token": "#/definitions/ApiKeyAuthenticator", 1985 "oauth": "#/definitions/OAuthAuthenticator", 1986 "jwt": "#/definitions/JwtAuthenticator", 1987 } 1988 } 1989 ], 1990 title="Authenticators", 1991 ) 1992 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters")
authenticators: Dict[str, Union[ApiKeyAuthenticator, BasicHttpAuthenticator, BearerAuthenticator, CustomAuthenticator, OAuthAuthenticator, JwtAuthenticator, NoAuth, SessionTokenAuthenticator, LegacySessionTokenAuthenticator]]
class
SelectiveAuthenticator.Config:
class
DeclarativeStream(pydantic.v1.main.BaseModel):
1995class DeclarativeStream(BaseModel): 1996 class Config: 1997 extra = Extra.allow 1998 1999 type: Literal["DeclarativeStream"] 2000 retriever: Union[AsyncRetriever, CustomRetriever, SimpleRetriever] = Field( 2001 ..., 2002 description="Component used to coordinate how records are extracted across stream slices and request pages.", 2003 title="Retriever", 2004 ) 2005 incremental_sync: Optional[ 2006 Union[CustomIncrementalSync, DatetimeBasedCursor, IncrementingCountCursor] 2007 ] = Field( 2008 None, 2009 description="Component used to fetch data incrementally based on a time field in the data.", 2010 title="Incremental Sync", 2011 ) 2012 name: Optional[str] = Field("", description="The stream name.", example=["Users"], title="Name") 2013 primary_key: Optional[PrimaryKey] = Field( 2014 "", description="The primary key of the stream.", title="Primary Key" 2015 ) 2016 schema_loader: Optional[ 2017 Union[ 2018 DynamicSchemaLoader, 2019 InlineSchemaLoader, 2020 JsonFileSchemaLoader, 2021 CustomSchemaLoader, 2022 ] 2023 ] = Field( 2024 None, 2025 description="Component used to retrieve the schema for the current stream.", 2026 title="Schema Loader", 2027 ) 2028 transformations: Optional[ 2029 List[ 2030 Union[ 2031 AddFields, 2032 CustomTransformation, 2033 RemoveFields, 2034 KeysToLower, 2035 KeysToSnakeCase, 2036 FlattenFields, 2037 DpathFlattenFields, 2038 KeysReplace, 2039 ] 2040 ] 2041 ] = Field( 2042 None, 2043 description="A list of transformations to be applied to each output record.", 2044 title="Transformations", 2045 ) 2046 state_migrations: Optional[ 2047 List[Union[LegacyToPerPartitionStateMigration, CustomStateMigration]] 2048 ] = Field( 2049 [], 2050 description="Array of state migrations to be applied on the input state", 2051 title="State Migrations", 2052 ) 2053 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters")
retriever: Union[AsyncRetriever, CustomRetriever, SimpleRetriever]
incremental_sync: Union[IncrementingCountCursor, DatetimeBasedCursor, CustomIncrementalSync, NoneType]
primary_key: Optional[PrimaryKey]
schema_loader: Union[DynamicSchemaLoader, InlineSchemaLoader, JsonFileSchemaLoader, CustomSchemaLoader, NoneType]
transformations: Optional[List[Union[AddFields, CustomTransformation, RemoveFields, KeysToLower, KeysToSnakeCase, FlattenFields, DpathFlattenFields, KeysReplace]]]
state_migrations: Optional[List[Union[LegacyToPerPartitionStateMigration, CustomStateMigration]]]
class
DeclarativeStream.Config:
class
SessionTokenAuthenticator(pydantic.v1.main.BaseModel):
2056class SessionTokenAuthenticator(BaseModel): 2057 type: Literal["SessionTokenAuthenticator"] 2058 login_requester: HttpRequester = Field( 2059 ..., 2060 description="Description of the request to perform to obtain a session token to perform data requests. The response body is expected to be a JSON object with a session token property.", 2061 examples=[ 2062 { 2063 "type": "HttpRequester", 2064 "url_base": "https://my_api.com", 2065 "path": "/login", 2066 "authenticator": { 2067 "type": "BasicHttpAuthenticator", 2068 "username": "{{ config.username }}", 2069 "password": "{{ config.password }}", 2070 }, 2071 } 2072 ], 2073 title="Login Requester", 2074 ) 2075 session_token_path: List[str] = Field( 2076 ..., 2077 description="The path in the response body returned from the login requester to the session token.", 2078 examples=[["access_token"], ["result", "token"]], 2079 title="Session Token Path", 2080 ) 2081 expiration_duration: Optional[str] = Field( 2082 None, 2083 description="The duration in ISO 8601 duration notation after which the session token expires, starting from the time it was obtained. Omitting it will result in the session token being refreshed for every request.", 2084 examples=["PT1H", "P1D"], 2085 title="Expiration Duration", 2086 ) 2087 request_authentication: Union[ 2088 SessionTokenRequestApiKeyAuthenticator, SessionTokenRequestBearerAuthenticator 2089 ] = Field( 2090 ..., 2091 description="Authentication method to use for requests sent to the API, specifying how to inject the session token.", 2092 title="Data Request Authentication", 2093 ) 2094 decoder: Optional[Union[JsonDecoder, XmlDecoder]] = Field( 2095 None, description="Component used to decode the response.", title="Decoder" 2096 ) 2097 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters")
login_requester: HttpRequester
request_authentication: Union[SessionTokenRequestApiKeyAuthenticator, SessionTokenRequestBearerAuthenticator]
decoder: Union[JsonDecoder, XmlDecoder, NoneType]
class
HttpRequester(pydantic.v1.main.BaseModel):
2100class HttpRequester(BaseModel): 2101 type: Literal["HttpRequester"] 2102 url_base: str = Field( 2103 ..., 2104 description="Base URL of the API source. Do not put sensitive information (e.g. API tokens) into this field - Use the Authentication component for this.", 2105 examples=[ 2106 "https://connect.squareup.com/v2", 2107 "{{ config['base_url'] or 'https://app.posthog.com'}}/api", 2108 "https://connect.squareup.com/v2/quotes/{{ stream_partition['id'] }}/quote_line_groups", 2109 "https://example.com/api/v1/resource/{{ next_page_token['id'] }}", 2110 ], 2111 title="API Base URL", 2112 ) 2113 path: Optional[str] = Field( 2114 None, 2115 description="Path the specific API endpoint that this stream represents. Do not put sensitive information (e.g. API tokens) into this field - Use the Authentication component for this.", 2116 examples=[ 2117 "/products", 2118 "/quotes/{{ stream_partition['id'] }}/quote_line_groups", 2119 "/trades/{{ config['symbol_id'] }}/history", 2120 ], 2121 title="URL Path", 2122 ) 2123 authenticator: Optional[ 2124 Union[ 2125 ApiKeyAuthenticator, 2126 BasicHttpAuthenticator, 2127 BearerAuthenticator, 2128 CustomAuthenticator, 2129 OAuthAuthenticator, 2130 JwtAuthenticator, 2131 NoAuth, 2132 SessionTokenAuthenticator, 2133 LegacySessionTokenAuthenticator, 2134 SelectiveAuthenticator, 2135 ] 2136 ] = Field( 2137 None, 2138 description="Authentication method to use for requests sent to the API.", 2139 title="Authenticator", 2140 ) 2141 error_handler: Optional[ 2142 Union[DefaultErrorHandler, CustomErrorHandler, CompositeErrorHandler] 2143 ] = Field( 2144 None, 2145 description="Error handler component that defines how to handle errors.", 2146 title="Error Handler", 2147 ) 2148 http_method: Optional[HttpMethod] = Field( 2149 HttpMethod.GET, 2150 description="The HTTP method used to fetch data from the source (can be GET or POST).", 2151 examples=["GET", "POST"], 2152 title="HTTP Method", 2153 ) 2154 request_body_data: Optional[Union[str, Dict[str, str]]] = Field( 2155 None, 2156 description="Specifies how to populate the body of the request with a non-JSON payload. Plain text will be sent as is, whereas objects will be converted to a urlencoded form.", 2157 examples=[ 2158 '[{"clause": {"type": "timestamp", "operator": 10, "parameters":\n [{"value": {{ stream_interval[\'start_time\'] | int * 1000 }} }]\n }, "orderBy": 1, "columnName": "Timestamp"}]/\n' 2159 ], 2160 title="Request Body Payload (Non-JSON)", 2161 ) 2162 request_body_json: Optional[Union[str, Dict[str, Any]]] = Field( 2163 None, 2164 description="Specifies how to populate the body of the request with a JSON payload. Can contain nested objects.", 2165 examples=[ 2166 {"sort_order": "ASC", "sort_field": "CREATED_AT"}, 2167 {"key": "{{ config['value'] }}"}, 2168 {"sort": {"field": "updated_at", "order": "ascending"}}, 2169 ], 2170 title="Request Body JSON Payload", 2171 ) 2172 request_headers: Optional[Union[str, Dict[str, str]]] = Field( 2173 None, 2174 description="Return any non-auth headers. Authentication headers will overwrite any overlapping headers returned from this method.", 2175 examples=[{"Output-Format": "JSON"}, {"Version": "{{ config['version'] }}"}], 2176 title="Request Headers", 2177 ) 2178 request_parameters: Optional[Union[str, Dict[str, str]]] = Field( 2179 None, 2180 description="Specifies the query parameters that should be set on an outgoing HTTP request given the inputs.", 2181 examples=[ 2182 {"unit": "day"}, 2183 { 2184 "query": 'last_event_time BETWEEN TIMESTAMP "{{ stream_interval.start_time }}" AND TIMESTAMP "{{ stream_interval.end_time }}"' 2185 }, 2186 {"searchIn": "{{ ','.join(config.get('search_in', [])) }}"}, 2187 {"sort_by[asc]": "updated_at"}, 2188 ], 2189 title="Query Parameters", 2190 ) 2191 use_cache: Optional[bool] = Field( 2192 False, 2193 description="Enables stream requests caching. This field is automatically set by the CDK.", 2194 title="Use Cache", 2195 ) 2196 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters")
authenticator: Union[ApiKeyAuthenticator, BasicHttpAuthenticator, BearerAuthenticator, CustomAuthenticator, OAuthAuthenticator, JwtAuthenticator, NoAuth, SessionTokenAuthenticator, LegacySessionTokenAuthenticator, SelectiveAuthenticator, NoneType]
error_handler: Union[DefaultErrorHandler, CustomErrorHandler, CompositeErrorHandler, NoneType]
http_method: Optional[HttpMethod]
class
DynamicSchemaLoader(pydantic.v1.main.BaseModel):
2199class DynamicSchemaLoader(BaseModel): 2200 type: Literal["DynamicSchemaLoader"] 2201 retriever: Union[AsyncRetriever, CustomRetriever, SimpleRetriever] = Field( 2202 ..., 2203 description="Component used to coordinate how records are extracted across stream slices and request pages.", 2204 title="Retriever", 2205 ) 2206 schema_transformations: Optional[ 2207 List[ 2208 Union[ 2209 AddFields, 2210 CustomTransformation, 2211 RemoveFields, 2212 KeysToLower, 2213 KeysToSnakeCase, 2214 FlattenFields, 2215 DpathFlattenFields, 2216 KeysReplace, 2217 ] 2218 ] 2219 ] = Field( 2220 None, 2221 description="A list of transformations to be applied to the schema.", 2222 title="Schema Transformations", 2223 ) 2224 schema_type_identifier: SchemaTypeIdentifier 2225 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters")
retriever: Union[AsyncRetriever, CustomRetriever, SimpleRetriever]
schema_transformations: Optional[List[Union[AddFields, CustomTransformation, RemoveFields, KeysToLower, KeysToSnakeCase, FlattenFields, DpathFlattenFields, KeysReplace]]]
schema_type_identifier: SchemaTypeIdentifier
class
ParentStreamConfig(pydantic.v1.main.BaseModel):
2228class ParentStreamConfig(BaseModel): 2229 type: Literal["ParentStreamConfig"] 2230 lazy_read_pointer: Optional[List[str]] = Field( 2231 [], 2232 description="If set, this will enable lazy reading, using the initial read of parent records to extract child records.", 2233 title="Lazy Read Pointer", 2234 ) 2235 parent_key: str = Field( 2236 ..., 2237 description="The primary key of records from the parent stream that will be used during the retrieval of records for the current substream. This parent identifier field is typically a characteristic of the child records being extracted from the source API.", 2238 examples=["id", "{{ config['parent_record_id'] }}"], 2239 title="Parent Key", 2240 ) 2241 stream: Union[DeclarativeStream, StateDelegatingStream] = Field( 2242 ..., description="Reference to the parent stream.", title="Parent Stream" 2243 ) 2244 partition_field: str = Field( 2245 ..., 2246 description="While iterating over parent records during a sync, the parent_key value can be referenced by using this field.", 2247 examples=["parent_id", "{{ config['parent_partition_field'] }}"], 2248 title="Current Parent Key Value Identifier", 2249 ) 2250 request_option: Optional[RequestOption] = Field( 2251 None, 2252 description="A request option describing where the parent key value should be injected into and under what field name if applicable.", 2253 title="Request Option", 2254 ) 2255 incremental_dependency: Optional[bool] = Field( 2256 False, 2257 description="Indicates whether the parent stream should be read incrementally based on updates in the child stream.", 2258 title="Incremental Dependency", 2259 ) 2260 extra_fields: Optional[List[List[str]]] = Field( 2261 None, 2262 description="Array of field paths to include as additional fields in the stream slice. Each path is an array of strings representing keys to access fields in the respective parent record. Accessible via `stream_slice.extra_fields`. Missing fields are set to `None`.", 2263 title="Extra Fields", 2264 ) 2265 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters")
stream: Union[DeclarativeStream, StateDelegatingStream]
request_option: Optional[RequestOption]
class
StateDelegatingStream(pydantic.v1.main.BaseModel):
2268class StateDelegatingStream(BaseModel): 2269 type: Literal["StateDelegatingStream"] 2270 name: str = Field(..., description="The stream name.", example=["Users"], title="Name") 2271 full_refresh_stream: DeclarativeStream = Field( 2272 ..., 2273 description="Component used to coordinate how records are extracted across stream slices and request pages when the state is empty or not provided.", 2274 title="Retriever", 2275 ) 2276 incremental_stream: DeclarativeStream = Field( 2277 ..., 2278 description="Component used to coordinate how records are extracted across stream slices and request pages when the state provided.", 2279 title="Retriever", 2280 ) 2281 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters")
full_refresh_stream: DeclarativeStream
incremental_stream: DeclarativeStream
class
SimpleRetriever(pydantic.v1.main.BaseModel):
2284class SimpleRetriever(BaseModel): 2285 type: Literal["SimpleRetriever"] 2286 record_selector: RecordSelector = Field( 2287 ..., 2288 description="Component that describes how to extract records from a HTTP response.", 2289 ) 2290 requester: Union[CustomRequester, HttpRequester] = Field( 2291 ..., 2292 description="Requester component that describes how to prepare HTTP requests to send to the source API.", 2293 ) 2294 paginator: Optional[Union[DefaultPaginator, NoPagination]] = Field( 2295 None, 2296 description="Paginator component that describes how to navigate through the API's pages.", 2297 ) 2298 ignore_stream_slicer_parameters_on_paginated_requests: Optional[bool] = Field( 2299 False, 2300 description="If true, the partition router and incremental request options will be ignored when paginating requests. Request options set directly on the requester will not be ignored.", 2301 ) 2302 partition_router: Optional[ 2303 Union[ 2304 CustomPartitionRouter, 2305 ListPartitionRouter, 2306 SubstreamPartitionRouter, 2307 List[Union[CustomPartitionRouter, ListPartitionRouter, SubstreamPartitionRouter]], 2308 ] 2309 ] = Field( 2310 [], 2311 description="PartitionRouter component that describes how to partition the stream, enabling incremental syncs and checkpointing.", 2312 title="Partition Router", 2313 ) 2314 decoder: Optional[ 2315 Union[ 2316 CustomDecoder, 2317 CsvDecoder, 2318 GzipDecoder, 2319 JsonDecoder, 2320 JsonlDecoder, 2321 IterableDecoder, 2322 XmlDecoder, 2323 ZipfileDecoder, 2324 ] 2325 ] = Field( 2326 None, 2327 description="Component decoding the response so records can be extracted.", 2328 title="Decoder", 2329 ) 2330 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters")
record_selector: RecordSelector
requester: Union[CustomRequester, HttpRequester]
paginator: Union[DefaultPaginator, NoPagination, NoneType]
partition_router: Union[CustomPartitionRouter, ListPartitionRouter, SubstreamPartitionRouter, List[Union[CustomPartitionRouter, ListPartitionRouter, SubstreamPartitionRouter]], NoneType]
decoder: Union[CustomDecoder, CsvDecoder, GzipDecoder, JsonDecoder, JsonlDecoder, IterableDecoder, XmlDecoder, ZipfileDecoder, NoneType]
class
AsyncRetriever(pydantic.v1.main.BaseModel):
2333class AsyncRetriever(BaseModel): 2334 type: Literal["AsyncRetriever"] 2335 record_selector: RecordSelector = Field( 2336 ..., 2337 description="Component that describes how to extract records from a HTTP response.", 2338 ) 2339 status_mapping: AsyncJobStatusMap = Field( 2340 ..., description="Async Job Status to Airbyte CDK Async Job Status mapping." 2341 ) 2342 status_extractor: Union[CustomRecordExtractor, DpathExtractor] = Field( 2343 ..., description="Responsible for fetching the actual status of the async job." 2344 ) 2345 download_target_extractor: Union[CustomRecordExtractor, DpathExtractor] = Field( 2346 ..., 2347 description="Responsible for fetching the final result `urls` provided by the completed / finished / ready async job.", 2348 ) 2349 download_extractor: Optional[ 2350 Union[CustomRecordExtractor, DpathExtractor, ResponseToFileExtractor] 2351 ] = Field(None, description="Responsible for fetching the records from provided urls.") 2352 creation_requester: Union[CustomRequester, HttpRequester] = Field( 2353 ..., 2354 description="Requester component that describes how to prepare HTTP requests to send to the source API to create the async server-side job.", 2355 ) 2356 polling_requester: Union[CustomRequester, HttpRequester] = Field( 2357 ..., 2358 description="Requester component that describes how to prepare HTTP requests to send to the source API to fetch the status of the running async job.", 2359 ) 2360 polling_job_timeout: Optional[Union[int, str]] = Field( 2361 None, 2362 description="The time in minutes after which the single Async Job should be considered as Timed Out.", 2363 ) 2364 download_target_requester: Optional[Union[CustomRequester, HttpRequester]] = Field( 2365 None, 2366 description="Requester component that describes how to prepare HTTP requests to send to the source API to extract the url from polling response by the completed async job.", 2367 ) 2368 download_requester: Union[CustomRequester, HttpRequester] = Field( 2369 ..., 2370 description="Requester component that describes how to prepare HTTP requests to send to the source API to download the data provided by the completed async job.", 2371 ) 2372 download_paginator: Optional[Union[DefaultPaginator, NoPagination]] = Field( 2373 None, 2374 description="Paginator component that describes how to navigate through the API's pages during download.", 2375 ) 2376 abort_requester: Optional[Union[CustomRequester, HttpRequester]] = Field( 2377 None, 2378 description="Requester component that describes how to prepare HTTP requests to send to the source API to abort a job once it is timed out from the source's perspective.", 2379 ) 2380 delete_requester: Optional[Union[CustomRequester, HttpRequester]] = Field( 2381 None, 2382 description="Requester component that describes how to prepare HTTP requests to send to the source API to delete a job once the records are extracted.", 2383 ) 2384 partition_router: Optional[ 2385 Union[ 2386 CustomPartitionRouter, 2387 ListPartitionRouter, 2388 SubstreamPartitionRouter, 2389 List[Union[CustomPartitionRouter, ListPartitionRouter, SubstreamPartitionRouter]], 2390 ] 2391 ] = Field( 2392 [], 2393 description="PartitionRouter component that describes how to partition the stream, enabling incremental syncs and checkpointing.", 2394 title="Partition Router", 2395 ) 2396 decoder: Optional[ 2397 Union[ 2398 CustomDecoder, 2399 CsvDecoder, 2400 GzipDecoder, 2401 JsonDecoder, 2402 JsonlDecoder, 2403 IterableDecoder, 2404 XmlDecoder, 2405 ZipfileDecoder, 2406 ] 2407 ] = Field( 2408 None, 2409 description="Component decoding the response so records can be extracted.", 2410 title="Decoder", 2411 ) 2412 download_decoder: Optional[ 2413 Union[ 2414 CustomDecoder, 2415 CsvDecoder, 2416 GzipDecoder, 2417 JsonDecoder, 2418 JsonlDecoder, 2419 IterableDecoder, 2420 XmlDecoder, 2421 ZipfileDecoder, 2422 ] 2423 ] = Field( 2424 None, 2425 description="Component decoding the download response so records can be extracted.", 2426 title="Download Decoder", 2427 ) 2428 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters")
record_selector: RecordSelector
status_mapping: AsyncJobStatusMap
status_extractor: Union[CustomRecordExtractor, DpathExtractor]
download_target_extractor: Union[CustomRecordExtractor, DpathExtractor]
download_extractor: Union[CustomRecordExtractor, DpathExtractor, ResponseToFileExtractor, NoneType]
creation_requester: Union[CustomRequester, HttpRequester]
polling_requester: Union[CustomRequester, HttpRequester]
download_target_requester: Union[CustomRequester, HttpRequester, NoneType]
download_requester: Union[CustomRequester, HttpRequester]
download_paginator: Union[DefaultPaginator, NoPagination, NoneType]
abort_requester: Union[CustomRequester, HttpRequester, NoneType]
delete_requester: Union[CustomRequester, HttpRequester, NoneType]
partition_router: Union[CustomPartitionRouter, ListPartitionRouter, SubstreamPartitionRouter, List[Union[CustomPartitionRouter, ListPartitionRouter, SubstreamPartitionRouter]], NoneType]
decoder: Union[CustomDecoder, CsvDecoder, GzipDecoder, JsonDecoder, JsonlDecoder, IterableDecoder, XmlDecoder, ZipfileDecoder, NoneType]
download_decoder: Union[CustomDecoder, CsvDecoder, GzipDecoder, JsonDecoder, JsonlDecoder, IterableDecoder, XmlDecoder, ZipfileDecoder, NoneType]
class
SubstreamPartitionRouter(pydantic.v1.main.BaseModel):
2431class SubstreamPartitionRouter(BaseModel): 2432 type: Literal["SubstreamPartitionRouter"] 2433 parent_stream_configs: List[ParentStreamConfig] = Field( 2434 ..., 2435 description="Specifies which parent streams are being iterated over and how parent records should be used to partition the child stream data set.", 2436 title="Parent Stream Configs", 2437 ) 2438 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters")
parent_stream_configs: List[ParentStreamConfig]
class
HttpComponentsResolver(pydantic.v1.main.BaseModel):
2441class HttpComponentsResolver(BaseModel): 2442 type: Literal["HttpComponentsResolver"] 2443 retriever: Union[AsyncRetriever, CustomRetriever, SimpleRetriever] = Field( 2444 ..., 2445 description="Component used to coordinate how records are extracted across stream slices and request pages.", 2446 title="Retriever", 2447 ) 2448 components_mapping: List[ComponentMappingDefinition] 2449 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters")
retriever: Union[AsyncRetriever, CustomRetriever, SimpleRetriever]
components_mapping: List[ComponentMappingDefinition]
class
DynamicDeclarativeStream(pydantic.v1.main.BaseModel):
2452class DynamicDeclarativeStream(BaseModel): 2453 type: Literal["DynamicDeclarativeStream"] 2454 stream_template: DeclarativeStream = Field( 2455 ..., description="Reference to the stream template.", title="Stream Template" 2456 ) 2457 components_resolver: Union[HttpComponentsResolver, ConfigComponentsResolver] = Field( 2458 ..., 2459 description="Component resolve and populates stream templates with components values.", 2460 title="Components Resolver", 2461 )
stream_template: DeclarativeStream
components_resolver: Union[HttpComponentsResolver, ConfigComponentsResolver]